mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-08 05:09:24 +00:00
Code Issues Projects Releases Wiki Activity

oeqa: Streamline oe-selftest startup time

Browse Source 
"bitbake -e" executions from get_bb_var calls are slow and slow down oe-selftest
startup. Rationalise the code to avoid them and minimise the number of "parsing"
locations we use by caching key variables and passing them around more.

This was particularly problematic with oe-selftest -j usage since it would
have multiple bitbake -e executions per process making parallel usage
particularly slow.

(From OE-Core rev: 3689cadeb07d76e66f97d890e844f899f69666fe)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2023-09-21 17:49:25 +01:00
parent fbaa7da33b
commit 7e6514b28b
3 changed files with 28 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
meta/lib/oeqa/core/utils/concurrencytest.py
+3 -2
View File
@@ -193,11 +193,12 @@ class dummybuf(object):
#
class ConcurrentTestSuite(unittest.TestSuite):
def __init__(self, suite, processes, setupfunc, removefunc):
def __init__(self, suite, processes, setupfunc, removefunc, bb_vars):
super(ConcurrentTestSuite, self).__init__([suite])
self.processes = processes
self.setupfunc = setupfunc
self.removefunc = removefunc
self.bb_vars = bb_vars
def run(self, result):
testservers, totaltests = fork_for_tests(self.processes, self)
@@ -243,7 +244,7 @@ class ConcurrentTestSuite(unittest.TestSuite):
def fork_for_tests(concurrency_num, suite):
testservers = []
if 'BUILDDIR' in os.environ:
selftestdir = get_test_layer()
selftestdir = get_test_layer(suite.bb_vars['BBLAYERS'])
test_blocks = partition_tests(suite, concurrency_num)
# Clear the tests from the original suite so it doesn't keep them alive
meta/lib/oeqa/selftest/context.py
+21 -14
View File
@@ -35,12 +35,13 @@ def get_oeselftest_metadata(args):
return result
class NonConcurrentTestSuite(unittest.TestSuite):
def __init__(self, suite, processes, setupfunc, removefunc):
def __init__(self, suite, processes, setupfunc, removefunc, bb_vars):
super().__init__([suite])
self.processes = processes
self.suite = suite
self.setupfunc = setupfunc
self.removefunc = removefunc
self.bb_vars = bb_vars
def run(self, result):
(builddir, newbuilddir) = self.setupfunc("-st", None, self.suite)
@@ -79,16 +80,15 @@ class OESelftestTestContext(OETestContext):
else:
self.removebuilddir = removebuilddir
def set_variables(self, vars):
self.bb_vars = vars
def setup_builddir(self, suffix, selftestdir, suite):
# Get SSTATE_DIR from the parent build dir
with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
tinfoil.prepare(quiet=2, config_only=True)
d = tinfoil.config_data
sstatedir = str(d.getVar('SSTATE_DIR'))
sstatedir = self.bb_vars['SSTATE_DIR']
builddir = os.environ['BUILDDIR']
if not selftestdir:
selftestdir = get_test_layer()
selftestdir = get_test_layer(self.bb_vars['BBLAYERS'])
if self.newbuilddir:
newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix)
else:
@@ -155,9 +155,9 @@ class OESelftestTestContext(OETestContext):
if processes:
from oeqa.core.utils.concurrencytest import ConcurrentTestSuite
return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir)
return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars)
else:
return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir)
return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars)
def runTests(self, processes=None, machine=None, skips=[]):
if machine:
@@ -270,7 +270,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
builddir = os.environ.get("BUILDDIR")
self.tc_kwargs['init']['config_paths'] = {}
self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer()
self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"])
self.tc_kwargs['init']['config_paths']['builddir'] = builddir
self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf")
self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf")
@@ -310,10 +310,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
meta_selftestdir = os.path.join(
self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest')
if os.path.isdir(meta_selftestdir):
runCmd("bitbake-layers add-layer %s" %meta_selftestdir)
runCmd("bitbake-layers add-layer %s" % meta_selftestdir)
# reload data is needed because a meta-selftest layer was add
self.tc.td = get_bb_vars()
self.tc.config_paths['testlayer_path'] = get_test_layer()
self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"])
else:
self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir)
raise OEQAPreRun
@@ -351,8 +351,15 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
_add_layer_libs()
self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
runCmd("bitbake -e")
self.tc.logger.info("Checking base configuration is valid/parsable")
with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
tinfoil.prepare(quiet=2, config_only=True)
d = tinfoil.config_data
vars = {}
vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR'))
vars['BBLAYERS'] = str(d.getVar('BBLAYERS'))
self.tc.set_variables(vars)
def get_json_result_dir(self, args):
json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa')
meta/lib/oeqa/utils/commands.py
+4 -2
View File
@@ -285,8 +285,10 @@ def get_bb_vars(variables=None, target=None, postconfig=None):
def get_bb_var(var, target=None, postconfig=None):
return get_bb_vars([var], target, postconfig)[var]
def get_test_layer():
layers = get_bb_var("BBLAYERS").split()
def get_test_layer(bblayers=None):
if bblayers is None:
bblayers = get_bb_var("BBLAYERS")
layers = bblayers.split()
testlayer = None
for l in layers:
if '~' in l: