mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-30 12:29:55 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: runqueue: Fix task migration problems

Browse Source 
Tasks were not migrating consistently, particularly:

* if a task was rehashed which had already run
* if a task which was valid became invalid due to a rehash

We need to always run the migration code for rehashed tasks and then
reprocess them for hash validity. This means rearranging the code.

It also means several tests are no longer correct and can't be written
correctly to work on all possible workflows so those are removed.

(Bitbake rev: 8443989ee41e9b162972935513e437b5c66ea74d)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2019-08-19 15:19:39 +01:00
parent b83d5c15ad
commit d0b7471ba8
2 changed files with 62 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/runqueue.py
+62 -51
View File
@@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname):
def pending_hash_index(tid, rqdata):
(mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
h = rqdata.runtaskentries[tid].hash
h = rqdata.runtaskentries[tid].unihash
return pn + ":" + "taskname" + h
class RunQueueStats:
@@ -2299,9 +2299,6 @@ class RunQueueExecute:
for tid in changed:
if tid not in self.rqdata.runq_setscene_tids:
continue
valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
if not valid:
continue
if tid in self.runq_running:
continue
if tid not in self.pending_migrations:
@@ -2358,6 +2355,7 @@ class RunQueueExecute:
logger.info("Setscene task %s now valid and being rerun" % tid)
self.sqdone = False
update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
if changed:
self.holdoff_need_update = True
@@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
rqdata.init_progress_reporter.next_stage()
multiconfigs = set()
sqdata.multiconfigs = set()
for tid in sqdata.sq_revdeps:
multiconfigs.add(mc_from_tid(tid))
sqdata.multiconfigs.add(mc_from_tid(tid))
if len(sqdata.sq_revdeps[tid]) == 0:
sqrq.sq_buildable.add(tid)
rqdata.init_progress_reporter.finish()
if rq.hashvalidate:
noexec = []
stamppresent = []
tocheck = set()
sqdata.noexec = set()
sqdata.stamppresent = set()
sqdata.valid = set()
update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
tocheck = set()
for tid in sorted(tids):
if tid in sqdata.stamppresent:
sqdata.stamppresent.remove(tid)
if tid in sqdata.valid:
sqdata.valid.remove(tid)
(mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
if 'noexec' in taskdep and taskname in taskdep['noexec']:
sqdata.noexec.add(tid)
sqrq.sq_task_skip(tid)
bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
continue
if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
logger.debug(2, 'Setscene stamp current for task %s', tid)
sqdata.stamppresent.add(tid)
sqrq.sq_task_skip(tid)
continue
if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
logger.debug(2, 'Normal stamp current for task %s', tid)
sqdata.stamppresent.add(tid)
sqrq.sq_task_skip(tid)
continue
tocheck.add(tid)
sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
sqdata.hashes = {}
for mc in sorted(sqdata.multiconfigs):
for tid in sorted(sqdata.sq_revdeps):
(mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
if 'noexec' in taskdep and taskname in taskdep['noexec']:
noexec.append(tid)
sqrq.sq_task_skip(tid)
bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
continue
if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
logger.debug(2, 'Setscene stamp current for task %s', tid)
stamppresent.append(tid)
sqrq.sq_task_skip(tid)
continue
if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
logger.debug(2, 'Normal stamp current for task %s', tid)
stamppresent.append(tid)
sqrq.sq_task_skip(tid)
continue
tocheck.add(tid)
valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False)
valid_new = stamppresent
for v in valid:
valid_new.append(v)
hashes = {}
for mc in sorted(multiconfigs):
for tid in sorted(sqdata.sq_revdeps):
if mc_from_tid(tid) != mc:
continue
if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
sqdata.outrightfail.add(tid)
if tid in sqdata.stamppresent:
continue
if tid in sqdata.valid:
continue
if tid in sqdata.noexec:
continue
if tid in sqrq.scenequeue_notcovered:
continue
sqdata.outrightfail.add(tid)
h = pending_hash_index(tid, rqdata)
if h not in hashes:
hashes[h] = tid
else:
sqrq.sq_deferred[tid] = hashes[h]
bb.warn("Deferring %s after %s" % (tid, hashes[h]))
h = pending_hash_index(tid, rqdata)
if h not in sqdata.hashes:
sqdata.hashes[h] = tid
else:
sqrq.sq_deferred[tid] = sqdata.hashes[h]
bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
class TaskFailure(Exception):