mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-06-02 13:29:49 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: runqueue: Rework process_possible_migrations() to improve performance

Browse Source 
The looping over multiple changed hashes causes many calls to get_taskhash
and get_unihash which are potentially slow and then overwritten.

Instead, batch up all the tasks which have changed unihashes and then
do one big loop over the changed tasks rather than each in turn.

This makes worlds of difference to the performance graphs and should speed
up build where many tasks are being rehashed.

(Bitbake rev: c9ab598f6f1ea3ae3a0713dc6692b4c4bafbfb50)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit c9c68d898985cf0bec6fc95f54c151cc50255cac)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2019-12-13 16:06:47 +00:00
parent 7278f3f786
commit 57efd2721b
1 changed files with 52 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/runqueue.py
+52 -43
View File
@@ -2248,6 +2248,7 @@ class RunQueueExecute:
def process_possible_migrations(self): def process_possible_migrations(self):
changed = set() changed = set()
toprocess = set()
for tid, unihash in self.updated_taskhash_queue.copy(): for tid, unihash in self.updated_taskhash_queue.copy():
if tid in self.runq_running and tid not in self.runq_complete: if tid in self.runq_running and tid not in self.runq_complete:
continue continue
@@ -2258,53 +2259,61 @@ class RunQueueExecute:
logger.info("Task %s unihash changed to %s" % (tid, unihash)) logger.info("Task %s unihash changed to %s" % (tid, unihash))
self.rqdata.runtaskentries[tid].unihash = unihash self.rqdata.runtaskentries[tid].unihash = unihash
bb.parse.siggen.set_unihash(tid, unihash) bb.parse.siggen.set_unihash(tid, unihash)
toprocess.add(tid)
# Work out all tasks which depend on this one # Work out all tasks which depend upon these
total = set() total = set()
next = set(self.rqdata.runtaskentries[tid].revdeps) for p in toprocess:
while next: next = set(self.rqdata.runtaskentries[p].revdeps)
current = next.copy() while next:
total = total |next current = next.copy()
next = set() total = total | next
for ntid in current: next = set()
next |= self.rqdata.runtaskentries[ntid].revdeps for ntid in current:
next.difference_update(total) next |= self.rqdata.runtaskentries[ntid].revdeps
next.difference_update(total)
# Now iterate those tasks in dependency order to regenerate their taskhash/unihash # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
done = set() next = set()
next = set(self.rqdata.runtaskentries[tid].revdeps) for p in total:
while next: if len(self.rqdata.runtaskentries[p].depends) == 0:
current = next.copy() next.add(p)
next = set() elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
for tid in current: next.add(p)
if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
continue
procdep = []
for dep in self.rqdata.runtaskentries[tid].depends:
procdep.append(dep)
orighash = self.rqdata.runtaskentries[tid].hash
newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
origuni = self.rqdata.runtaskentries[tid].unihash
newuni = bb.parse.siggen.get_unihash(tid)
# FIXME, need to check it can come from sstate at all for determinism?
remapped = False
if newuni == origuni:
# Nothing to do, we match, skip code below
remapped = True
elif tid in self.scenequeue_covered or tid in self.sq_live:
# Already ran this setscene task or it running. Report the new taskhash
remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
if not remapped: # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) while next:
self.rqdata.runtaskentries[tid].hash = newhash current = next.copy()
self.rqdata.runtaskentries[tid].unihash = newuni next = set()
changed.add(tid) for tid in current:
if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
continue
procdep = []
for dep in self.rqdata.runtaskentries[tid].depends:
procdep.append(dep)
orighash = self.rqdata.runtaskentries[tid].hash
newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
origuni = self.rqdata.runtaskentries[tid].unihash
newuni = bb.parse.siggen.get_unihash(tid)
# FIXME, need to check it can come from sstate at all for determinism?
remapped = False
if newuni == origuni:
# Nothing to do, we match, skip code below
remapped = True
elif tid in self.scenequeue_covered or tid in self.sq_live:
# Already ran this setscene task or it running. Report the new taskhash
remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
next |= self.rqdata.runtaskentries[tid].revdeps if not remapped:
total.remove(tid) #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
next.intersection_update(total) self.rqdata.runtaskentries[tid].hash = newhash
self.rqdata.runtaskentries[tid].unihash = newuni
changed.add(tid)
next |= self.rqdata.runtaskentries[tid].revdeps
total.remove(tid)
next.intersection_update(total)
if changed: if changed:
for mc in self.rq.worker: for mc in self.rq.worker: