mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-06-02 13:29:49 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: runqueue: Rework process_possible_migrations() to improve performance

Browse Source 
The looping over multiple changed hashes causes many calls to get_taskhash
and get_unihash which are potentially slow and then overwritten.

Instead, batch up all the tasks which have changed unihashes and then
do one big loop over the changed tasks rather than each in turn.

This makes worlds of difference to the performance graphs and should speed
up build where many tasks are being rehashed.

(Bitbake rev: c9ab598f6f1ea3ae3a0713dc6692b4c4bafbfb50)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit c9c68d898985cf0bec6fc95f54c151cc50255cac)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2019-12-13 16:06:47 +00:00
parent 7278f3f786
commit 57efd2721b
1 changed files with 52 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/runqueue.py
+15 -6
View File
@@ -2248,6 +2248,7 @@ class RunQueueExecute:
def process_possible_migrations(self): def process_possible_migrations(self):
changed = set() changed = set()
toprocess = set()
for tid, unihash in self.updated_taskhash_queue.copy(): for tid, unihash in self.updated_taskhash_queue.copy():
if tid in self.runq_running and tid not in self.runq_complete: if tid in self.runq_running and tid not in self.runq_complete:
continue continue
@@ -2258,21 +2259,29 @@ class RunQueueExecute:
logger.info("Task %s unihash changed to %s" % (tid, unihash)) logger.info("Task %s unihash changed to %s" % (tid, unihash))
self.rqdata.runtaskentries[tid].unihash = unihash self.rqdata.runtaskentries[tid].unihash = unihash
bb.parse.siggen.set_unihash(tid, unihash) bb.parse.siggen.set_unihash(tid, unihash)
toprocess.add(tid)
# Work out all tasks which depend on this one # Work out all tasks which depend upon these
total = set() total = set()
next = set(self.rqdata.runtaskentries[tid].revdeps) for p in toprocess:
next = set(self.rqdata.runtaskentries[p].revdeps)
while next: while next:
current = next.copy() current = next.copy()
total = total |next total = total | next
next = set() next = set()
for ntid in current: for ntid in current:
next |= self.rqdata.runtaskentries[ntid].revdeps next |= self.rqdata.runtaskentries[ntid].revdeps
next.difference_update(total) next.difference_update(total)
# Now iterate those tasks in dependency order to regenerate their taskhash/unihash # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
done = set() next = set()
next = set(self.rqdata.runtaskentries[tid].revdeps) for p in total:
if len(self.rqdata.runtaskentries[p].depends) == 0:
next.add(p)
elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
next.add(p)
# When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
while next: while next:
current = next.copy() current = next.copy()
next = set() next = set()
@@ -2297,7 +2306,7 @@ class RunQueueExecute:
logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid)) logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
if not remapped: if not remapped:
logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
self.rqdata.runtaskentries[tid].hash = newhash self.rqdata.runtaskentries[tid].hash = newhash
self.rqdata.runtaskentries[tid].unihash = newuni self.rqdata.runtaskentries[tid].unihash = newuni
changed.add(tid) changed.add(tid)