mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-08 17:19:20 +00:00
Code Issues Projects Releases Wiki Activity

cache: sync the cache file to disk in the background

Browse Source 
This version uses a thread rather than a process, to avoid problems with
waitpid handling.  This gives slightly less overall build time reduction than
the separate process for it did (this reduces a -c compile coreutils-native by
about 3 seconds, while the process reduced it by 7 seconds), however this time
is quite insignificant relative to a typical build.

The biggest issue with non-backgrounded syncing is the perceived delay before
work begins, and this resolves that without breaking anything, or so it seems.

(Bitbake rev: 5ab6c5c7b007b8c77c751582141afc07c183d672)

Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
This commit is contained in:
Chris Larson
2010-11-18 22:47:36 -07:00
committed by Richard Purdie
parent 95d2f56126
commit 7c9444e9a5
4 changed files with 61 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/cooker.py
+16 -15
View File
@@ -25,6 +25,7 @@ from __future__ import print_function
import sys, os, glob, os.path, re, time
import logging
import sre_constants
import threading
import multiprocessing
import signal
from cStringIO import StringIO
@@ -983,6 +984,7 @@ class CookerParser(object):
# Internal data
self.filelist = filelist
self.cooker = cooker
self.cfgdata = cooker.configuration.data
# Accounting statistics
self.parsed = 0
@@ -1006,7 +1008,6 @@ class CookerParser(object):
self.result_queue = multiprocessing.Queue()
self.fromcache = []
cfgdata = self.cooker.configuration.data
for filename in self.filelist:
appends = self.cooker.get_file_appends(filename)
if not self.cooker.bb_cache.cacheValid(filename):
@@ -1021,13 +1022,13 @@ class CookerParser(object):
output.put(infos)
self.processes = []
num_processes = int(cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
multiprocessing.cpu_count())
for i in xrange(num_processes):
process = multiprocessing.Process(target=worker,
args=(self.task_queue,
self.result_queue,
cfgdata))
self.cfgdata))
process.start()
self.processes.append(process)
@@ -1041,29 +1042,29 @@ class CookerParser(object):
self.task_queue.close()
for process in self.processes:
process.join()
self.cooker.bb_cache.sync()
bb.codeparser.parser_cache_save(self.cooker.configuration.data)
threading.Thread(target=self.cooker.bb_cache.sync).start()
threading.Thread(target=bb.codeparser.parser_cache_save(self.cooker.configuration.data)).start()
if self.error > 0:
raise ParsingErrorsFound()
def progress(self):
bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed,
self.skipped, self.masked,
self.virtuals, self.error,
self.total),
self.cooker.configuration.event_data)
def parse_next(self):
cooker = self.cooker
if self.current >= self.total:
event = bb.event.ParseCompleted(self.cached, self.parsed,
self.skipped, self.masked,
self.virtuals, self.error,
self.total)
bb.event.fire(event, self.cfgdata)
self.shutdown()
return False
elif self.current == 0:
bb.event.fire(bb.event.ParseStarted(self.total, self.skipped, self.masked),
self.cfgdata)
try:
if self.result_queue.empty() and self.fromcache:
filename, appends = self.fromcache.pop()
_, infos = cooker.bb_cache.load(filename, appends,
self.cooker.configuration.data)
_, infos = cooker.bb_cache.load(filename, appends, self.cfgdata)
parsed = False
else:
infos = self.result_queue.get()
@@ -1087,7 +1088,7 @@ class CookerParser(object):
if info.skipped:
self.skipped += 1
finally:
self.progress()
bb.event.fire(bb.event.ParseProgress(self.current), self.cfgdata)
self.current += 1
return True
bitbake/lib/bb/event.py
+16 -4
View File
@@ -296,10 +296,16 @@ class MultipleProviders(Event):
"""
return self._candidates
class ParseProgress(Event):
"""
Parsing Progress Event
"""
class ParseStarted(Event):
"""Recipe parsing for the runqueue has begun"""
def __init__(self, total, skipped, masked):
Event.__init__(self)
self.total = total
self.skipped = skipped
self.masked = masked
class ParseCompleted(Event):
"""Recipe parsing for the runqueue has completed"""
def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total):
Event.__init__(self)
@@ -312,6 +318,12 @@ class ParseProgress(Event):
self.sofar = cached + parsed
self.total = total
class ParseProgress(Event):
"""Recipe parsing progress"""
def __init__(self, current):
self.current = current
class DepTreeGenerated(Event):
"""
Event when a dependency tree has been generated
bitbake/lib/bb/ui/knotty.py
+20 -16
View File
@@ -78,6 +78,7 @@ def init(server, eventHandler):
return 1
pbar = None
interactive = os.isatty(sys.stdout.fileno())
shutdown = 0
return_value = 0
while True:
@@ -132,23 +133,26 @@ def init(server, eventHandler):
if isinstance(event, bb.build.TaskBase):
logger.info(event._message)
continue
if isinstance(event, bb.event.ParseProgress):
current, total = event.sofar, event.total
if os.isatty(sys.stdout.fileno()):
if not pbar:
pbar = progressbar.ProgressBar(widgets=widgets,
maxval=total).start()
pbar.update(current)
if isinstance(event, bb.event.ParseStarted):
if interactive:
pbar = progressbar.ProgressBar(widgets=widgets,
maxval=event.total).start()
else:
if current == 1:
sys.stdout.write("Parsing .bb files, please wait...")
sys.stdout.flush()
if current == total:
sys.stdout.write("done.")
sys.stdout.flush()
if current == total:
print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
% ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
sys.stdout.write("Parsing recipes...")
sys.stdout.flush()
continue
if isinstance(event, bb.event.ParseProgress):
if interactive:
pbar.update(event.current)
continue
if isinstance(event, bb.event.ParseCompleted):
if interactive:
pbar.update(event.total)
else:
sys.stdout.write("done.\n")
sys.stdout.flush()
print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
% ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
continue
if isinstance(event, bb.command.CookerCommandCompleted):
bitbake/lib/bb/ui/uievent.py
+9 -9
View File
@@ -37,8 +37,8 @@ class BBUIEventQueue:
self.BBServer = BBServer
self.t = threading.Thread()
self.t.setDaemon(True)
self.t.run = self.startCallbackHandler
self.t.setDaemon(True)
self.t.run = self.startCallbackHandler
self.t.start()
def getEvent(self):
@@ -70,7 +70,7 @@ class BBUIEventQueue:
def startCallbackHandler(self):
server = UIXMLRPCServer()
self.host, self.port = server.socket.getsockname()
self.host, self.port = server.socket.getsockname()
server.register_function( self.system_quit, "event.quit" )
server.register_function( self.queue_event, "event.send" )
@@ -83,7 +83,7 @@ class BBUIEventQueue:
server.handle_request()
server.server_close()
def system_quit( self ):
def system_quit( self ):
"""
Shut down the callback thread
"""
@@ -95,11 +95,11 @@ class BBUIEventQueue:
class UIXMLRPCServer (SimpleXMLRPCServer):
def __init__( self, interface = ("localhost", 0) ):
def __init__( self, interface = ("localhost", 0) ):
self.quit = False
SimpleXMLRPCServer.__init__( self,
interface,
requestHandler=SimpleXMLRPCRequestHandler,
SimpleXMLRPCServer.__init__( self,
interface,
requestHandler=SimpleXMLRPCRequestHandler,
logRequests=False, allow_none=True)
def get_request(self):
@@ -121,4 +121,4 @@ class UIXMLRPCServer (SimpleXMLRPCServer):
if request is None:
return
SimpleXMLRPCServer.process_request(self, request, client_address)