mirror of
https://git.yoctoproject.org/poky
synced 2026-06-03 01:40:07 +00:00
bitbake: add option to write offline event log file
This patch adds a "-w/--write-log" option to bitbake that writes an event log file for the current build. The name of the file is passed as a parameter to the "-w" argument. If the parameter is the empty string '', the file name is generated in the form bitbake_eventlog_DATE.json, where DATE is the current date and time, with second precision. The "-w" option can also be supplied as the BBEVENTLOG environment variable. We add a script, toater-eventreplay, that reads an event log file and loads the data into a Toaster database, creating a build entry. We modify the toasterui to fix minor issues with reading events from an event log file. Performance impact is undetectable under no-task executed builds. (Bitbake rev: 1befb4a783bb7b7b387d4b5ee08830d9516f1ac2) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN
committed by
Richard Purdie
Richard Purdie
parent
d086fa3aed
commit
85a17f86ea
@@ -205,6 +205,75 @@ class BBCooker:
|
||||
self.data = self.databuilder.data
|
||||
self.data_hash = self.databuilder.data_hash
|
||||
|
||||
|
||||
# we log all events to a file if so directed
|
||||
if self.configuration.writeeventlog:
|
||||
import json, pickle
|
||||
DEFAULT_EVENTFILE = self.configuration.writeeventlog
|
||||
class EventLogWriteHandler():
|
||||
|
||||
class EventWriter():
|
||||
def __init__(self, cooker):
|
||||
self.file_inited = None
|
||||
self.cooker = cooker
|
||||
self.event_queue = []
|
||||
|
||||
def init_file(self):
|
||||
try:
|
||||
# delete the old log
|
||||
os.remove(DEFAULT_EVENTFILE)
|
||||
except:
|
||||
pass
|
||||
|
||||
# write current configuration data
|
||||
with open(DEFAULT_EVENTFILE, "w") as f:
|
||||
f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
|
||||
|
||||
def write_event(self, event):
|
||||
with open(DEFAULT_EVENTFILE, "a") as f:
|
||||
try:
|
||||
f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print(e, traceback.format_exc(e))
|
||||
|
||||
|
||||
def send(self, event):
|
||||
event_class = event.__module__ + "." + event.__class__.__name__
|
||||
|
||||
# init on bb.event.BuildStarted
|
||||
if self.file_inited is None:
|
||||
if event_class == "bb.event.BuildStarted":
|
||||
self.init_file()
|
||||
self.file_inited = True
|
||||
|
||||
# write pending events
|
||||
for e in self.event_queue:
|
||||
self.write_event(e)
|
||||
|
||||
# also write the current event
|
||||
self.write_event(event)
|
||||
|
||||
else:
|
||||
# queue all events until the file is inited
|
||||
self.event_queue.append(event)
|
||||
|
||||
else:
|
||||
# we have the file, just write the event
|
||||
self.write_event(event)
|
||||
|
||||
# set our handler's event processor
|
||||
event = EventWriter(self) # self is the cooker here
|
||||
|
||||
|
||||
# set up cooker features for this mock UI handler
|
||||
|
||||
# we need to write the dependency tree in the log
|
||||
self.featureset.setFeature(CookerFeatures.SEND_DEPENDS_TREE)
|
||||
# register the log file writer as UI Handler
|
||||
bb.event.register_UIHhandler(EventLogWriteHandler())
|
||||
|
||||
|
||||
#
|
||||
# Special updated configuration we use for firing events
|
||||
#
|
||||
@@ -505,7 +574,7 @@ class BBCooker:
|
||||
taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False)
|
||||
|
||||
return runlist, taskdata
|
||||
|
||||
|
||||
######## WARNING : this function requires cache_extra to be enabled ########
|
||||
|
||||
def generateTaskDepTreeData(self, pkgs_to_build, task):
|
||||
@@ -1550,10 +1619,10 @@ class CookerCollectFiles(object):
|
||||
for p in pkgfns:
|
||||
realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
|
||||
priorities[p] = self.calc_bbfile_priority(realfn, matched)
|
||||
|
||||
|
||||
# Don't show the warning if the BBFILE_PATTERN did match .bbappend files
|
||||
unmatched = set()
|
||||
for _, _, regex, pri in self.bbfile_config_priorities:
|
||||
for _, _, regex, pri in self.bbfile_config_priorities:
|
||||
if not regex in matched:
|
||||
unmatched.add(regex)
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class CookerConfiguration(object):
|
||||
self.dry_run = False
|
||||
self.tracking = False
|
||||
self.interface = []
|
||||
self.writeeventlog = False
|
||||
|
||||
self.env = {}
|
||||
|
||||
|
||||
@@ -556,7 +556,6 @@ class ORMWrapper(object):
|
||||
assert isinstance(build_obj, Build)
|
||||
|
||||
helptext_objects = []
|
||||
|
||||
for k in vardump:
|
||||
desc = vardump[k]['doc']
|
||||
if desc is None:
|
||||
@@ -667,9 +666,11 @@ class BuildInfoHelper(object):
|
||||
if (path.startswith(bl.layer.local_path)):
|
||||
return bl
|
||||
|
||||
#TODO: if we get here, we didn't read layers correctly
|
||||
assert False
|
||||
return None
|
||||
#if we get here, we didn't read layers correctly; mockup the new layer
|
||||
unknown_layer, created = Layer.objects.get_or_create(name="unknown", local_path="/", layer_index_url="")
|
||||
unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
|
||||
|
||||
return unknown_layer_version_obj
|
||||
|
||||
def _get_recipe_information_from_taskfile(self, taskfile):
|
||||
localfilepath = taskfile.split(":")[-1]
|
||||
@@ -732,7 +733,6 @@ class BuildInfoHelper(object):
|
||||
|
||||
def store_started_build(self, event):
|
||||
assert '_pkgs' in vars(event)
|
||||
assert 'lvs' in self.internal_state, "Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass."
|
||||
build_information = self._get_build_information()
|
||||
|
||||
build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe)
|
||||
@@ -740,10 +740,13 @@ class BuildInfoHelper(object):
|
||||
self.internal_state['build'] = build_obj
|
||||
|
||||
# save layer version information for this build
|
||||
for layer_obj in self.internal_state['lvs']:
|
||||
self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj])
|
||||
if not 'lvs' in self.internal_state:
|
||||
logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.")
|
||||
else:
|
||||
for layer_obj in self.internal_state['lvs']:
|
||||
self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj])
|
||||
|
||||
del self.internal_state['lvs']
|
||||
del self.internal_state['lvs']
|
||||
|
||||
# create target information
|
||||
target_information = {}
|
||||
@@ -753,7 +756,8 @@ class BuildInfoHelper(object):
|
||||
self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
|
||||
|
||||
# Save build configuration
|
||||
self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0])
|
||||
data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
|
||||
self.orm_wrapper.save_build_variables(build_obj, [])
|
||||
|
||||
return self.brbe
|
||||
|
||||
@@ -980,14 +984,29 @@ class BuildInfoHelper(object):
|
||||
|
||||
recipe_info = {}
|
||||
recipe_info['name'] = pn
|
||||
recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
|
||||
recipe_info['layer_version'] = layer_version_obj
|
||||
recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
|
||||
recipe_info['license'] = event._depgraph['pn'][pn]['license']
|
||||
recipe_info['description'] = event._depgraph['pn'][pn]['description']
|
||||
recipe_info['section'] = event._depgraph['pn'][pn]['section']
|
||||
recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
|
||||
recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
|
||||
|
||||
if 'version' in event._depgraph['pn'][pn]:
|
||||
recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
|
||||
|
||||
if 'summary' in event._depgraph['pn'][pn]:
|
||||
recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
|
||||
|
||||
if 'license' in event._depgraph['pn'][pn]:
|
||||
recipe_info['license'] = event._depgraph['pn'][pn]['license']
|
||||
|
||||
if 'description' in event._depgraph['pn'][pn]:
|
||||
recipe_info['description'] = event._depgraph['pn'][pn]['description']
|
||||
|
||||
if 'section' in event._depgraph['pn'][pn]:
|
||||
recipe_info['section'] = event._depgraph['pn'][pn]['section']
|
||||
|
||||
if 'homepage' in event._depgraph['pn'][pn]:
|
||||
recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
|
||||
|
||||
if 'bugtracker' in event._depgraph['pn'][pn]:
|
||||
recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
|
||||
|
||||
recipe_info['file_path'] = file_name
|
||||
recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
|
||||
recipe.is_image = False
|
||||
@@ -1146,4 +1165,4 @@ class BuildInfoHelper(object):
|
||||
|
||||
if 'backlog' in self.internal_state:
|
||||
for event in self.internal_state['backlog']:
|
||||
print "NOTE: Unsaved log: ", event.msg
|
||||
logger.error("Unsaved log: %s", event.msg)
|
||||
|
||||
@@ -309,7 +309,7 @@ def main(server, eventHandler, params ):
|
||||
try:
|
||||
buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data))
|
||||
except Exception as ce:
|
||||
print("CRITICAL: failed to to save toaster exception to the database: %s" % str(ce))
|
||||
logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce))
|
||||
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user