mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-31 00:39:46 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: bitbake: Switch to using new override syntax

Browse Source 
This change updates the datastore to use the new override syntax using
colons instead of underscores exclusively. It is expected layers would
have to be converted to work with bitbake after this change.

Supporting mixed syntax isn't possible, it is only feasible to have
one internal representation of overrides.

Whilst we can't warn for every possible override that may be set in the
old format, show errors for _append/_prepend/_remove since those should
never be present.

(Bitbake rev: 7dcf317cc141dc980634f8c18bfa84f83e57206a)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2021-07-25 22:23:57 +01:00
parent f735627e7c
commit 2abf8a699e
4 changed files with 59 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/cache.py
+2 -2
View File
@@ -53,12 +53,12 @@ class RecipeInfoCommon(object):
@classmethod @classmethod
def pkgvar(cls, var, packages, metadata): def pkgvar(cls, var, packages, metadata):
return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata)) return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata))
for pkg in packages) for pkg in packages)
@classmethod @classmethod
def taskvar(cls, var, tasks, metadata): def taskvar(cls, var, tasks, metadata):
return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata))
for task in tasks) for task in tasks)
@classmethod @classmethod
bitbake/lib/bb/data_smart.py
+55 -58
View File
@@ -26,8 +26,8 @@ from bb.COW import COWDictBase
logger = logging.getLogger("BitBake.Data") logger = logging.getLogger("BitBake.Data")
__setvar_keyword__ = ["_append", "_prepend", "_remove"] __setvar_keyword__ = [":append", ":prepend", ":remove"]
__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$') __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
__expand_python_regexp__ = re.compile(r"\${@.+?}") __expand_python_regexp__ = re.compile(r"\${@.+?}")
__whitespace_split__ = re.compile(r'(\s)') __whitespace_split__ = re.compile(r'(\s)')
@@ -277,7 +277,7 @@ class VariableHistory(object):
for (r, override) in d.overridedata[var]: for (r, override) in d.overridedata[var]:
for event in self.variable(r): for event in self.variable(r):
loginfo = event.copy() loginfo = event.copy()
if 'flag' in loginfo and not loginfo['flag'].startswith("_"): if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
continue continue
loginfo['variable'] = var loginfo['variable'] = var
loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
@@ -342,7 +342,7 @@ class VariableHistory(object):
for event in history: for event in history:
if 'flag' in event: if 'flag' in event:
continue continue
if event['op'] == '_remove': if event['op'] == ':remove':
continue continue
if isset and event['op'] == 'set?': if isset and event['op'] == 'set?':
continue continue
@@ -481,7 +481,15 @@ class DataSmart(MutableMapping):
def setVar(self, var, value, **loginfo): def setVar(self, var, value, **loginfo):
#print("var=" + str(var) + " val=" + str(value)) #print("var=" + str(var) + " val=" + str(value))
var = var.replace(":", "_")
if "_append" in var or "_prepend" in var or "_remove" in var:
info = "%s" % var
if "filename" in loginfo:
info += " file: %s" % loginfo[filename]
if "lineno" in loginfo:
info += " line: %s" % loginfo[lineno]
bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
self.expand_cache = {} self.expand_cache = {}
parsing=False parsing=False
if 'parsing' in loginfo: if 'parsing' in loginfo:
@@ -510,7 +518,7 @@ class DataSmart(MutableMapping):
# pay the cookie monster # pay the cookie monster
# more cookies for the cookie monster # more cookies for the cookie monster
if '_' in var: if ':' in var:
self._setvar_update_overrides(base, **loginfo) self._setvar_update_overrides(base, **loginfo)
if base in self.overridevars: if base in self.overridevars:
@@ -521,27 +529,27 @@ class DataSmart(MutableMapping):
self._makeShadowCopy(var) self._makeShadowCopy(var)
if not parsing: if not parsing:
if "_append" in self.dict[var]: if ":append" in self.dict[var]:
del self.dict[var]["_append"] del self.dict[var][":append"]
if "_prepend" in self.dict[var]: if ":prepend" in self.dict[var]:
del self.dict[var]["_prepend"] del self.dict[var][":prepend"]
if "_remove" in self.dict[var]: if ":remove" in self.dict[var]:
del self.dict[var]["_remove"] del self.dict[var][":remove"]
if var in self.overridedata: if var in self.overridedata:
active = [] active = []
self.need_overrides() self.need_overrides()
for (r, o) in self.overridedata[var]: for (r, o) in self.overridedata[var]:
if o in self.overridesset: if o in self.overridesset:
active.append(r) active.append(r)
elif "_" in o: elif ":" in o:
if set(o.split("_")).issubset(self.overridesset): if set(o.split(":")).issubset(self.overridesset):
active.append(r) active.append(r)
for a in active: for a in active:
self.delVar(a) self.delVar(a)
del self.overridedata[var] del self.overridedata[var]
# more cookies for the cookie monster # more cookies for the cookie monster
if '_' in var: if ':' in var:
self._setvar_update_overrides(var, **loginfo) self._setvar_update_overrides(var, **loginfo)
# setting var # setting var
@@ -567,8 +575,8 @@ class DataSmart(MutableMapping):
def _setvar_update_overrides(self, var, **loginfo): def _setvar_update_overrides(self, var, **loginfo):
# aka pay the cookie monster # aka pay the cookie monster
override = var[var.rfind('_')+1:] override = var[var.rfind(':')+1:]
shortvar = var[:var.rfind('_')] shortvar = var[:var.rfind(':')]
while override and __override_regexp__.match(override): while override and __override_regexp__.match(override):
if shortvar not in self.overridedata: if shortvar not in self.overridedata:
self.overridedata[shortvar] = [] self.overridedata[shortvar] = []
@@ -577,9 +585,9 @@ class DataSmart(MutableMapping):
self.overridedata[shortvar] = list(self.overridedata[shortvar]) self.overridedata[shortvar] = list(self.overridedata[shortvar])
self.overridedata[shortvar].append([var, override]) self.overridedata[shortvar].append([var, override])
override = None override = None
if "_" in shortvar: if ":" in shortvar:
override = var[shortvar.rfind('_')+1:] override = var[shortvar.rfind(':')+1:]
shortvar = var[:shortvar.rfind('_')] shortvar = var[:shortvar.rfind(':')]
if len(shortvar) == 0: if len(shortvar) == 0:
override = None override = None
@@ -590,8 +598,6 @@ class DataSmart(MutableMapping):
""" """
Rename the variable key to newkey Rename the variable key to newkey
""" """
key = key.replace(":", "_")
newkey = newkey.replace(":", "_")
if key == newkey: if key == newkey:
bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
return return
@@ -620,7 +626,7 @@ class DataSmart(MutableMapping):
self.overridedata[newkey].append([v.replace(key, newkey), o]) self.overridedata[newkey].append([v.replace(key, newkey), o])
self.renameVar(v, v.replace(key, newkey)) self.renameVar(v, v.replace(key, newkey))
if '_' in newkey and val is None: if ':' in newkey and val is None:
self._setvar_update_overrides(newkey, **loginfo) self._setvar_update_overrides(newkey, **loginfo)
loginfo['variable'] = key loginfo['variable'] = key
@@ -632,15 +638,14 @@ class DataSmart(MutableMapping):
def appendVar(self, var, value, **loginfo): def appendVar(self, var, value, **loginfo):
loginfo['op'] = 'append' loginfo['op'] = 'append'
self.varhistory.record(**loginfo) self.varhistory.record(**loginfo)
self.setVar(var + "_append", value, ignore=True, parsing=True) self.setVar(var + ":append", value, ignore=True, parsing=True)
def prependVar(self, var, value, **loginfo): def prependVar(self, var, value, **loginfo):
loginfo['op'] = 'prepend' loginfo['op'] = 'prepend'
self.varhistory.record(**loginfo) self.varhistory.record(**loginfo)
self.setVar(var + "_prepend", value, ignore=True, parsing=True) self.setVar(var + ":prepend", value, ignore=True, parsing=True)
def delVar(self, var, **loginfo): def delVar(self, var, **loginfo):
var = var.replace(":", "_")
self.expand_cache = {} self.expand_cache = {}
loginfo['detail'] = "" loginfo['detail'] = ""
@@ -649,9 +654,9 @@ class DataSmart(MutableMapping):
self.dict[var] = {} self.dict[var] = {}
if var in self.overridedata: if var in self.overridedata:
del self.overridedata[var] del self.overridedata[var]
if '_' in var: if ':' in var:
override = var[var.rfind('_')+1:] override = var[var.rfind(':')+1:]
shortvar = var[:var.rfind('_')] shortvar = var[:var.rfind(':')]
while override and override.islower(): while override and override.islower():
try: try:
if shortvar in self.overridedata: if shortvar in self.overridedata:
@@ -661,14 +666,13 @@ class DataSmart(MutableMapping):
except ValueError as e: except ValueError as e:
pass pass
override = None override = None
if "_" in shortvar: if ":" in shortvar:
override = var[shortvar.rfind('_')+1:] override = var[shortvar.rfind(':')+1:]
shortvar = var[:shortvar.rfind('_')] shortvar = var[:shortvar.rfind(':')]
if len(shortvar) == 0: if len(shortvar) == 0:
override = None override = None
def setVarFlag(self, var, flag, value, **loginfo): def setVarFlag(self, var, flag, value, **loginfo):
var = var.replace(":", "_")
self.expand_cache = {} self.expand_cache = {}
if 'op' not in loginfo: if 'op' not in loginfo:
@@ -679,7 +683,7 @@ class DataSmart(MutableMapping):
self._makeShadowCopy(var) self._makeShadowCopy(var)
self.dict[var][flag] = value self.dict[var][flag] = value
if flag == "_defaultval" and '_' in var: if flag == "_defaultval" and ':' in var:
self._setvar_update_overrides(var, **loginfo) self._setvar_update_overrides(var, **loginfo)
if flag == "_defaultval" and var in self.overridevars: if flag == "_defaultval" and var in self.overridevars:
self._setvar_update_overridevars(var, value) self._setvar_update_overridevars(var, value)
@@ -692,7 +696,6 @@ class DataSmart(MutableMapping):
self.dict["__exportlist"]["_content"].add(var) self.dict["__exportlist"]["_content"].add(var)
def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
var = var.replace(":", "_")
if flag == "_content": if flag == "_content":
cachename = var cachename = var
else: else:
@@ -712,11 +715,11 @@ class DataSmart(MutableMapping):
active = {} active = {}
self.need_overrides() self.need_overrides()
for (r, o) in overridedata: for (r, o) in overridedata:
# What about double overrides both with "_" in the name? # FIXME What about double overrides both with "_" in the name?
if o in self.overridesset: if o in self.overridesset:
active[o] = r active[o] = r
elif "_" in o: elif ":" in o:
if set(o.split("_")).issubset(self.overridesset): if set(o.split(":")).issubset(self.overridesset):
active[o] = r active[o] = r
mod = True mod = True
@@ -724,10 +727,10 @@ class DataSmart(MutableMapping):
mod = False mod = False
for o in self.overrides: for o in self.overrides:
for a in active.copy(): for a in active.copy():
if a.endswith("_" + o): if a.endswith(":" + o):
t = active[a] t = active[a]
del active[a] del active[a]
active[a.replace("_" + o, "")] = t active[a.replace(":" + o, "")] = t
mod = True mod = True
elif a == o: elif a == o:
match = active[a] match = active[a]
@@ -746,28 +749,28 @@ class DataSmart(MutableMapping):
value = copy.copy(local_var["_defaultval"]) value = copy.copy(local_var["_defaultval"])
if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
if not value: if not value:
value = "" value = ""
self.need_overrides() self.need_overrides()
for (r, o) in local_var["_append"]: for (r, o) in local_var[":append"]:
match = True match = True
if o: if o:
for o2 in o.split("_"): for o2 in o.split(":"):
if not o2 in self.overrides: if not o2 in self.overrides:
match = False match = False
if match: if match:
value = value + r value = value + r
if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
if not value: if not value:
value = "" value = ""
self.need_overrides() self.need_overrides()
for (r, o) in local_var["_prepend"]: for (r, o) in local_var[":prepend"]:
match = True match = True
if o: if o:
for o2 in o.split("_"): for o2 in o.split(":"):
if not o2 in self.overrides: if not o2 in self.overrides:
match = False match = False
if match: if match:
@@ -779,12 +782,12 @@ class DataSmart(MutableMapping):
if expand: if expand:
value = parser.value value = parser.value
if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing: if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
self.need_overrides() self.need_overrides()
for (r, o) in local_var["_remove"]: for (r, o) in local_var[":remove"]:
match = True match = True
if o: if o:
for o2 in o.split("_"): for o2 in o.split(":"):
if not o2 in self.overrides: if not o2 in self.overrides:
match = False match = False
if match: if match:
@@ -820,7 +823,6 @@ class DataSmart(MutableMapping):
return value return value
def delVarFlag(self, var, flag, **loginfo): def delVarFlag(self, var, flag, **loginfo):
var = var.replace(":", "_")
self.expand_cache = {} self.expand_cache = {}
local_var, _ = self._findVar(var) local_var, _ = self._findVar(var)
@@ -838,7 +840,6 @@ class DataSmart(MutableMapping):
del self.dict[var][flag] del self.dict[var][flag]
def appendVarFlag(self, var, flag, value, **loginfo): def appendVarFlag(self, var, flag, value, **loginfo):
var = var.replace(":", "_")
loginfo['op'] = 'append' loginfo['op'] = 'append'
loginfo['flag'] = flag loginfo['flag'] = flag
self.varhistory.record(**loginfo) self.varhistory.record(**loginfo)
@@ -846,7 +847,6 @@ class DataSmart(MutableMapping):
self.setVarFlag(var, flag, newvalue, ignore=True) self.setVarFlag(var, flag, newvalue, ignore=True)
def prependVarFlag(self, var, flag, value, **loginfo): def prependVarFlag(self, var, flag, value, **loginfo):
var = var.replace(":", "_")
loginfo['op'] = 'prepend' loginfo['op'] = 'prepend'
loginfo['flag'] = flag loginfo['flag'] = flag
self.varhistory.record(**loginfo) self.varhistory.record(**loginfo)
@@ -854,7 +854,6 @@ class DataSmart(MutableMapping):
self.setVarFlag(var, flag, newvalue, ignore=True) self.setVarFlag(var, flag, newvalue, ignore=True)
def setVarFlags(self, var, flags, **loginfo): def setVarFlags(self, var, flags, **loginfo):
var = var.replace(":", "_")
self.expand_cache = {} self.expand_cache = {}
infer_caller_details(loginfo) infer_caller_details(loginfo)
if not var in self.dict: if not var in self.dict:
@@ -869,13 +868,12 @@ class DataSmart(MutableMapping):
self.dict[var][i] = flags[i] self.dict[var][i] = flags[i]
def getVarFlags(self, var, expand = False, internalflags=False): def getVarFlags(self, var, expand = False, internalflags=False):
var = var.replace(":", "_")
local_var, _ = self._findVar(var) local_var, _ = self._findVar(var)
flags = {} flags = {}
if local_var: if local_var:
for i in local_var: for i in local_var:
if i.startswith("_") and not internalflags: if i.startswith(("_", ":")) and not internalflags:
continue continue
flags[i] = local_var[i] flags[i] = local_var[i]
if expand and i in expand: if expand and i in expand:
@@ -886,7 +884,6 @@ class DataSmart(MutableMapping):
def delVarFlags(self, var, **loginfo): def delVarFlags(self, var, **loginfo):
var = var.replace(":", "_")
self.expand_cache = {} self.expand_cache = {}
if not var in self.dict: if not var in self.dict:
self._makeShadowCopy(var) self._makeShadowCopy(var)
@@ -974,8 +971,8 @@ class DataSmart(MutableMapping):
for (r, o) in self.overridedata[var]: for (r, o) in self.overridedata[var]:
if o in self.overridesset: if o in self.overridesset:
overrides.add(var) overrides.add(var)
elif "_" in o: elif ":" in o:
if set(o.split("_")).issubset(self.overridesset): if set(o.split(":")).issubset(self.overridesset):
overrides.add(var) overrides.add(var)
for k in keylist(self.dict): for k in keylist(self.dict):
bitbake/lib/bb/parse/ast.py
-2
View File
@@ -97,7 +97,6 @@ class DataNode(AstNode):
def eval(self, data): def eval(self, data):
groupd = self.groupd groupd = self.groupd
key = groupd["var"] key = groupd["var"]
key = key.replace(":", "_")
loginfo = { loginfo = {
'variable': key, 'variable': key,
'file': self.filename, 'file': self.filename,
@@ -208,7 +207,6 @@ class ExportFuncsNode(AstNode):
def eval(self, data): def eval(self, data):
for func in self.n: for func in self.n:
func = func.replace(":", "_")
calledfunc = self.classname + "_" + func calledfunc = self.classname + "_" + func
if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
bitbake/lib/bb/siggen.py
+2 -2
View File
@@ -228,7 +228,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
# self.dump_sigtask(fn, task, d.getVar("STAMP"), False) # self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
for task in taskdeps: for task in taskdeps:
d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task]) d.setVar("BB_BASEHASH:task-%s" % task, self.basehash[fn + ":" + task])
def postparsing_clean_cache(self): def postparsing_clean_cache(self):
# #
@@ -325,7 +325,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
h = hashlib.sha256(data.encode("utf-8")).hexdigest() h = hashlib.sha256(data.encode("utf-8")).hexdigest()
self.taskhash[tid] = h self.taskhash[tid] = h
#d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task])
return h return h
def writeout_file_checksum_cache(self): def writeout_file_checksum_cache(self):