mirror of
https://git.yoctoproject.org/poky
synced 2026-05-30 00:20:08 +00:00
image: Create separate tasks for rootfs construction
This patch splits the code in lib/oe/image into separate tasks, one per image type. This removes the need for the simple task graph code and defers to the bitbake task management code to handle this instead. This is a good step forward in splitting up the monolithic code and starting to make it more accessible to people. It should also make it easier for people to hook in other tasks and processes into the rootfs code. Incidentally, the reason this code was all combined originally was due to limitations of fakeroot where if you exited the session, you lost permissions data. With pseudo this constraint was removed. We did start to rework the rootfs/image code previously and got so far with untangling it however we did prioritise some performance tweaks over splitting into separate tasks and in hindsight, this was a mistake and should have been done the other way around. That work was suspended due to changes in the people working on the project but this split has always been intended, now is the time to finish it IMO. There were some side effects of doing this: * The symlink for the manifest moves to the rootfs-postcommands class and into the manifest function. * There is no seperate "symlink removal" and "symlink creation", they are merged * The date/time stamps of the manifest and the built images can now be different since the tasks can be run separately and the datetime stamp will then be different between do_rootfs and the do_image_* tasks. (From OE-Core rev: c2dab181c1cdabac3be6197f4b9ea4235cbbc140) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
+204
-23
@@ -106,21 +106,6 @@ python () {
|
||||
d.setVarFlag(var, 'func', '1')
|
||||
}
|
||||
|
||||
def fstype_variables(d):
|
||||
import oe.image
|
||||
|
||||
image = oe.image.Image(d)
|
||||
alltypes, fstype_groups, cimages = image._get_image_types()
|
||||
fstype_vars = set()
|
||||
for fstype_group in fstype_groups:
|
||||
for fstype in fstype_group:
|
||||
fstype_vars.add('IMAGE_CMD_' + fstype)
|
||||
if fstype in cimages:
|
||||
for ctype in cimages[fstype]:
|
||||
fstype_vars.add('COMPRESS_CMD_' + ctype)
|
||||
|
||||
return sorted(fstype_vars)
|
||||
|
||||
def rootfs_variables(d):
|
||||
from oe.rootfs import variable_depends
|
||||
variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE',
|
||||
@@ -129,7 +114,6 @@ def rootfs_variables(d):
|
||||
'MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS',
|
||||
'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
|
||||
'COMPRESSIONTYPES', 'IMAGE_GEN_DEBUGFS']
|
||||
variables.extend(fstype_variables(d))
|
||||
variables.extend(command_variables(d))
|
||||
variables.extend(variable_depends(d))
|
||||
return " ".join(variables)
|
||||
@@ -250,19 +234,13 @@ do_rootfs[umask] = "022"
|
||||
addtask rootfs before do_build
|
||||
|
||||
fakeroot python do_image () {
|
||||
from oe.image import create_image
|
||||
from oe.image import Image
|
||||
from oe.utils import execute_pre_post_process
|
||||
|
||||
i = Image(d)
|
||||
|
||||
pre_process_cmds = d.getVar("IMAGE_PREPROCESS_COMMAND", True)
|
||||
|
||||
execute_pre_post_process(d, pre_process_cmds)
|
||||
|
||||
i._remove_old_symlinks()
|
||||
|
||||
i.create()
|
||||
write_wic_env(d)
|
||||
}
|
||||
do_image[dirs] = "${TOPDIR}"
|
||||
do_image[umask] = "022"
|
||||
@@ -279,6 +257,209 @@ do_image_complete[dirs] = "${TOPDIR}"
|
||||
do_image_complete[umask] = "022"
|
||||
addtask do_image_complete after do_image before do_build
|
||||
|
||||
#
|
||||
# Write environment variables used by wic
|
||||
# to tmp/sysroots/<machine>/imgdata/<image>.env
|
||||
#
|
||||
def write_wic_env(d):
|
||||
wicvars = d.getVar('WICVARS', True)
|
||||
if not wicvars:
|
||||
return
|
||||
|
||||
stdir = d.getVar('STAGING_DIR_TARGET', True)
|
||||
outdir = os.path.join(stdir, 'imgdata')
|
||||
bb.utils.mkdirhier(outdir)
|
||||
basename = d.getVar('IMAGE_BASENAME', True)
|
||||
with open(os.path.join(outdir, basename) + '.env', 'w') as envf:
|
||||
for var in wicvars.split():
|
||||
value = d.getVar(var, True)
|
||||
if value:
|
||||
envf.write('%s="%s"\n' % (var, value.strip()))
|
||||
|
||||
def setup_debugfs_variables(d):
|
||||
d.appendVar('IMAGE_ROOTFS', '-dbg')
|
||||
d.appendVar('IMAGE_LINK_NAME', '-dbg')
|
||||
d.appendVar('IMAGE_NAME','-dbg')
|
||||
debugfs_image_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS', True)
|
||||
if debugfs_image_fstypes:
|
||||
d.setVar('IMAGE_FSTYPES', debugfs_image_fstypes)
|
||||
|
||||
python setup_debugfs () {
|
||||
setup_debugfs_variables(d)
|
||||
}
|
||||
|
||||
python () {
|
||||
vardeps = set()
|
||||
ctypes = d.getVar('COMPRESSIONTYPES', True).split()
|
||||
old_overrides = d.getVar('OVERRIDES', 0)
|
||||
|
||||
def _image_base_type(type):
|
||||
if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]:
|
||||
type = "ext4"
|
||||
basetype = type
|
||||
for ctype in ctypes:
|
||||
if type.endswith("." + ctype):
|
||||
basetype = type[:-len("." + ctype)]
|
||||
break
|
||||
|
||||
return basetype
|
||||
|
||||
basetypes = {}
|
||||
alltypes = d.getVar('IMAGE_FSTYPES', True).split()
|
||||
typedeps = {}
|
||||
|
||||
if d.getVar('IMAGE_GEN_DEBUGFS', True) == "1":
|
||||
debugfs_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS', True).split()
|
||||
for t in debugfs_fstypes:
|
||||
alltypes.append("debugfs_" + t)
|
||||
|
||||
def _add_type(t):
|
||||
baset = _image_base_type(t)
|
||||
if baset not in basetypes:
|
||||
basetypes[baset]= []
|
||||
if t not in basetypes[baset]:
|
||||
basetypes[baset].append(t)
|
||||
debug = ""
|
||||
if t.startswith("debugfs_"):
|
||||
t = t[8:]
|
||||
debug = "debugfs_"
|
||||
deps = (d.getVar('IMAGE_TYPEDEP_' + t, True) or "").split()
|
||||
vardeps.add('IMAGE_TYPEDEP_' + t)
|
||||
if baset not in typedeps:
|
||||
typedeps[baset] = set()
|
||||
deps = [debug + dep for dep in deps]
|
||||
for dep in deps:
|
||||
if dep not in alltypes:
|
||||
alltypes.append(dep)
|
||||
_add_type(dep)
|
||||
basedep = _image_base_type(dep)
|
||||
typedeps[baset].add(basedep)
|
||||
|
||||
for t in alltypes[:]:
|
||||
_add_type(t)
|
||||
|
||||
d.appendVarFlag('do_image', 'vardeps', ' '.join(vardeps))
|
||||
|
||||
for t in basetypes:
|
||||
vardeps = set()
|
||||
cmds = []
|
||||
subimages = []
|
||||
realt = t
|
||||
|
||||
localdata = bb.data.createCopy(d)
|
||||
debug = ""
|
||||
if t.startswith("debugfs_"):
|
||||
setup_debugfs_variables(localdata)
|
||||
debug = "setup_debugfs "
|
||||
realt = t[8:]
|
||||
localdata.setVar('OVERRIDES', '%s:%s' % (realt, old_overrides))
|
||||
bb.data.update_data(localdata)
|
||||
localdata.setVar('type', realt)
|
||||
|
||||
image_cmd = localdata.getVar("IMAGE_CMD", True)
|
||||
vardeps.add('IMAGE_CMD_' + realt)
|
||||
if image_cmd:
|
||||
cmds.append("\t" + image_cmd)
|
||||
else:
|
||||
bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % t)
|
||||
cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}"))
|
||||
|
||||
for bt in basetypes[t]:
|
||||
for ctype in ctypes:
|
||||
if bt.endswith("." + ctype):
|
||||
cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True))
|
||||
vardeps.add('COMPRESS_CMD_' + ctype)
|
||||
subimages.append(realt + "." + ctype)
|
||||
|
||||
if realt not in alltypes:
|
||||
cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}"))
|
||||
else:
|
||||
subimages.append(realt)
|
||||
|
||||
d.setVar('do_image_%s' % t, '\n'.join(cmds))
|
||||
d.setVarFlag('do_image_%s' % t, 'func', '1')
|
||||
d.setVarFlag('do_image_%s' % t, 'fakeroot', '1')
|
||||
d.setVarFlag('do_image_%s' % t, 'prefuncs', debug + 'set_image_size')
|
||||
d.setVarFlag('do_image_%s' % t, 'postfuncs', 'create_symlinks')
|
||||
d.setVarFlag('do_image_%s' % t, 'subimages', subimages)
|
||||
d.appendVarFlag('do_image_%s' % t, 'vardeps', ' '.join(vardeps))
|
||||
|
||||
after = 'do_image'
|
||||
for dep in typedeps[t]:
|
||||
after += ' do_image_%s' % dep
|
||||
|
||||
bb.debug(2, "Adding type %s before %s, after %s" % (t, 'do_image_complete', after))
|
||||
bb.build.addtask('do_image_%s' % t, 'do_image_complete', after, d)
|
||||
}
|
||||
|
||||
#
|
||||
# Compute the rootfs size
|
||||
#
|
||||
def get_rootfs_size(d):
|
||||
import subprocess
|
||||
|
||||
rootfs_alignment = int(d.getVar('IMAGE_ROOTFS_ALIGNMENT', True))
|
||||
overhead_factor = float(d.getVar('IMAGE_OVERHEAD_FACTOR', True))
|
||||
rootfs_req_size = int(d.getVar('IMAGE_ROOTFS_SIZE', True))
|
||||
rootfs_extra_space = eval(d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True))
|
||||
rootfs_maxsize = d.getVar('IMAGE_ROOTFS_MAXSIZE', True)
|
||||
|
||||
output = subprocess.check_output(['du', '-ks',
|
||||
d.getVar('IMAGE_ROOTFS', True)])
|
||||
size_kb = int(output.split()[0])
|
||||
base_size = size_kb * overhead_factor
|
||||
base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \
|
||||
rootfs_extra_space
|
||||
|
||||
if base_size != int(base_size):
|
||||
base_size = int(base_size + 1)
|
||||
else:
|
||||
base_size = int(base_size)
|
||||
|
||||
base_size += rootfs_alignment - 1
|
||||
base_size -= base_size % rootfs_alignment
|
||||
|
||||
# Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set)
|
||||
if rootfs_maxsize:
|
||||
rootfs_maxsize_int = int(rootfs_maxsize)
|
||||
if base_size > rootfs_maxsize_int:
|
||||
bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \
|
||||
(base_size, rootfs_maxsize_int))
|
||||
return base_size
|
||||
|
||||
python set_image_size () {
|
||||
rootfs_size = get_rootfs_size(d)
|
||||
d.setVar('ROOTFS_SIZE', str(rootfs_size))
|
||||
d.setVarFlag('ROOTFS_SIZE', 'export', '1')
|
||||
}
|
||||
|
||||
#
|
||||
# Create symlinks to the newly created image
|
||||
#
|
||||
python create_symlinks() {
|
||||
|
||||
deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True)
|
||||
img_name = d.getVar('IMAGE_NAME', True)
|
||||
link_name = d.getVar('IMAGE_LINK_NAME', True)
|
||||
manifest_name = d.getVar('IMAGE_MANIFEST', True)
|
||||
taskname = d.getVar("BB_CURRENTTASK", True)
|
||||
subimages = d.getVarFlag("do_" + taskname, 'subimages', False)
|
||||
os.chdir(deploy_dir)
|
||||
|
||||
if not link_name:
|
||||
return
|
||||
for type in subimages:
|
||||
if os.path.exists(img_name + ".rootfs." + type):
|
||||
dst = deploy_dir + "/" + link_name + "." + type
|
||||
src = img_name + ".rootfs." + type
|
||||
bb.note("Creating symlink: %s -> %s" % (dst, src))
|
||||
if os.path.islink(dst):
|
||||
if d.getVar('RM_OLD_IMAGE', True) == "1" and \
|
||||
os.path.exists(os.path.realpath(dst)):
|
||||
os.remove(os.path.realpath(dst))
|
||||
os.remove(dst)
|
||||
os.symlink(src, dst)
|
||||
}
|
||||
|
||||
MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|${sysconfdir}|${nonarch_base_libdir}/udev|/lib/modules/[^/]*/modules.*|"
|
||||
MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
|
||||
|
||||
Reference in New Issue
Block a user