bitbake: Toaster: Implement the project-specific feature and releated enhancements and defects.

Here is the primary driving enhancement: * Bug 12785 - Support Project Specific configuration for external tools (e.g. ISS, Eclipse) - Isolated project-specific configuration page (full Toaster context hidden) - Support for new project, reconfigure existing project, and import existing command line project - Ability to define variables (e.g. image recipe) and pass them back to external GUI - Ability to execute the cloning phase, so that external GUI receive a buildable project - Ability to call back to the external GUI when updates are completed and ready - Compatibility of above projects with the normal full Toaster interface - Ability to pass to a 'complete' or 'cancel' web page so that the external GUI can immediately stop that Toaster instance, and not leave dangling servers nor edit sessions open Here are the supporting enhancements, where at least the back end is implemented: * Bug 12821 - Make Toaster conf changes compatible with command line usage * Bug 12822 - Support importing user changes to conf files into Toaster * Bug 12823 - Support importing user build directories into Toaster * Bug 12824 - Scan imported layers for content so that they are immediately available * Bug 12825 - show layer clone item in progress bar Here are defects fixed: * Bug 12817 - builddelete.py requires explicit 'add_arguments' * Bug 12818 - Remove orphaned imported layers when project is deleted * Bug 12826 - fix imported layer management * Bug 12819 - build using selected bitbake env, not Toaster's env * Bug 12820 - Toaster randomizes the layer order in toaster_bblayers.conf [YOCTO #12785] (Bitbake rev: 985d6cec290bdd80998a63483561a73c75d82d65) Signed-off-by: David Reyna <David.Reyna@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2026-06-01 00:59:48 +00:00 · 2018-08-15 18:04:09 -07:00
parent 9af0f1a46b
commit 08fbdc02e3
29 changed files with 1899 additions and 54 deletions
@@ -27,8 +27,9 @@ import shutil
 import time
 from django.db import transaction
 from django.db.models import Q
-from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
+from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build
+from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting
+from orm.models import signal_runbuilds
 import subprocess

 from toastermain import settings
@@ -38,6 +39,8 @@ from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdExceptio
 import logging
 logger = logging.getLogger("toaster")

+install_dir = os.environ.get('TOASTER_DIR')
+
 from pprint import pprint, pformat

 class LocalhostBEController(BuildEnvironmentController):
@@ -87,10 +90,10 @@ class LocalhostBEController(BuildEnvironmentController):
        #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
        return local_checkout_path

-
-    def setCloneStatus(self,bitbake,status,total,current):
+    def setCloneStatus(self,bitbake,status,total,current,repo_name):
        bitbake.req.build.repos_cloned=current
        bitbake.req.build.repos_to_clone=total
+        bitbake.req.build.progress_item=repo_name
        bitbake.req.build.save()

    def setLayers(self, bitbake, layers, targets):
@@ -100,6 +103,7 @@ class LocalhostBEController(BuildEnvironmentController):

        layerlist = []
        nongitlayerlist = []
+        layer_index = 0
        git_env = os.environ.copy()
        # (note: add custom environment settings here)

@@ -113,7 +117,7 @@ class LocalhostBEController(BuildEnvironmentController):
        if bitbake.giturl and bitbake.commit:
            gitrepos[(bitbake.giturl, bitbake.commit)] = []
            gitrepos[(bitbake.giturl, bitbake.commit)].append(
-                ("bitbake", bitbake.dirpath))
+                ("bitbake", bitbake.dirpath, 0))

        for layer in layers:
            # We don't need to git clone the layer for the CustomImageRecipe
@@ -124,12 +128,13 @@ class LocalhostBEController(BuildEnvironmentController):
            # If we have local layers then we don't need clone them
            # For local layers giturl will be empty
            if not layer.giturl:
-                nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
+                nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) )
                continue

            if not (layer.giturl, layer.commit) in gitrepos:
                gitrepos[(layer.giturl, layer.commit)] = []
-            gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
+            gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) )
+            layer_index += 1


        logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
@@ -159,9 +164,9 @@ class LocalhostBEController(BuildEnvironmentController):
        # 3. checkout the repositories
        clone_count=0
        clone_total=len(gitrepos.keys())
-        self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'')
        for giturl, commit in gitrepos.keys():
-            self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
+            self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0])
            clone_count += 1

            localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
@@ -205,16 +210,16 @@ class LocalhostBEController(BuildEnvironmentController):
                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)

            # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
+            for name, dirpath, index in gitrepos[(giturl, commit)]:
                localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
                if not os.path.exists(localdirpath):
                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))

                if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
+                    layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))

-        self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'')
        logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))

        if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
@@ -232,7 +237,7 @@ class LocalhostBEController(BuildEnvironmentController):
                    customrecipe, layers)

                if os.path.isdir(custom_layer_path):
-                    layerlist.append(custom_layer_path)
+                    layerlist.append("%03d:%s" % (layer_index,custom_layer_path))

            except CustomImageRecipe.DoesNotExist:
                continue  # not a custom recipe, skip
@@ -240,7 +245,11 @@ class LocalhostBEController(BuildEnvironmentController):
        layerlist.extend(nongitlayerlist)
        logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
        self.islayerset = True
-        return layerlist
+
+        # restore the order of layer list for bblayers.conf
+        layerlist.sort()
+        sorted_layerlist = [l[4:] for l in layerlist]
+        return sorted_layerlist

    def setup_custom_image_recipe(self, customrecipe, layers):
        """ Set up toaster-custom-images layer and recipe files """
@@ -310,31 +319,115 @@ class LocalhostBEController(BuildEnvironmentController):

    def triggerBuild(self, bitbake, layers, variables, targets, brbe):
        layers = self.setLayers(bitbake, layers, targets)
+        is_merged_attr = bitbake.req.project.merged_attr
+
+        git_env = os.environ.copy()
+        # (note: add custom environment settings here)
+        try:
+            # insure that the project init/build uses the selected bitbake, and not Toaster's
+            del git_env['TEMPLATECONF']
+            del git_env['BBBASEDIR']
+            del git_env['BUILDDIR']
+        except KeyError:
+            pass

        # init build environment from the clone
-        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
+        if bitbake.req.project.builddir:
+            builddir = bitbake.req.project.builddir
+        else:
+            builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
        oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
        # init build environment
        try:
            custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
            custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
-            self._shellcmd("bash -c 'source %s'" % (custom_script))
+            self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
        except ToasterSetting.DoesNotExist:
            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
-                       self.be.sourcedir)
+                       self.be.sourcedir,env=git_env)

        # update bblayers.conf
-        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
-        with open(bblconfpath, 'w') as bblayers:
-            bblayers.write('# line added by toaster build control\n'
-                           'BBLAYERS = "%s"' % ' '.join(layers))
+        if not is_merged_attr:
+            bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
+            with open(bblconfpath, 'w') as bblayers:
+                bblayers.write('# line added by toaster build control\n'
+                               'BBLAYERS = "%s"' % ' '.join(layers))

-        # write configuration file
-        confpath = os.path.join(builddir, 'conf/toaster.conf')
-        with open(confpath, 'w') as conf:
-            for var in variables:
-                conf.write('%s="%s"\n' % (var.name, var.value))
-            conf.write('INHERIT+="toaster buildhistory"')
+            # write configuration file
+            confpath = os.path.join(builddir, 'conf/toaster.conf')
+            with open(confpath, 'w') as conf:
+                for var in variables:
+                    conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('INHERIT+="toaster buildhistory"')
+        else:
+            # Append the Toaster-specific values directly to the bblayers.conf
+            bblconfpath = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf")
+            bblconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf.save")
+            shutil.copyfile(bblconfpath, bblconfpath_save)
+            with open(bblconfpath) as bblayers:
+                content = bblayers.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bblconfpath,'w') as bblayers:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        bblayers.write(line)
+                if not was_toaster:
+                    bblayers.write('\n')
+                bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                bblayers.write('BBLAYERS = "\\\n')
+                for layer in layers:
+                    bblayers.write('  %s \\\n' % layer)
+                bblayers.write('  "\n')
+                bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+            # Append the Toaster-specific values directly to the local.conf
+            bbconfpath = os.path.join(bitbake.req.project.builddir, "conf/local.conf")
+            bbconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/local.conf.save")
+            shutil.copyfile(bbconfpath, bbconfpath_save)
+            with open(bbconfpath) as f:
+                content = f.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bbconfpath,'w') as conf:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        conf.write(line)
+                if not was_toaster:
+                    conf.write('\n')
+                conf.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                for var in variables:
+                    if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"):
+                        conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+
+        # If 'target' is just the project preparation target, then we are done
+        for target in targets:
+            if "_PROJECT_PREPARE_" == target.target:
+                logger.debug('localhostbecontroller: Project has been prepared. Done.')
+                # Update the Build Request and release the build environment
+                bitbake.req.state = BuildRequest.REQ_COMPLETED
+                bitbake.req.save()
+                self.be.lock = BuildEnvironment.LOCK_FREE
+                self.be.save()
+                # Close the project build and progress bar
+                bitbake.req.build.outcome = Build.SUCCEEDED
+                bitbake.req.build.save()
+                # Update the project status
+                bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS)
+                signal_runbuilds()
+                return

        # clean the Toaster to build environment
        env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
@@ -342,9 +435,14 @@ class LocalhostBEController(BuildEnvironmentController):
        # run bitbake server from the clone
        bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
        toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
-        self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
-                       builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        if not is_merged_attr:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        else:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake), self.be.sourcedir)

        # read port number from bitbake.lock
        self.be.bbport = -1
@@ -390,12 +488,20 @@ class LocalhostBEController(BuildEnvironmentController):
        log = os.path.join(builddir, 'toaster_ui.log')
        local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                     'bitbake')
-        self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+        if not is_merged_attr:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                        '%s %s -u toasterui  --read %s --read %s --token="" >>%s 2>&1;'
                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
                        self.be.bbport, bitbake,)],
                        builddir, nowait=True)
+        else:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+                        '%s %s -u toasterui  --token="" >>%s 2>&1;'
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
+                        self.be.bbport, bitbake,)],
+                        builddir, nowait=True)

        logger.debug('localhostbecontroller: Build launched, exiting. '
                     'Follow build logs at %s' % log)
@@ -49,7 +49,7 @@ class Command(BaseCommand):
                # we could not find a BEC; postpone the BR
                br.state = BuildRequest.REQ_QUEUED
                br.save()
-                logger.debug("runbuilds: No build env")
+                logger.debug("runbuilds: No build env (%s)" % e)
                return

            logger.info("runbuilds: starting build %s, environment %s" %