bitbake: toaster: add fields for sourcedir and builddir paths

We add explicit absolute paths for a directory where
the layer sources will be checked out (sourcedir) and
where the build activities will take place.

Adding minimal checking when starting the application in
order to make sure that BuildEnvironment (BE) settings are
usable. This check is ran by the toaster script at startup.

Modify the localhost bbcontroller to use the BE settings
instead of trying to self-configure on checked out sources.

(Bitbake rev: d17500d3f73fdeeef5f11fb3773a65e927be3f02)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/toaster

@@ -64,6 +64,7 @@ function webserverStartAll()
             python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=1
         fi
         python $BBBASEDIR/lib/toaster/manage.py migrate bldcontrol || retval=1
+	python $BBBASEDIR/lib/toaster/manage.py checksettings  || retval=1
 
         if [ $retval -eq 0 ]; then
             python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -170,27 +170,16 @@ class LocalhostBEController(BuildEnvironmentController):
         this controller manages the default build directory,
         the server setup and system start and stop for the localhost-type build environment
 
-        The address field is used as working directory; if not set, the build/ directory
-        is created
     """
+    from os.path import dirname as DN
 
     def __init__(self, be):
         super(LocalhostBEController, self).__init__(be)
         from os.path import dirname as DN
-        self.cwd = DN(DN(DN(DN(DN(os.path.realpath(__file__))))))
-        if self.be.address is None or len(self.be.address) == 0:
-            self.be.address = "build"
-            self.be.save()
-        self.bwd = self.be.address
         self.dburl = settings.getDATABASE_URL()
 
-        # transform relative paths to absolute ones
-        if not self.bwd.startswith("/"):
-            self.bwd = os.path.join(self.cwd, self.bwd)
-        self._createBE()
-
     def _shellcmd(self, command):
-        p = subprocess.Popen(command, cwd=self.cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(command, cwd=self.be.sourcedir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (out,err) = p.communicate()
         if p.returncode:
             if len(err) == 0:
@@ -201,39 +190,49 @@ class LocalhostBEController(BuildEnvironmentController):
         else:
             return out
 
-    def _createBE(self):
+    def _createdirpath(self, path):
-        assert self.cwd and os.path.exists(self.cwd)
+        if not os.path.exists(DN(path)):
-        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.cwd, self.bwd))
+            self._createdirpath(DN(path))
+        if not os.path.exists(path):
+            os.mkdir(path, 0755)
+
+    def _startBE(self):
+        assert self.be.sourcedir and os.path.exists(self.be.sourcedir)
+        self._createdirpath(self.be.builddir)
+        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.be.sourcedir, self.be.builddir))
 
     def startBBServer(self):
-        assert self.cwd and os.path.exists(self.cwd)
+        assert self.be.sourcedir and os.path.exists(self.be.sourcedir)
-        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.cwd, self.bwd, self.dburl))
+
+        self._startBE()
+
+        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.be.sourcedir, self.be.builddir, self.dburl))
         # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
         # but since they start async without any return, we just wait a bit
         print "Started server"
-        assert self.cwd and os.path.exists(self.bwd)
+        assert self.be.sourcedir and os.path.exists(self.be.builddir)
         self.be.bbaddress = "localhost"
         self.be.bbport = "8200"
         self.be.bbstate = BuildEnvironment.SERVER_STARTED
         self.be.save()
 
     def stopBBServer(self):
-        assert self.cwd
+        assert self.be.sourcedir
         print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
-            (self.cwd, self.bwd, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
+            (self.be.sourcedir, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
         self.be.bbstate = BuildEnvironment.SERVER_STOPPED
         self.be.save()
         print "Stopped server"
 
     def setLayers(self, layers):
-        assert self.cwd is not None
+        assert self.be.sourcedir is not None
-        layerconf = os.path.join(self.bwd, "conf/bblayers.conf")
+        layerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
         if not os.path.exists(layerconf):
             raise Exception("BE is not consistent: bblayers.conf file missing at ", layerconf)
         return True
 
     def release(self):
-        assert self.cwd and os.path.exists(self.bwd)
+        assert self.be.sourcedir and os.path.exists(self.be.builddir)
         import shutil
-        shutil.rmtree(os.path.join(self.cwd, "build"))
+        shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
-        assert not os.path.exists(self.bwd)
+        assert not os.path.exists(self.be.builddir)

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -0,0 +1,33 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from django.db import transaction
+from orm.models import Build
+from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
+from bldcontrol.models import BuildRequest, BuildEnvironment
+import os
+
+class Command(NoArgsCommand):
+    args = ""
+    help = "Verifies thid %dthe configured settings are valid and usable, or prompts the user to fix the settings."
+
+    def handle(self, **options):
+        # we make sure we have builddir and sourcedir for all defined build envionments
+        for be in BuildEnvironment.objects.all():
+            def _verify_be():
+                is_changed = False
+                print("Verifying the Build Environment type %s id %d." % (be.get_betype_display(), be.pk))
+                if len(be.sourcedir) == 0:
+                    be.sourcedir = raw_input(" -- sourcedir may not be empty:")
+                    is_changed = True
+                if not be.sourcedir.startswith("/"):
+                    be.sourcedir = raw_input(" -- sourcedir must be an absolute path:")
+                    is_changed = True
+                if len(be.builddir) == 0:
+                    be.builddir = raw_input(" -- builddir may not be empty:")
+                    is_changed = True
+                if not be.builddir.startswith("/"):
+                    be.builddir = raw_input(" -- builddir must be an absolute path:")
+                    is_changed = True
+                return is_changed
+
+            while (_verify_be()):
+                pass

bitbake/lib/toaster/bldcontrol/migrations/0002_auto__add_field_buildenvironment_sourcedir__add_field_buildenvironment.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'BuildEnvironment.sourcedir'
+        db.add_column(u'bldcontrol_buildenvironment', 'sourcedir',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True),
+                      keep_default=False)
+
+        # Adding field 'BuildEnvironment.builddir'
+        db.add_column(u'bldcontrol_buildenvironment', 'builddir',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'BuildEnvironment.sourcedir'
+        db.delete_column(u'bldcontrol_buildenvironment', 'sourcedir')
+
+        # Deleting field 'BuildEnvironment.builddir'
+        db.delete_column(u'bldcontrol_buildenvironment', 'builddir')
+
+
+    models = {
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/models.py

@@ -33,6 +33,8 @@ class BuildEnvironment(models.Model):
     bbport      = models.IntegerField(default = -1)
     bbtoken     = models.CharField(max_length = 126, blank = True)
     bbstate     = models.IntegerField(choices = SERVER_STATE, default = SERVER_STOPPED)
+    sourcedir   = models.CharField(max_length = 512, blank = True)
+    builddir    = models.CharField(max_length = 512, blank = True)
     lock        = models.IntegerField(choices = LOCK_STATE, default = LOCK_FREE)
     created     = models.DateTimeField(auto_now_add = True)
     updated     = models.DateTimeField(auto_now = True)