bitbake: toaster: show progress of recipe parsing in recent builds area

Modify buildinfohelper and toasterui so that they record the
recipe parse progress (from ParseProgress events in bitbake)
on the Build object.

Note that because the Build object is now created at the
point when ParseStarted occurs, it is necessary to set the
build name to the empty string initially (hence the migration).
The build name can be set when the build properly starts,
i.e. at the BuildStarted event.

Then use this additional data to determine whether a Build
is in a "Parsing" state, and report this in the JSON API.
This enables the most recent builds area to show the recipe
parse progress.

Add additional logic to update the progress bar if the progress
for a build object changes.

[YOCTO #9631]

(Bitbake rev: f33d51d46d70e73e04e325807c1bc4eb68462f7b)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0012_use_release_instead_of_up_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='recipes_parsed',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='build',
+            name='recipes_to_parse',
+            field=models.IntegerField(default=1),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0013_recipe_parse_progress_fields'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='build_name',
+            field=models.CharField(default='', max_length=100),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py

@@ -397,9 +397,15 @@ class Build(models.Model):
     completed_on = models.DateTimeField()
     outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
     cooker_log_path = models.CharField(max_length=500)
-    build_name = models.CharField(max_length=100)
+    build_name = models.CharField(max_length=100, default='')
     bitbake_version = models.CharField(max_length=50)
 
+    # number of recipes to parse for this build
+    recipes_to_parse = models.IntegerField(default=1)
+
+    # number of recipes parsed so far for this build
+    recipes_parsed = models.IntegerField(default=0)
+
     @staticmethod
     def get_recent(project=None):
         """
@@ -615,6 +621,13 @@ class Build(models.Model):
         else:
             return False
 
+    def is_parsing(self):
+        """
+        True if the build is still parsing recipes
+        """
+        return self.outcome == Build.IN_PROGRESS and \
+            self.recipes_parsed < self.recipes_to_parse
+
     def get_state(self):
         """
         Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
@@ -628,6 +641,8 @@ class Build(models.Model):
             return 'Cancelling';
         elif self.is_queued():
             return 'Queued'
+        elif self.is_parsing():
+            return 'Parsing'
         else:
             return self.get_outcome_text()
 

bitbake/lib/toaster/toastergui/api.py

@@ -87,7 +87,7 @@ class XhrBuildRequest(View):
                     br.save()
 
                 except BuildRequest.DoesNotExist:
-                    return error_response('No such build id %s' % i)
+                    return error_response('No such build request id %s' % i)
 
             return error_response('ok')
 
@@ -256,6 +256,14 @@ class MostRecentBuildsView(View):
             build['id'] = build_obj.pk
             build['dashboard_url'] = dashboard_url
 
+            buildrequest_id = None
+            if hasattr(build_obj, 'buildrequest'):
+                buildrequest_id = build_obj.buildrequest.pk
+            build['buildrequest_id'] = buildrequest_id
+
+            build['recipes_parsed_percentage'] = \
+                int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100)
+
             tasks_complete_percentage = 0
             if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
                 tasks_complete_percentage = 100

bitbake/lib/toaster/toastergui/static/js/mrbsection.js

@@ -33,8 +33,8 @@ function mrbSectionInit(ctx){
     return buildData[build.id] || {};
   }
 
-  // returns true if a build's state changed to "Succeeded" or "Failed"
-  // from some other value
+  // returns true if a build's state changed to "Succeeded", "Failed"
+  // or "Cancelled" from some other value
   function buildFinished(build) {
     var cached = getCached(build);
     return cached.state &&
@@ -49,12 +49,18 @@ function mrbSectionInit(ctx){
     return (cached.state !== build.state);
   }
 
-  // returns true if the complete_percentage changed
-  function progressChanged(build) {
+  // returns true if the tasks_complete_percentage changed
+  function tasksProgressChanged(build) {
     var cached = getCached(build);
     return (cached.tasks_complete_percentage !== build.tasks_complete_percentage);
   }
 
+  // returns true if the number of recipes parsed/to parse changed
+  function recipeProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
+  }
+
   function refreshMostRecentBuilds(){
     libtoaster.getMostRecentBuilds(
       libtoaster.ctx.mostRecentBuildsUrl,
@@ -68,10 +74,6 @@ function mrbSectionInit(ctx){
         var colourClass;
         var elements;
 
-        // classes on the parent which signify the build state and affect
-        // the colour of the container for the build
-        var buildStateClasses = 'alert-info alert-success alert-danger';
-
         for (var i = 0; i < data.length; i++) {
           build = data[i];
 
@@ -91,31 +93,25 @@ function mrbSectionInit(ctx){
             container = $(selector);
 
             container.html(html);
-
-            // style the outermost container for this build to reflect
-            // the new build state (red, green, blue);
-            // NB class set here should be in buildStateClasses
-            colourClass = 'alert-info';
-            if (build.state == 'Succeeded') {
-              colourClass = 'alert-success';
-            }
-            else if (build.state == 'Failed') {
-              colourClass = 'alert-danger';
-            }
-
-            elements = $('[data-latest-build-result="' + build.id + '"]');
-            elements.removeClass(buildStateClasses);
-            elements.addClass(colourClass);
           }
-          else if (progressChanged(build)) {
-            // update the progress text
+          else if (tasksProgressChanged(build)) {
+            // update the task progress text
             selector = '#build-pc-done-' + build.id;
             $(selector).html(build.tasks_complete_percentage);
 
-            // update the progress bar
+            // update the task progress bar
             selector = '#build-pc-done-bar-' + build.id;
             $(selector).width(build.tasks_complete_percentage + '%');
           }
+          else if (recipeProgressChanged(build)) {
+            // update the recipe progress text
+            selector = '#recipes-parsed-percentage-' + build.id;
+            $(selector).html(build.recipes_parsed_percentage);
+
+            // update the recipe progress bar
+            selector = '#recipes-parsed-percentage-bar-' + build.id;
+            $(selector).width(build.recipes_parsed_percentage + '%');
+          }
 
           buildData[build.id] = build;
         }
@@ -128,6 +124,6 @@ function mrbSectionInit(ctx){
     );
   }
 
-  window.setInterval(refreshMostRecentBuilds, 1000);
+  window.setInterval(refreshMostRecentBuilds, 1500);
   refreshMostRecentBuilds();
 }

bitbake/lib/toaster/toastergui/templates/mrb_section.html

@@ -26,7 +26,9 @@
           <div class="row project-name">
             <div class="col-md-12">
               <small>
-                <a class="alert-link text-uppercase" href={% project_url build.project %}>{{build.project.name}}</a>
+                <a class="alert-link text-uppercase" href="{% project_url build.project %}">
+                  {{build.project.name}}
+                </a>
               </small>
             </div>
           </div>
@@ -52,14 +54,18 @@
           <%:targets_abbreviated%>
         </span>
       </a>
-    <%else%>
+    <%else targets_abbreviated !== ''%>
       <span data-toggle="tooltip" data-role="targets-text" title="Recipes: <%:targets%>">
         <%:targets_abbreviated%>
       </span>
+    <%else%>
+      ...targets not yet available...
     <%/if%>
   </div>
 
-  <%if state == 'Queued'%>
+  <%if state == 'Parsing'%>
+    <%include tmpl='#parsing-recipes-build-template'/%>
+  <%else state == 'Queued'%>
     <%include tmpl='#queued-build-template'/%>
   <%else state == 'Succeeded' || state == 'Failed'%>
     <%include tmpl='#succeeded-or-failed-build-template'/%>
@@ -75,21 +81,38 @@
 <!-- queued build -->
 <script id="queued-build-template" type="text/x-jsrender">
   <div class="col-md-5">
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue" title="This build is waiting for
+the build directory to become available"></span>
+
     Build queued
   </div>
 
   <div class="col-md-4">
-    <%if is_default_project_build%>
-      <!-- no cancel icon -->
-      <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
-    <%else%>
-      <!-- cancel button -->
-      <span class="cancel-build-btn pull-right alert-link"
-      data-buildrequest-id="<%:id%>" data-request-url="<%:cancel_url%>">
-        <span class="glyphicon glyphicon-remove-circle"></span>
-        Cancel
-      </span>
-    <%/if%>
+    <!-- cancel button -->
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
+<!-- parsing recipes build -->
+<script id="parsing-recipes-build-template" type="text/x-jsrender">
+  <!-- progress bar and parse completion percentage -->
+  <div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
+    <!-- progress bar -->
+    <div class="progress">
+      <div id="recipes-parsed-percentage-bar-<%:id%>"
+           style="width: <%:recipes_parsed_percentage%>%;"
+           class="progress-bar">
+      </div>
+    </div>
+  </div>
+
+  <div class="col-md-4 progress-info">
+    <!-- parse completion percentage -->
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue" title="BitBake is parsing the layers required for your build"></span>
+
+    Parsing <span id="recipes-parsed-percentage-<%:id%>"><%:recipes_parsed_percentage%></span>% complete
+
+    <%include tmpl='#cancel-template'/%>
   </div>
 </script>
 
@@ -110,17 +133,9 @@
     <!-- task completion percentage -->
     <span id="build-pc-done-<%:id%>"><%:tasks_complete_percentage%></span>% of
     tasks complete
-    <%if is_default_project_build%>
-      <!-- no cancel icon -->
-      <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
-    <%else%>
-      <!-- cancel button -->
-      <span class="cancel-build-btn pull-right alert-link"
-      data-buildrequest-id="<%:id%>" data-request-url="<%:cancel_url%>">
-        <span class="glyphicon glyphicon-remove-circle"></span>
-        Cancel
-      </span>
-    <%/if%>
+
+    <!-- cancel button -->
+    <%include tmpl='#cancel-template'/%>
   </div>
 </script>
 
@@ -162,19 +177,8 @@
   <div class="col-md-3">
     Build time: <a class="alert-link" href="<%:buildtime_url%>"><%:buildtime%></a>
 
-    <%if is_default_project_build%>
-      <!-- info icon -->
-      <span class="pull-right glyphicon glyphicon-question-sign get-help <%if state == 'Success'%>get-help-green<%else state == 'Failed'%>get-help-red<%else%>get-help-blue<%/if%>"
-      title="Builds in this project cannot be started from Toaster: they are started from the command line">
-      </span>
-    <%else%>
-      <!-- rebuild button -->
-      <span class="rebuild-btn alert-link <%if state == 'Success'%>success<%else state == 'Failed'%>danger<%else%>info<%/if%> pull-right"
-      data-request-url="<%:rebuild_url%>" data-target='<%:build_targets_json%>'>
-        <span class="glyphicon glyphicon-repeat"></span>
-        Rebuild
-      </span>
-    <%/if%>
+    <!-- rebuild button -->
+    <%include tmpl='#rebuild-template'/%>
   </div>
 </script>
 
@@ -187,12 +191,40 @@
 
   <!-- rebuild button -->
   <div class="col-md-3">
-    <span class="info pull-right rebuild-btn alert-link"
+    <%include tmpl='#rebuild-template'/%>
+  </div>
+</script>
+
+<!-- rebuild button or no rebuild icon -->
+<script id="rebuild-template" type="text/x-jsrender">
+  <%if is_default_project_build%>
+    <!-- no rebuild info icon -->
+    <span class="pull-right glyphicon glyphicon-question-sign get-help <%if state == 'Success'%>get-help-green<%else state == 'Failed'%>get-help-red<%else%>get-help-blue<%/if%>"
+    title="Builds in this project cannot be started from Toaster: they are started from the command line">
+    </span>
+  <%else%>
+    <!-- rebuild button -->
+    <span class="rebuild-btn alert-link <%if state == 'Success'%>success<%else state == 'Failed'%>danger<%else%>info<%/if%> pull-right"
     data-request-url="<%:rebuild_url%>" data-target='<%:build_targets_json%>'>
       <span class="glyphicon glyphicon-repeat"></span>
       Rebuild
     </span>
-  </div>
+  <%/if%>
+</script>
+
+<!-- cancel button or no cancel icon -->
+<script id="cancel-template" type="text/x-jsrender">
+  <%if is_default_project_build%>
+    <!-- no cancel icon -->
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
+  <%else%>
+    <!-- cancel button -->
+    <span class="cancel-build-btn pull-right alert-link"
+    data-buildrequest-id="<%:buildrequest_id%>" data-request-url="<%:cancel_url%>">
+      <span class="glyphicon glyphicon-remove-circle"></span>
+      Cancel
+    </span>
+  <%/if%>
 </script>
 
 <script>