mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-06-01 13:09:50 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: toaster: improve scan for SDK artifacts

Browse Source 
SDK artifacts were previously picked up by toaster.bbclass and
notified to buildinfohelper (via toasterui). The artifacts
were then added to the Build object, so that it wasn't clear
which artifact went with which target; we were also unable
to attach SDK artifacts to a Build if they had already been
attached to a previous build.

Now, toaster.bbclass just notifies the TOOLCHAIN_OUTPUTNAME when
a populate_sdk* target completes. The scan is moved to buildinfohelper,
where we search the SDK deploy directory for files matching
TOOLCHAIN_OUTPUTNAME and attach them to targets (not builds).

If an SDK file is not produced by a target, we now look for a
similar, previously-run target which did produce artifacts.
If there is one, we clone the SDK artifacts from that target
onto the current one.

This all means that we can show SDK artifacts by target, and should
always get artifacts associated with a target, regardless of whether
it really build them.

This requires an additional model, TargetSDKFile, which tracks
the size and path of SDK artifact files with respect to Target
objects.

[YOCTO #8556]

(Bitbake rev: 5e650c611605507e1e0d1588cd5eb6535c2d34fc)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: bavery <brian.avery@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Elliot Smith
2016-07-12 15:54:48 -07:00
committed by Richard Purdie
parent f39ae146ea
commit 00c2c0be5e
7 changed files with 304 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py
+39
View File
@@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('orm', '0007_auto_20160523_1446'),
]
operations = [
migrations.CreateModel(
name='TargetKernelFile',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
('file_name', models.FilePathField()),
('file_size', models.IntegerField()),
('target', models.ForeignKey(to='orm.Target')),
],
),
migrations.CreateModel(
name='TargetSDKFile',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
('file_name', models.FilePathField()),
('file_size', models.IntegerField()),
('target', models.ForeignKey(to='orm.Target')),
],
),
migrations.RemoveField(
model_name='buildartifact',
name='build',
),
migrations.DeleteModel(
name='BuildArtifact',
),
]
bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py
-23
View File
@@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('orm', '0007_auto_20160523_1446'),
]
operations = [
migrations.CreateModel(
name='TargetArtifactFile',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('file_name', models.FilePathField()),
('file_size', models.IntegerField()),
('target', models.ForeignKey(to='orm.Target')),
],
),
]
bitbake/lib/toaster/orm/models.py
+72 -44
View File
@@ -581,28 +581,6 @@ class Build(models.Model):
def __str__(self):
return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
# an Artifact is anything that results from a Build, and may be of interest to the user, and is not stored elsewhere
class BuildArtifact(models.Model):
build = models.ForeignKey(Build)
file_name = models.FilePathField()
file_size = models.IntegerField()
def get_local_file_name(self):
try:
deploydir = Variable.objects.get(build = self.build, variable_name="DEPLOY_DIR").variable_value
return self.file_name[len(deploydir)+1:]
except:
raise
return self.file_name
def get_basename(self):
return os.path.basename(self.file_name)
def is_available(self):
return self.build.buildrequest.environment.has_artifact(self.file_name)
class ProjectTarget(models.Model):
project = models.ForeignKey(Project)
target = models.CharField(max_length=100)
@@ -625,13 +603,19 @@ class Target(models.Model):
def get_similar_targets(self):
"""
Get targets for the same machine, task and target name
Get target sfor the same machine, task and target name
(e.g. 'core-image-minimal') from a successful build for this project
(but excluding this target).
Note that we look for targets built by this project because projects
can have different configurations from each other, and put their
artifacts in different directories.
Note that we only look for targets built by this project because
projects can have different configurations from each other, and put
their artifacts in different directories.
The possibility of error when retrieving candidate targets
is minimised by the fact that bitbake will rebuild artifacts if MACHINE
(or various other variables) change. In this case, there is no need to
clone artifacts from another target, as those artifacts will have
been re-generated for this target anyway.
"""
query = ~Q(pk=self.pk) & \
Q(target=self.target) & \
@@ -649,29 +633,29 @@ class Target(models.Model):
similar_target = None
candidates = self.get_similar_targets()
if candidates.count() < 1:
if candidates.count() == 0:
return similar_target
task_subquery = Q(task=self.task)
# we can look for a 'build' task if this task is a 'populate_sdk_ext'
# task, as it will have created images; and vice versa; note that
# task, as the latter also creates images; and vice versa; note that
# 'build' targets can have their task set to '';
# also note that 'populate_sdk' does not produce image files
image_tasks = [
'', # aka 'build'
'build',
'image',
'populate_sdk_ext'
]
if self.task in image_tasks:
task_subquery = Q(task__in=image_tasks)
query = task_subquery & Q(num_files__gt=0)
# annotate with the count of files, to exclude any targets which
# don't have associated files
candidates = candidates.annotate(
num_files=Count('target_image_file'))
candidates = candidates.annotate(num_files=Count('target_image_file'))
query = task_subquery & Q(num_files__gt=0)
candidates = candidates.filter(query)
@@ -681,11 +665,35 @@ class Target(models.Model):
return similar_target
def clone_artifacts_from(self, target):
def get_similar_target_with_sdk_files(self):
"""
Make clones of the BuildArtifacts, Target_Image_Files and
TargetArtifactFile objects associated with Target target, then
associate them with this target.
Get the most recent similar target with TargetSDKFiles associated
with it, for the purpose of cloning those files onto this target.
"""
similar_target = None
candidates = self.get_similar_targets()
if candidates.count() == 0:
return similar_target
# annotate with the count of files, to exclude any targets which
# don't have associated files
candidates = candidates.annotate(num_files=Count('targetsdkfile'))
query = Q(task=self.task) & Q(num_files__gt=0)
candidates = candidates.filter(query)
if candidates.count() > 0:
candidates.order_by('build__completed_on')
similar_target = candidates.last()
return similar_target
def clone_image_artifacts_from(self, target):
"""
Make clones of the Target_Image_Files and TargetKernelFile objects
associated with Target target, then associate them with this target.
Note that for Target_Image_Files, we only want files from the previous
build whose suffix matches one of the suffixes defined in this
@@ -711,18 +719,38 @@ class Target(models.Model):
image_file.target = self
image_file.save()
artifact_files = target.targetartifactfile_set.all()
for artifact_file in artifact_files:
artifact_file.pk = None
artifact_file.target = self
artifact_file.save()
kernel_files = target.targetkernelfile_set.all()
for kernel_file in kernel_files:
kernel_file.pk = None
kernel_file.target = self
kernel_file.save()
self.license_manifest_path = target.license_manifest_path
self.save()
# an Artifact is anything that results from a target being built, and may
# be of interest to the user, and is not an image file
class TargetArtifactFile(models.Model):
def clone_sdk_artifacts_from(self, target):
"""
Clone TargetSDKFile objects from target and associate them with this
target.
"""
sdk_files = target.targetsdkfile_set.all()
for sdk_file in sdk_files:
sdk_file.pk = None
sdk_file.target = self
sdk_file.save()
# kernel artifacts for a target: bzImage and modules*
class TargetKernelFile(models.Model):
target = models.ForeignKey(Target)
file_name = models.FilePathField()
file_size = models.IntegerField()
@property
def basename(self):
return os.path.basename(self.file_name)
# SDK artifacts for a target: sh and manifest files
class TargetSDKFile(models.Model):
target = models.ForeignKey(Target)
file_name = models.FilePathField()
file_size = models.IntegerField()