mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-06-01 13:09:50 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: toaster/orm/management/commands/lsupdates.py: Use new layerindexlib module

Browse Source 
Change lsupdates.py to use the new layerindexlib module to load the data from
the public layer index.  It still does all of the manual parsing.  This
is intended to be a stop gap until the toaster can use the module itself to
manage the data.

Everything else is functionally equivalent to the prior version.

(Bitbake rev: 8e482342c652e298b5f5ea58eda72c5eb14ce2bd)

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Mark Hatle
2018-07-25 11:48:54 -04:00
committed by Richard Purdie
parent cbf41cbe55
commit da24071e92
1 changed files with 97 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/toaster/orm/management/commands/lsupdates.py
+97 -131
View File
@@ -29,7 +29,6 @@ from orm.models import ToasterSetting
import os import os
import sys import sys
import json
import logging import logging
import threading import threading
import time import time
@@ -37,6 +36,18 @@ logger = logging.getLogger("toaster")
DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
# Add path to bitbake modules for layerindexlib
# lib/toaster/orm/management/commands/lsupdates.py (abspath)
# lib/toaster/orm/management/commands (dirname)
# lib/toaster/orm/management (dirname)
# lib/toaster/orm (dirname)
# lib/toaster/ (dirname)
# lib/ (dirname)
path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
sys.path.insert(0, path)
import layerindexlib
class Spinner(threading.Thread): class Spinner(threading.Thread):
""" A simple progress spinner to indicate download/parsing is happening""" """ A simple progress spinner to indicate download/parsing is happening"""
@@ -86,45 +97,6 @@ class Command(BaseCommand):
self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
assert self.apiurl is not None assert self.apiurl is not None
try:
from urllib.request import urlopen, URLError
from urllib.parse import urlparse
except ImportError:
from urllib2 import urlopen, URLError
from urlparse import urlparse
proxy_settings = os.environ.get("http_proxy", None)
def _get_json_response(apiurl=None):
if None == apiurl:
apiurl=self.apiurl
http_progress = Spinner()
http_progress.start()
_parsedurl = urlparse(apiurl)
path = _parsedurl.path
# logger.debug("Fetching %s", apiurl)
try:
res = urlopen(apiurl)
except URLError as e:
raise Exception("Failed to read %s: %s" % (path, e.reason))
parsed = json.loads(res.read().decode('utf-8'))
http_progress.stop()
return parsed
# verify we can get the basic api
try:
apilinks = _get_json_response()
except Exception as e:
import traceback
if proxy_settings is not None:
logger.info("EE: Using proxy %s" % proxy_settings)
logger.warning("EE: could not connect to %s, skipping update:"
"%s\n%s" % (self.apiurl, e, traceback.format_exc()))
return
# update branches; only those that we already have names listed in the # update branches; only those that we already have names listed in the
# Releases table # Releases table
@@ -133,112 +105,118 @@ class Command(BaseCommand):
if len(whitelist_branch_names) == 0: if len(whitelist_branch_names) == 0:
raise Exception("Failed to make list of branches to fetch") raise Exception("Failed to make list of branches to fetch")
logger.info("Fetching metadata releases for %s", logger.info("Fetching metadata for %s",
" ".join(whitelist_branch_names)) " ".join(whitelist_branch_names))
branches_info = _get_json_response(apilinks['branches'] + # We require a non-empty bb.data, but we can fake it with a dictionary
"?filter=name:%s" layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
% "OR".join(whitelist_branch_names))
http_progress = Spinner()
http_progress.start()
if whitelist_branch_names:
url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
else:
url_branches = ""
layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
http_progress.stop()
# We know we're only processing one entry, so we reference it here
# (this is cheating...)
index = layerindex.indexes[0]
# Map the layer index branches to toaster releases # Map the layer index branches to toaster releases
li_branch_id_to_toaster_release = {} li_branch_id_to_toaster_release = {}
total = len(branches_info) logger.info("Processing releases")
for i, branch in enumerate(branches_info):
li_branch_id_to_toaster_release[branch['id']] = \ total = len(index.branches)
Release.objects.get(name=branch['name']) for i, id in enumerate(index.branches):
li_branch_id_to_toaster_release[id] = \
Release.objects.get(name=index.branches[id].name)
self.mini_progress("Releases", i, total) self.mini_progress("Releases", i, total)
# keep a track of the layerindex (li) id mappings so that # keep a track of the layerindex (li) id mappings so that
# layer_versions can be created for these layers later on # layer_versions can be created for these layers later on
li_layer_id_to_toaster_layer_id = {} li_layer_id_to_toaster_layer_id = {}
logger.info("Fetching layers") logger.info("Processing layers")
layers_info = _get_json_response(apilinks['layerItems']) total = len(index.layerItems)
for i, id in enumerate(index.layerItems):
total = len(layers_info)
for i, li in enumerate(layers_info):
try: try:
l, created = Layer.objects.get_or_create(name=li['name']) l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
l.up_date = li['updated'] l.up_date = index.layerItems[id].updated
l.summary = li['summary'] l.summary = index.layerItems[id].summary
l.description = li['description'] l.description = index.layerItems[id].description
if created: if created:
# predefined layers in the fixtures (for example poky.xml) # predefined layers in the fixtures (for example poky.xml)
# always preempt the Layer Index for these values # always preempt the Layer Index for these values
l.vcs_url = li['vcs_url'] l.vcs_url = index.layerItems[id].vcs_url
l.vcs_web_url = li['vcs_web_url'] l.vcs_web_url = index.layerItems[id].vcs_web_url
l.vcs_web_tree_base_url = li['vcs_web_tree_base_url'] l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
l.vcs_web_file_base_url = li['vcs_web_file_base_url'] l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
l.save() l.save()
except Layer.MultipleObjectsReturned: except Layer.MultipleObjectsReturned:
logger.info("Skipped %s as we found multiple layers and " logger.info("Skipped %s as we found multiple layers and "
"don't know which to update" % "don't know which to update" %
li['name']) index.layerItems[id].name)
li_layer_id_to_toaster_layer_id[li['id']] = l.pk li_layer_id_to_toaster_layer_id[id] = l.pk
self.mini_progress("layers", i, total) self.mini_progress("layers", i, total)
# update layer_versions # update layer_versions
logger.info("Fetching layer versions") logger.info("Processing layer versions")
layerbranches_info = _get_json_response(
apilinks['layerBranches'] + "?filter=branch__name:%s" %
"OR".join(whitelist_branch_names))
# Map Layer index layer_branch object id to # Map Layer index layer_branch object id to
# layer_version toaster object id # layer_version toaster object id
li_layer_branch_id_to_toaster_lv_id = {} li_layer_branch_id_to_toaster_lv_id = {}
total = len(layerbranches_info) total = len(index.layerBranches)
for i, lbi in enumerate(layerbranches_info): for i, id in enumerate(index.layerBranches):
# release as defined by toaster map to layerindex branch # release as defined by toaster map to layerindex branch
release = li_branch_id_to_toaster_release[lbi['branch']] release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
try: try:
lv, created = Layer_Version.objects.get_or_create( lv, created = Layer_Version.objects.get_or_create(
layer=Layer.objects.get( layer=Layer.objects.get(
pk=li_layer_id_to_toaster_layer_id[lbi['layer']]), pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
release=release release=release
) )
except KeyError: except KeyError:
logger.warning( logger.warning(
"No such layerindex layer referenced by layerbranch %d" % "No such layerindex layer referenced by layerbranch %d" %
lbi['layer']) index.layerBranches[id].layer_id)
continue continue
if created: if created:
lv.release = li_branch_id_to_toaster_release[lbi['branch']] lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
lv.up_date = lbi['updated'] lv.up_date = index.layerBranches[id].updated
lv.commit = lbi['actual_branch'] lv.commit = index.layerBranches[id].actual_branch
lv.dirpath = lbi['vcs_subdir'] lv.dirpath = index.layerBranches[id].vcs_subdir
lv.save() lv.save()
li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\ li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
lv.pk lv.pk
self.mini_progress("layer versions", i, total) self.mini_progress("layer versions", i, total)
logger.info("Fetching layer version dependencies") logger.info("Processing layer version dependencies")
# update layer dependencies
layerdependencies_info = _get_json_response(
apilinks['layerDependencies'] +
"?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
dependlist = {} dependlist = {}
for ldi in layerdependencies_info: for id in index.layerDependencies:
try: try:
lv = Layer_Version.objects.get( lv = Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']]) pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
except Layer_Version.DoesNotExist as e: except Layer_Version.DoesNotExist as e:
continue continue
if lv not in dependlist: if lv not in dependlist:
dependlist[lv] = [] dependlist[lv] = []
try: try:
layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']] layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
dependlist[lv].append( dependlist[lv].append(
Layer_Version.objects.get(layer__pk=layer_id, Layer_Version.objects.get(layer__pk=layer_id,
@@ -247,7 +225,7 @@ class Command(BaseCommand):
except Layer_Version.DoesNotExist: except Layer_Version.DoesNotExist:
logger.warning("Cannot find layer version (ls:%s)," logger.warning("Cannot find layer version (ls:%s),"
"up_id:%s lv:%s" % "up_id:%s lv:%s" %
(self, ldi['dependency'], lv)) (self, index.layerDependencies[id].dependency_id, lv))
total = len(dependlist) total = len(dependlist)
for i, lv in enumerate(dependlist): for i, lv in enumerate(dependlist):
@@ -258,73 +236,61 @@ class Command(BaseCommand):
self.mini_progress("Layer version dependencies", i, total) self.mini_progress("Layer version dependencies", i, total)
# update Distros # update Distros
logger.info("Fetching distro information") logger.info("Processing distro information")
distros_info = _get_json_response(
apilinks['distros'] + "?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
total = len(distros_info) total = len(index.distros)
for i, di in enumerate(distros_info): for i, id in enumerate(index.distros):
distro, created = Distro.objects.get_or_create( distro, created = Distro.objects.get_or_create(
name=di['name'], name=index.distros[id].name,
layer_version=Layer_Version.objects.get( layer_version=Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']])) pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
distro.up_date = di['updated'] distro.up_date = index.distros[id].updated
distro.name = di['name'] distro.name = index.distros[id].name
distro.description = di['description'] distro.description = index.distros[id].description
distro.save() distro.save()
self.mini_progress("distros", i, total) self.mini_progress("distros", i, total)
# update machines # update machines
logger.info("Fetching machine information") logger.info("Processing machine information")
machines_info = _get_json_response(
apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
total = len(machines_info) total = len(index.machines)
for i, mi in enumerate(machines_info): for i, id in enumerate(index.machines):
mo, created = Machine.objects.get_or_create( mo, created = Machine.objects.get_or_create(
name=mi['name'], name=index.machines[id].name,
layer_version=Layer_Version.objects.get( layer_version=Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']])) pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
mo.up_date = mi['updated'] mo.up_date = index.machines[id].updated
mo.name = mi['name'] mo.name = index.machines[id].name
mo.description = mi['description'] mo.description = index.machines[id].description
mo.save() mo.save()
self.mini_progress("machines", i, total) self.mini_progress("machines", i, total)
# update recipes; paginate by layer version / layer branch # update recipes; paginate by layer version / layer branch
logger.info("Fetching recipe information") logger.info("Processing recipe information")
recipes_info = _get_json_response(
apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
total = len(recipes_info) total = len(index.recipes)
for i, ri in enumerate(recipes_info): for i, id in enumerate(index.recipes):
try: try:
lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']] lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
lv = Layer_Version.objects.get(pk=lv_id) lv = Layer_Version.objects.get(pk=lv_id)
ro, created = Recipe.objects.get_or_create( ro, created = Recipe.objects.get_or_create(
layer_version=lv, layer_version=lv,
name=ri['pn'] name=index.recipes[id].pn
) )
ro.layer_version = lv ro.layer_version = lv
ro.up_date = ri['updated'] ro.up_date = index.recipes[id].updated
ro.name = ri['pn'] ro.name = index.recipes[id].pn
ro.version = ri['pv'] ro.version = index.recipes[id].pv
ro.summary = ri['summary'] ro.summary = index.recipes[id].summary
ro.description = ri['description'] ro.description = index.recipes[id].description
ro.section = ri['section'] ro.section = index.recipes[id].section
ro.license = ri['license'] ro.license = index.recipes[id].license
ro.homepage = ri['homepage'] ro.homepage = index.recipes[id].homepage
ro.bugtracker = ri['bugtracker'] ro.bugtracker = index.recipes[id].bugtracker
ro.file_path = ri['filepath'] + "/" + ri['filename'] ro.file_path = index.recipes[id].fullpath
if 'inherits' in ri: ro.is_image = 'image' in index.recipes[id].inherits.split()
ro.is_image = 'image' in ri['inherits'].split()
else: # workaround for old style layer index
ro.is_image = "-image-" in ri['pn']
ro.save() ro.save()
except Exception as e: except Exception as e:
logger.warning("Failed saving recipe %s", e) logger.warning("Failed saving recipe %s", e)