package management: Allow dynamic loading of PM

Dynamic loading of package managers will allow other layers to simply
add their package manager code in package_manager/ and have bitbake find
it according to the package manager configuration. This is useful for
adding new (faster) package managers to Open Embedded while not increasing the
test scope or require Open Embedded to support more package managers.

How this is tested:
* Build core-image-minimal with all three package managers
* Build the sdk with all three package managers. dpkg fails, but
  it fails on master as well.
* Run the complete test suite, all tests passed except 16
* Run those 16 tests on master and verify that they fail there as well
* Fix errors making tests works on master but not with this patch.

(From OE-Core rev: 02670501dea192879ddf9f8048eea57a94719fc1)

Signed-off-by: Fredrik Gustafsson <fredrigu@axis.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/lib/oe/manifest.py

@@ -191,14 +191,8 @@ class Manifest(object, metaclass=ABCMeta):
 
 def create_manifest(d, final_manifest=False, manifest_dir=None,
                     manifest_type=Manifest.MANIFEST_TYPE_IMAGE):
-    from oe.package_manager.rpm.manifest import RpmManifest
-    from oe.package_manager.ipk.manifest import OpkgManifest
-    from oe.package_manager.deb.manifest import DpkgManifest
-    manifest_map = {'rpm': RpmManifest,
-                    'ipk': OpkgManifest,
-                    'deb': DpkgManifest}
-
-    manifest = manifest_map[d.getVar('IMAGE_PKGTYPE')](d, manifest_dir, manifest_type)
+    import importlib
+    manifest = importlib.import_module('oe.package_manager.' + d.getVar('IMAGE_PKGTYPE') + '.manifest').PkgManifest(d, manifest_dir, manifest_type)
 
     if final_manifest:
         manifest.create_final()

meta/lib/oe/package_manager/deb/__init__.py

@@ -79,7 +79,7 @@ class DpkgIndexer(Indexer):
         if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
             raise NotImplementedError('Package feed signing not implementd for dpkg')
 
-class DpkgPkgsList(PkgsList):
+class PMPkgsList(PkgsList):
 
     def list_pkgs(self):
         cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"),
@@ -461,7 +461,7 @@ class DpkgPM(OpkgDpkgPM):
                      "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
 
     def list_installed(self):
-        return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
+        return PMPkgsList(self.d, self.target_rootfs).list_pkgs()
 
     def package_info(self, pkg):
         """

meta/lib/oe/package_manager/deb/manifest.py

@@ -4,7 +4,7 @@
 
 from oe.manifest import Manifest
 
-class DpkgManifest(Manifest):
+class PkgManifest(Manifest):
     def create_initial(self):
         with open(self.initial_manifest, "w+") as manifest:
             manifest.write(self.initial_manifest_file_header)

meta/lib/oe/package_manager/deb/rootfs.py

@@ -7,7 +7,7 @@ import shutil
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
 from oe.utils import execute_pre_post_process
-from oe.package_manager.deb.manifest import DpkgManifest
+from oe.package_manager.deb.manifest import PkgManifest
 from oe.package_manager.deb import DpkgPM
 
 class DpkgOpkgRootfs(Rootfs):
@@ -120,9 +120,9 @@ class DpkgOpkgRootfs(Rootfs):
 
             num += 1
 
-class DpkgRootfs(DpkgOpkgRootfs):
+class PkgRootfs(DpkgOpkgRootfs):
     def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None):
-        super(DpkgRootfs, self).__init__(d, progress_reporter, logcatcher)
+        super(PkgRootfs, self).__init__(d, progress_reporter, logcatcher)
         self.log_check_regex = '^E:'
         self.log_check_expected_regexes = \
         [
@@ -131,7 +131,7 @@ class DpkgRootfs(DpkgOpkgRootfs):
 
         bb.utils.remove(self.image_rootfs, True)
         bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True)
-        self.manifest = DpkgManifest(d, manifest_dir)
+        self.manifest = PkgManifest(d, manifest_dir)
         self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'),
                          d.getVar('PACKAGE_ARCHS'),
                          d.getVar('DPKG_ARCH'))

meta/lib/oe/package_manager/deb/sdk.py

@@ -8,19 +8,19 @@ from oe.utils import execute_pre_post_process
 from oe.sdk import Sdk
 from oe.manifest import Manifest
 from oe.package_manager.deb import DpkgPM
+from oe.package_manager.deb.manifest import PkgManifest
 
-class DpkgSdk(Sdk):
+class PkgSdk(Sdk):
     def __init__(self, d, manifest_dir=None):
-        super(DpkgSdk, self).__init__(d, manifest_dir)
+        super(PkgSdk, self).__init__(d, manifest_dir)
 
         self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt")
         self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt-sdk")
 
-        from oe.package_manager.deb.manifest import DpkgManifest
 
-        self.target_manifest = DpkgManifest(d, self.manifest_dir,
+        self.target_manifest = PkgManifest(d, self.manifest_dir,
                                             Manifest.MANIFEST_TYPE_SDK_TARGET)
-        self.host_manifest = DpkgManifest(d, self.manifest_dir,
+        self.host_manifest = PkgManifest(d, self.manifest_dir,
                                           Manifest.MANIFEST_TYPE_SDK_HOST)
 
         deb_repo_workdir = "oe-sdk-repo"

meta/lib/oe/package_manager/ipk/__init__.py

@@ -59,9 +59,10 @@ class OpkgIndexer(Indexer):
                                    self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
                                    armor=is_ascii_sig)
 
-class OpkgPkgsList(PkgsList):
-    def __init__(self, d, rootfs_dir, config_file):
-        super(OpkgPkgsList, self).__init__(d, rootfs_dir)
+class PMPkgsList(PkgsList):
+    def __init__(self, d, rootfs_dir):
+        super(PMPkgsList, self).__init__(d, rootfs_dir)
+        config_file = d.getVar("IPKGCONF_TARGET")
 
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
         self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
@@ -416,7 +417,7 @@ class OpkgPM(OpkgDpkgPM):
             bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True)
 
     def list_installed(self):
-        return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs()
+        return PMPkgsList(self.d, self.target_rootfs).list_pkgs()
 
     def dummy_install(self, pkgs):
         """

meta/lib/oe/package_manager/ipk/manifest.py

@@ -4,7 +4,7 @@
 
 from oe.manifest import Manifest
 
-class OpkgManifest(Manifest):
+class PkgManifest(Manifest):
     """
     Returns a dictionary object with mip and mlp packages.
     """

meta/lib/oe/package_manager/ipk/rootfs.py

@@ -8,7 +8,7 @@ import shutil
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
 from oe.utils import execute_pre_post_process
-from oe.package_manager.ipk.manifest import OpkgManifest
+from oe.package_manager.ipk.manifest import PkgManifest
 from oe.package_manager.ipk import OpkgPM
 
 class DpkgOpkgRootfs(Rootfs):
@@ -121,12 +121,12 @@ class DpkgOpkgRootfs(Rootfs):
 
             num += 1
 
-class OpkgRootfs(DpkgOpkgRootfs):
+class PkgRootfs(DpkgOpkgRootfs):
     def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None):
-        super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher)
+        super(PkgRootfs, self).__init__(d, progress_reporter, logcatcher)
         self.log_check_regex = '(exit 1|Collected errors)'
 
-        self.manifest = OpkgManifest(d, manifest_dir)
+        self.manifest = PkgManifest(d, manifest_dir)
         self.opkg_conf = self.d.getVar("IPKGCONF_TARGET")
         self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")
 

meta/lib/oe/package_manager/ipk/sdk.py

@@ -6,20 +6,20 @@ import glob
 import shutil
 from oe.utils import execute_pre_post_process
 from oe.sdk import Sdk
+from oe.package_manager.ipk.manifest import PkgManifest
 from oe.manifest import Manifest
 from oe.package_manager.ipk import OpkgPM
 
-class OpkgSdk(Sdk):
+class PkgSdk(Sdk):
     def __init__(self, d, manifest_dir=None):
-        super(OpkgSdk, self).__init__(d, manifest_dir)
+        super(PkgSdk, self).__init__(d, manifest_dir)
 
         self.target_conf = self.d.getVar("IPKGCONF_TARGET")
         self.host_conf = self.d.getVar("IPKGCONF_SDK")
 
-        from oe.package_manager.ipk.manifest import OpkgManifest
-        self.target_manifest = OpkgManifest(d, self.manifest_dir,
+        self.target_manifest = PkgManifest(d, self.manifest_dir,
                                             Manifest.MANIFEST_TYPE_SDK_TARGET)
-        self.host_manifest = OpkgManifest(d, self.manifest_dir,
+        self.host_manifest = PkgManifest(d, self.manifest_dir,
                                           Manifest.MANIFEST_TYPE_SDK_HOST)
 
         ipk_repo_workdir = "oe-sdk-repo"

meta/lib/oe/package_manager/rpm/__init__.py

@@ -43,7 +43,7 @@ class RpmSubdirIndexer(RpmIndexer):
                         self.do_write_index(dir_path)
 
 
-class RpmPkgsList(PkgsList):
+class PMPkgsList(PkgsList):
     def list_pkgs(self):
         return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR'), needfeed=False).list_installed()
 

meta/lib/oe/package_manager/rpm/manifest.py

@@ -4,7 +4,7 @@
 
 from oe.manifest import Manifest
 
-class RpmManifest(Manifest):
+class PkgManifest(Manifest):
     """
     Returns a dictionary object with mip and mlp packages.
     """

meta/lib/oe/package_manager/rpm/rootfs.py

@@ -5,17 +5,17 @@
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
 from oe.utils import execute_pre_post_process
-from oe.package_manager.rpm.manifest import RpmManifest
+from oe.package_manager.rpm.manifest import PkgManifest
 from oe.package_manager.rpm import RpmPM
 
-class RpmRootfs(Rootfs):
+class PkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None):
-        super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher)
+        super(PkgRootfs, self).__init__(d, progress_reporter, logcatcher)
         self.log_check_regex = r'(unpacking of archive failed|Cannot find package'\
                                r'|exit 1|ERROR: |Error: |Error |ERROR '\
                                r'|Failed |Failed: |Failed$|Failed\(\d+\):)'
 
-        self.manifest = RpmManifest(d, manifest_dir)
+        self.manifest = PkgManifest(d, manifest_dir)
 
         self.pm = RpmPM(d,
                         d.getVar('IMAGE_ROOTFS'),

meta/lib/oe/package_manager/rpm/sdk.py

@@ -6,16 +6,16 @@ import glob
 from oe.utils import execute_pre_post_process
 from oe.sdk import Sdk
 from oe.manifest import Manifest
+from oe.package_manager.rpm.manifest import PkgManifest
 from oe.package_manager.rpm import RpmPM
 
-class RpmSdk(Sdk):
+class PkgSdk(Sdk):
     def __init__(self, d, manifest_dir=None, rpm_workdir="oe-sdk-repo"):
-        super(RpmSdk, self).__init__(d, manifest_dir)
+        super(PkgSdk, self).__init__(d, manifest_dir)
 
-        from oe.package_manager.rpm.manifest import RpmManifest
-        self.target_manifest = RpmManifest(d, self.manifest_dir,
+        self.target_manifest = PkgManifest(d, self.manifest_dir,
                                            Manifest.MANIFEST_TYPE_SDK_TARGET)
-        self.host_manifest = RpmManifest(d, self.manifest_dir,
+        self.host_manifest = PkgManifest(d, self.manifest_dir,
                                          Manifest.MANIFEST_TYPE_SDK_HOST)
 
         rpm_repo_workdir = "oe-sdk-repo"

meta/lib/oe/rootfs.py

@@ -10,12 +10,6 @@ import shutil
 import os
 import subprocess
 import re
-from oe.package_manager.rpm.manifest import RpmManifest
-from oe.package_manager.ipk.manifest import OpkgManifest
-from oe.package_manager.deb.manifest import DpkgManifest
-from oe.package_manager.rpm import RpmPkgsList
-from oe.package_manager.ipk import OpkgPkgsList
-from oe.package_manager.deb import DpkgPkgsList
 
 class Rootfs(object, metaclass=ABCMeta):
     """
@@ -360,12 +354,9 @@ class Rootfs(object, metaclass=ABCMeta):
 
 
 def get_class_for_type(imgtype):
-    from oe.package_manager.rpm.rootfs import RpmRootfs
-    from oe.package_manager.ipk.rootfs import OpkgRootfs
-    from oe.package_manager.deb.rootfs import DpkgRootfs
-    return {"rpm": RpmRootfs,
-            "ipk": OpkgRootfs,
-            "deb": DpkgRootfs}[imgtype]
+    import importlib
+    mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs')
+    return mod.PkgRootfs
 
 def variable_depends(d, manifest_dir=None):
     img_type = d.getVar('IMAGE_PKGTYPE')
@@ -375,17 +366,10 @@ def variable_depends(d, manifest_dir=None):
 def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
     env_bkp = os.environ.copy()
 
-    from oe.package_manager.rpm.rootfs import RpmRootfs
-    from oe.package_manager.ipk.rootfs import OpkgRootfs
-    from oe.package_manager.deb.rootfs import DpkgRootfs
     img_type = d.getVar('IMAGE_PKGTYPE')
-    if img_type == "rpm":
-        RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
-    elif img_type == "ipk":
-        OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
-    elif img_type == "deb":
-        DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
 
+    cls = get_class_for_type(img_type)
+    cls(d, manifest_dir, progress_reporter, logcatcher).create()
     os.environ.clear()
     os.environ.update(env_bkp)
 
@@ -395,12 +379,10 @@ def image_list_installed_packages(d, rootfs_dir=None):
         rootfs_dir = d.getVar('IMAGE_ROOTFS')
 
     img_type = d.getVar('IMAGE_PKGTYPE')
-    if img_type == "rpm":
-        return RpmPkgsList(d, rootfs_dir).list_pkgs()
-    elif img_type == "ipk":
-        return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs()
-    elif img_type == "deb":
-        return DpkgPkgsList(d, rootfs_dir).list_pkgs()
+
+    import importlib
+    cls = importlib.import_module('oe.package_manager.' + img_type)
+    return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
 
 if __name__ == "__main__":
     """

meta/lib/oe/sdk.py

@@ -115,33 +115,18 @@ def sdk_list_installed_packages(d, target, rootfs_dir=None):
 
         rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True]
 
-    from oe.package_manager.rpm import RpmPkgsList
-    from oe.package_manager.ipk import OpkgPkgsList
-    from oe.package_manager.deb import DpkgPkgsList
     img_type = d.getVar('IMAGE_PKGTYPE')
-    if img_type == "rpm":
-        arch_var = ["SDK_PACKAGE_ARCHS", None][target is True]
-        os_var = ["SDK_OS", None][target is True]
-        return RpmPkgsList(d, rootfs_dir).list_pkgs()
-    elif img_type == "ipk":
-        conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True]
-        return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs()
-    elif img_type == "deb":
-        return DpkgPkgsList(d, rootfs_dir).list_pkgs()
+    import importlib
+    cls = importlib.import_module('oe.package_manager.' + img_type)
+    return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
 
 def populate_sdk(d, manifest_dir=None):
     env_bkp = os.environ.copy()
 
     img_type = d.getVar('IMAGE_PKGTYPE')
-    from oe.package_manager.rpm.sdk import RpmSdk
-    from oe.package_manager.ipk.sdk import OpkgSdk
-    from oe.package_manager.deb.sdk import DpkgSdk
-    if img_type == "rpm":
-        RpmSdk(d, manifest_dir).populate()
-    elif img_type == "ipk":
-        OpkgSdk(d, manifest_dir).populate()
-    elif img_type == "deb":
-        DpkgSdk(d, manifest_dir).populate()
+    import importlib
+    cls = importlib.import_module('oe.package_manager.' + img_type + '.sdk')
+    cls.PkgSdk(d, manifest_dir).populate()
 
     os.environ.clear()
     os.environ.update(env_bkp)