devtool: add support for git submodules

Adding the support of submodules required a lot of changes on the
internal data structures:
* initial_rev/startcommit used as a starting point for looking at new
  / updated commits was replaced by a dictionary where the keys are the
  submodule name ("." for main repo) and the values are the
  initial_rev/startcommit

* the extractPatches function now extracts patch for the main repo and
  for all submodules and stores them in a hierarchical way describing the
    submodule path

* store initial_rev/commit also for all submodules inside the recipe
  bbappend file

* _export_patches now returns dictionaries that contains the 'patchdir'
  parameter (if any). This parameter is used to add the correct
  'patchdir=' parameter on the recipe

Also, recipe can extract a secondary git tree inside the workdir.

By default, at the end of the do_patch function, there is a hook in
devtool that commits everything that was modified to have a clean
repository. It uses the command: "git add .; git commit ..."

The issue here is that, it adds the secondary git tree as a submodule
but in a wrong way. Doing "git add <git dir>" declares a submodule but do
not adds a url associated to it, and all following "git submodule foreach"
commands will fail.

So detect that a git tree was extracted inside S and correctly add it
using "git submodule add <url> <path>", so that it will be considered as a
regular git submodule

(From OE-Core rev: 900129cbdf25297a42ab5dbd02d1adbea405c935)

Signed-off-by: Julien Stephan <jstephan@baylibre.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/lib/devtool/upgrade.py

@@ -90,7 +90,7 @@ def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
     _rename_recipe_dirs(oldpv, newpv, path)
     return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
 
-def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, rev, copied, workspace, d):
+def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d):
     """Writes an append file"""
     if not os.path.exists(rc):
         raise DevtoolError("bbappend not created because %s does not exist" % rc)
@@ -119,8 +119,9 @@ def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, rev, copied,
         if b_is_s:
             f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
         f.write('\n')
-        if rev:
-            f.write('# initial_rev: %s\n' % rev)
+        if revs:
+            for name, rev in revs.items():
+                f.write('# initial_rev %s: %s\n' % (name, rev))
         if copied:
             f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
             f.write('# original_files: %s\n' % ' '.join(copied))
@@ -182,10 +183,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
     uri, rev = _get_uri(crd)
     if srcrev:
         rev = srcrev
+    paths = [srctree]
     if uri.startswith('git://') or uri.startswith('gitsm://'):
         __run('git fetch')
         __run('git checkout %s' % rev)
         __run('git tag -f devtool-base-new')
+        __run('git submodule update --recursive')
+        __run('git submodule foreach \'git tag -f devtool-base-new\'')
+        (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'')
+        paths += [os.path.join(srctree, p) for p in stdout.splitlines()]
         md5 = None
         sha256 = None
         _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
@@ -256,29 +262,32 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
         __run('git tag -f devtool-base-%s' % newpv)
 
-    (stdout, _) = __run('git rev-parse HEAD')
-    rev = stdout.rstrip()
+    revs = {}
+    for path in paths:
+        (stdout, _) = _run('git rev-parse HEAD', cwd=path)
+        revs[os.path.relpath(path,srctree)] = stdout.rstrip()
 
     if no_patch:
         patches = oe.recipeutils.get_recipe_patches(crd)
         if patches:
             logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n  %s' % '\n  '.join([os.path.basename(patch) for patch in patches]))
     else:
-        __run('git checkout devtool-patched -b %s' % branch)
-        (stdout, _) = __run('git branch --list devtool-override-*')
-        branches_to_rebase = [branch] + stdout.split()
-        for b in branches_to_rebase:
-            logger.info("Rebasing {} onto {}".format(b, rev))
-            __run('git checkout %s' % b)
-            try:
-                __run('git rebase %s' % rev)
-            except bb.process.ExecutionError as e:
-                if 'conflict' in e.stdout:
-                    logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
-                    __run('git rebase --abort')
-                else:
-                    logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
-        __run('git checkout %s' % branch)
+        for path in paths:
+            _run('git checkout devtool-patched -b %s' % branch, cwd=path)
+            (stdout, _) = _run('git branch --list devtool-override-*', cwd=path)
+            branches_to_rebase = [branch] + stdout.split()
+            for b in branches_to_rebase:
+                logger.info("Rebasing {} onto {}".format(b, revs[os.path.relpath(path,srctree)]))
+                _run('git checkout %s' % b, cwd=path)
+                try:
+                    _run('git rebase %s' % revs[os.path.relpath(path, srctree)], cwd=path)
+                except bb.process.ExecutionError as e:
+                    if 'conflict' in e.stdout:
+                        logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
+                        _run('git rebase --abort', cwd=path)
+                    else:
+                        logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+            _run('git checkout %s' % branch, cwd=path)
 
     if tmpsrctree:
         if keep_temp:
@@ -288,7 +297,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
             if tmpdir != tmpsrctree:
                 shutil.rmtree(tmpdir)
 
-    return (rev, md5, sha256, srcbranch, srcsubdir_rel)
+    return (revs, md5, sha256, srcbranch, srcsubdir_rel)
 
 def _add_license_diff_to_recipe(path, diff):
     notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.