Add multi-manifest support with <submanifest> element

To be addressed in another change:
 - a partial `repo sync` (with a list of projects/paths to sync)
   requires `--this-tree-only`.

Change-Id: I6c7400bf001540e9d7694fa70934f8f204cb5f57
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322657
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

SUBMITTING_PATCHES.md

@@ -3,7 +3,7 @@
 # Short Version
 
  - Make small logical changes.
- - Provide a meaningful commit message.
+ - [Provide a meaningful commit message][commit-message-style].
  - Check for coding errors and style nits with flake8.
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
@@ -26,10 +26,11 @@ yourself with the following relevant bits.
 
 ## Make separate commits for logically separate changes.
 
-Unless your patch is really trivial, you should not be sending
-out a patch that was generated between your working tree and your
-commit head.  Instead, always make a commit with complete commit
-message and generate a series of patches from your repository.
+Unless your patch is really trivial, you should not be sending out a patch that
+was generated between your working tree and your commit head.
+Instead, always make a commit with a complete
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
 It is a good discipline.
 
 Describe the technical detail of the change(s).
@@ -171,3 +172,6 @@ After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
 The maintainer will then submit your changes to the repository.
+
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

command.py

@@ -61,13 +61,21 @@ class Command(object):
   # it is the number of parallel jobs to default to.
   PARALLEL_JOBS = None
 
+  # Whether this command supports Multi-manifest.  If False, then main.py will
+  # iterate over the manifests and invoke the command once per (sub)manifest.
+  # This is only checked after calling ValidateOptions, so that partially
+  # migrated subcommands can set it to False.
+  MULTI_MANIFEST_SUPPORT = True
+
   def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
-               git_event_log=None):
+               git_event_log=None, outer_client=None, outer_manifest=None):
     self.repodir = repodir
     self.client = client
+    self.outer_client = outer_client or client
     self.manifest = manifest
     self.gitc_manifest = gitc_manifest
     self.git_event_log = git_event_log
+    self.outer_manifest = outer_manifest
 
     # Cache for the OptionParser property.
     self._optparse = None
@@ -135,6 +143,18 @@ class Command(object):
           type=int, default=self.PARALLEL_JOBS,
           help=f'number of jobs to run in parallel (default: {default})')
 
+    m = p.add_option_group('Multi-manifest options')
+    m.add_option('--outer-manifest', action='store_true',
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', default=None,
+                 help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
+
   def _Options(self, p):
     """Initialize the option parser with subcommand-specific options."""
 
@@ -252,16 +272,19 @@ class Command(object):
     return project
 
   def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
-                  submodules_ok=False):
+                  submodules_ok=False, all_manifests=False):
     """A list of projects that match the arguments.
     """
-    if not manifest:
-      manifest = self.manifest
-    all_projects_list = manifest.projects
+    if all_manifests:
+      if not manifest:
+        manifest = self.manifest.outer_client
+      all_projects_list = manifest.all_projects
+    else:
+      if not manifest:
+        manifest = self.manifest
+      all_projects_list = manifest.projects
     result = []
 
-    mp = manifest.manifestProject
-
     if not groups:
       groups = manifest.GetGroupsStr()
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
@@ -282,12 +305,19 @@ class Command(object):
       for arg in args:
         # We have to filter by manifest groups in case the requested project is
         # checked out multiple times or differently based on them.
-        projects = [project for project in manifest.GetProjectsWithName(arg)
+        projects = [project for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests)
                     if project.MatchesGroups(groups)]
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(manifest, path)
+          tree = manifest
+          if all_manifests:
+            # Look for the deepest matching submanifest.
+            for tree in reversed(list(manifest.all_manifests)):
+              if path.startswith(tree.topdir):
+                break
+          project = self._GetProjectByPath(tree, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -308,7 +338,8 @@ class Command(object):
 
         for project in projects:
           if not missing_ok and not project.Exists:
-            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+            raise NoSuchProjectError('%s (%s)' % (
+                arg, project.RelPath(local=not all_manifests)))
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 
@@ -319,12 +350,22 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
-  def FindProjects(self, args, inverse=False):
+  def FindProjects(self, args, inverse=False, all_manifests=False):
+    """Find projects from command line arguments.
+
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      inverse: a boolean, if True, then projects not matching any |args| are
+               returned.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+    """
     result = []
     patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
-    for project in self.GetProjects(''):
+    for project in self.GetProjects('', all_manifests=all_manifests):
+      paths = [project.name, project.RelPath(local=not all_manifests)]
       for pattern in patterns:
-        match = pattern.search(project.name) or pattern.search(project.relpath)
+        match = any(pattern.search(x) for x in paths)
         if not inverse and match:
           result.append(project)
           break
@@ -333,9 +374,24 @@ class Command(object):
       else:
         if inverse:
           result.append(project)
-    result.sort(key=lambda project: project.relpath)
+    result.sort(key=lambda project: (project.manifest.path_prefix,
+                                     project.relpath))
     return result
 
+  def ManifestList(self, opt):
+    """Yields all of the manifests to traverse.
+
+    Args:
+      opt: The command options.
+    """
+    top = self.outer_manifest
+    if opt.outer_manifest is False or opt.this_manifest_only:
+      top = self.manifest
+    yield top
+    if not opt.this_manifest_only:
+      for child in top.all_children:
+        yield child
+
 
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and

docs/internal-fs-layout.md

@@ -50,6 +50,10 @@ For example, if you want to change the manifest branch, you can simply run
 For more documentation on the manifest format, including the local_manifests
 support, see the [manifest-format.md] file.
 
+*   `submanifests/{submanifest.path}/`: The path prefix to the manifest state of
+    a submanifest included in a multi-manifest checkout.  The outermost manifest
+    manifest state is found adjacent to `submanifests/`.
+
 *   `manifests/`: A git checkout of the manifest project.  Its `.git/` state
     points to the `manifest.git` bare checkout (see below).  It tracks the git
     branch specified at `repo init` time via `--manifest-branch`.
@@ -157,11 +161,13 @@ User controlled settings are initialized when running `repo init`.
 | Setting                  | `repo init` Option        | Use/Meaning |
 |-------------------       |---------------------------|-------------|
 | manifest.groups          | `--groups` & `--platform` | The manifest groups to sync |
+| manifest.standalone      | `--standalone-manifest`   | Download manifest as static file instead of creating checkout |
 | repo.archive             | `--archive`               | Use `git archive` for checkouts |
 | repo.clonebundle         | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
 | repo.mirror              | `--mirror`                | Checkout is a repo mirror |
 | repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
 | repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
@@ -253,6 +259,7 @@ Repo will create & maintain a few files in the user's home directory.
 
 
 [git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
 [git worktree]: https://git-scm.com/docs/git-worktree
 [gitsubmodules]: https://git-scm.com/docs/gitsubmodules
 [manifest-format.md]: ./manifest-format.md

docs/manifest-format.md

@@ -26,6 +26,7 @@ following DTD:
                       remote*,
                       default?,
                       manifest-server?,
+                      submanifest*?,
                       remove-project*,
                       project*,
                       extend-project*,
@@ -57,6 +58,15 @@ following DTD:
   <!ELEMENT manifest-server EMPTY>
   <!ATTLIST manifest-server url CDATA #REQUIRED>
 
+  <!ELEMENT submanifest EMPTY>
+  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest remote         IDREF #IMPLIED>
+  <!ATTLIST submanifest project        CDATA #IMPLIED>
+  <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
+  <!ATTLIST submanifest revision       CDATA #IMPLIED>
+  <!ATTLIST submanifest path           CDATA #IMPLIED>
+  <!ATTLIST submanifest groups         CDATA #IMPLIED>
+
   <!ELEMENT project (annotation*,
                      project*,
                      copyfile*,
@@ -90,6 +100,7 @@ following DTD:
   <!ELEMENT extend-project EMPTY>
   <!ATTLIST extend-project name CDATA #REQUIRED>
   <!ATTLIST extend-project path CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-path CDATA #IMPLIED>
   <!ATTLIST extend-project groups CDATA #IMPLIED>
   <!ATTLIST extend-project revision CDATA #IMPLIED>
   <!ATTLIST extend-project remote CDATA #IMPLIED>
@@ -103,8 +114,9 @@ following DTD:
   <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
 
   <!ELEMENT superproject EMPTY>
-  <!ATTLIST superproject name    CDATA #REQUIRED>
-  <!ATTLIST superproject remote  IDREF #IMPLIED>
+  <!ATTLIST superproject name     CDATA #REQUIRED>
+  <!ATTLIST superproject remote   IDREF #IMPLIED>
+  <!ATTLIST superproject revision CDATA #IMPLIED>
 
   <!ELEMENT contactinfo EMPTY>
   <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -234,6 +246,60 @@ the specified tag. This is used by repo sync when the --smart-tag option
 is given.
 
 
+### Element submanifest
+
+One or more submanifest elements may be specified.  Each element describes a
+single manifest to be checked out as a child.
+
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below.  The same name can be
+used for submanifests with different parent (sub)manifests.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `project`: The manifest project name.  The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+
+Attribute `manifest-name`: The manifest filename in the manifest project.  If
+not supplied, `default.xml` is used.
+
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash.  If not supplied, `name` is
+used.
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the submanifest repo client top directory
+should be placed.  If not supplied, `revision` is used.
+
+`path` may not be an absolute path or use "." or ".." path components.
+
+Attribute `groups`: List of additional groups to which all projects
+in the included submanifest belong. This appends and recurses, meaning
+all projects in submanifests carry all parent submanifest groups.
+Same syntax as the corresponding element of `project`.
+
+
 ### Element project
 
 One or more project elements may be specified.  Each element
@@ -336,6 +402,11 @@ against changes to the original manifest.
 Attribute `path`: If specified, limit the change to projects checked out
 at the specified path, rather than all projects with the given name.
 
+Attribute `dest-path`: If specified, a path relative to the top directory
+of the repo client where the Git working directory for this project
+should be placed.  This is used to move a project in the checkout by
+overriding the existing `path` setting.
+
 Attribute `groups`: List of additional groups to which this project
 belongs.  Same syntax as the corresponding element of `project`.
 
@@ -432,6 +503,11 @@ same meaning as project's name attribute. See the
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
 
+Attribute `revision`: Name of the Git branch the manifest wants
+to track for this superproject. If not supplied the revision given
+by the remote element is used if applicable, else the default
+element is used.
+
 ### Element contactinfo
 
 ***
@@ -459,7 +535,7 @@ These restrictions are not enforced for [Local Manifests].
 
 Attribute `groups`: List of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
-all projects in sub-manifests carry all parent include groups.
+all projects in included manifests carry all parent include groups.
 Same syntax as the corresponding element of `project`.
 
 ## Local Manifests {#local-manifests}

docs/release-process.md

@@ -208,85 +208,132 @@ Things in bold indicate stuff to take note of, but does not guarantee that we
 still support them.
 Things in italics are things we used to care about but probably don't anymore.
 
-|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python |
-|:--------:|:------------:|--------------|-----------------|-----------------------------------|-----|--------|
-| Oct 2008 | *Oct 2013*   |              | 2.6.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
+|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|-----|--------|-----|
+| Apr 2008 |              |              |                 | 5.0          |
+| Jun 2008 |              |              |                 | 5.1          |
+| Oct 2008 | *Oct 2013*   |              | 2.6.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
 | Dec 2008 | *Feb 2009*   |              | 3.0.0           |
-| Feb 2009 | *Mar 2012*   |              |                 | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
-| Jun 2009 | *Jun 2016*   |              | 3.1.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
-| Feb 2010 | *Oct 2012*   | 1.7.0        |                 | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
-| Apr 2010 | *Apr 2015*   |              |                 | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  |
-| Jul 2010 | *Dec 2019*   |              | **2.7.0**       | 11.04 Natty - **<current>** |
-| Oct 2010 |              |              |                 | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  |
-| Feb 2011 | *Feb 2016*   |              |                 | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
-| Apr 2011 |              |              |                 | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  |
-| Oct 2011 | *Feb 2016*   |              | 3.2.0           | 11.04 Natty - 12.10 Quantal |
-| Oct 2011 |              |              |                 | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  |
-| Apr 2012 | *Apr 2019*   |              |                 | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  |
-| Sep 2012 | *Sep 2017*   |              | 3.3.0           | 13.04 Raring - 13.10 Saucy |
-| Oct 2012 | *Dec 2014*   | 1.8.0        |                 | 13.04 Raring - 13.10 Saucy |
-| Oct 2012 |              |              |                 | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  |
-| Apr 2013 |              |              |                 | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  |
-| May 2013 | *May 2018*   |              |                 | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
-| Oct 2013 |              |              |                 | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  |
-| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 | **14.04 Trusty** |
-| Mar 2014 | *Mar 2019*   |              | **3.4.0**       | **14.04 Trusty** - 15.10 Wily / **Jessie** |
-| Apr 2014 | **Apr 2022** |              |                 | **14.04 Trusty**     | 1.9.1    | 2.7.5 3.4.0  |
+| Feb 2009 |              |              |                 | 5.2          |
+| Feb 2009 | *Mar 2012*   |              |                 |              | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
+| Jun 2009 | *Jun 2016*   |              | 3.1.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Sep 2009 |              |              |                 | 5.3          | *10.04 Lucid* |
+| Feb 2010 | *Oct 2012*   | 1.7.0        |                 |              | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
+| Mar 2010 |              |              |                 | 5.4          |
+| Apr 2010 |              |              |                 | 5.5          | 10.10 Maverick |
+| Apr 2010 | *Apr 2015*   |              |                 |              | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  | 5.3 |
+| Jul 2010 | *Dec 2019*   |              | *2.7.0*         |              | 11.04 Natty - *<current>* |
+| Aug 2010 |              |              |                 | 5.6          |
+| Oct 2010 |              |              |                 |              | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  | 5.5 |
+| Jan 2011 |              |              |                 | 5.7          |
+| Feb 2011 |              |              |                 | 5.8          | 11.04 Natty |
+| Feb 2011 | *Feb 2016*   |              |                 |              | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
+| Apr 2011 |              |              |                 |              | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  | 5.8 |
+| Sep 2011 |              |              |                 | 5.9          | *12.04 Precise* |
+| Oct 2011 | *Feb 2016*   |              | 3.2.0           |              | 11.04 Natty - 12.10 Quantal |
+| Oct 2011 |              |              |                 |              | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  | 5.8 |
+| Apr 2012 |              |              |                 | 6.0          | 12.10 Quantal |
+| Apr 2012 | *Apr 2019*   |              |                 |              | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  | 5.9 |
+| Aug 2012 |              |              |                 | 6.1          | 13.04 Raring |
+| Sep 2012 | *Sep 2017*   |              | 3.3.0           |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 | *Dec 2014*   | 1.8.0        |                 |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 |              |              |                 |              | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  | 6.0 |
+| Mar 2013 |              |              |                 | 6.2          | 13.10 Saucy |
+| Apr 2013 |              |              |                 |              | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  | 6.1 |
+| May 2013 | *May 2018*   |              |                 |              | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
+| Sep 2013 |              |              |                 | 6.3          |
+| Oct 2013 |              |              |                 |              | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  | 6.2 |
+| Nov 2013 |              |              |                 | 6.4          |
+| Jan 2014 |              |              |                 | 6.5          |
+| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 |              | *14.04 Trusty* |
+| Mar 2014 | *Mar 2019*   |              | *3.4.0*         |              | *14.04 Trusty* - 15.10 Wily / *Jessie* |
+| Mar 2014 |              |              |                 | 6.6          | *14.04 Trusty* - 14.10 Utopic |
+| Apr 2014 | *Apr 2022*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
 | May 2014 | *Dec 2014*   | 2.0.0        |
-| Aug 2014 | *Dec 2014*   | **2.1.0**    |                 | 14.10 Utopic - 15.04 Vivid / **Jessie** |
-| Oct 2014 |              |              |                 | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  |
+| Aug 2014 | *Dec 2014*   | *2.1.0*      |                 |              | 14.10 Utopic - 15.04 Vivid / *Jessie* |
+| Oct 2014 |              |              |                 | 6.7          | 15.04 Vivid |
+| Oct 2014 |              |              |                 |              | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  | 6.6 |
 | Nov 2014 | *Sep 2015*   | 2.2.0        |
 | Feb 2015 | *Sep 2015*   | 2.3.0        |
+| Mar 2015 |              |              |                 | 6.8          |
 | Apr 2015 | *May 2017*   | 2.4.0        |
-| Apr 2015 | **Jun 2020** |              |                 | **Debian 8 Jessie**  | 2.1.4    | 2.7.9 3.4.2  |
-| Apr 2015 |              |              |                 | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  |
-| Jul 2015 | *May 2017*   | 2.5.0        |                 | 15.10 Wily |
+| Apr 2015 | *Jun 2020*   |              |                 |              | *Debian 8 Jessie*    | 2.1.4    | 2.7.9 3.4.2  |
+| Apr 2015 |              |              |                 |              | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  | 6.7 |
+| Jul 2015 | *May 2017*   | 2.5.0        |                 |              | 15.10 Wily |
+| Jul 2015 |              |              |                 | 6.9          | 15.10 Wily |
+| Aug 2015 |              |              |                 | 7.0          |
+| Aug 2015 |              |              |                 | 7.1          |
 | Sep 2015 | *May 2017*   | 2.6.0        |
-| Sep 2015 | **Sep 2020** |              | **3.5.0**       | **16.04 Xenial** - 17.04 Zesty / **Stretch** |
-| Oct 2015 |              |              |                 | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  |
-| Jan 2016 | *Jul 2017*   | **2.7.0**    |                 | **16.04 Xenial** |
+| Sep 2015 | *Sep 2020*   |              | *3.5.0*         |              | *16.04 Xenial* - 17.04 Zesty / *Stretch* |
+| Oct 2015 |              |              |                 |              | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  | 6.9 |
+| Jan 2016 | *Jul 2017*   | *2.7.0*      |                 |              | *16.04 Xenial* |
+| Feb 2016 |              |              |                 | 7.2          | *16.04 Xenial* |
 | Mar 2016 | *Jul 2017*   | 2.8.0        |
-| Apr 2016 | **Apr 2024** |              |                 | **16.04 Xenial**     | 2.7.4    | 2.7.11 3.5.1 |
-| Jun 2016 | *Jul 2017*   | 2.9.0        |                 | 16.10 Yakkety |
+| Apr 2016 | *Apr 2024*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
+| Jun 2016 | *Jul 2017*   | 2.9.0        |                 |              | 16.10 Yakkety |
+| Jul 2016 |              |              |                 | 7.3          | 16.10 Yakkety |
 | Sep 2016 | *Sep 2017*   | 2.10.0       |
-| Oct 2016 |              |              |                 | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 |
-| Nov 2016 | *Sep 2017*   | **2.11.0**   |                 | 17.04 Zesty / **Stretch** |
-| Dec 2016 | **Dec 2021** |              | **3.6.0**       | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Oct 2016 |              |              |                 |              | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 | 7.3 |
+| Nov 2016 | *Sep 2017*   | *2.11.0*     |                 |              | 17.04 Zesty / *Stretch* |
+| Dec 2016 | **Dec 2021** |              | **3.6.0**       |              | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Dec 2016 |              |              |                 | 7.4          | 17.04 Zesty / *Debian 9 Stretch* |
 | Feb 2017 | *Sep 2017*   | 2.12.0       |
-| Apr 2017 |              |              |                 | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 |
+| Mar 2017 |              |              |                 | 7.5          | 17.10 Artful |
+| Apr 2017 |              |              |                 |              | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
 | May 2017 | *May 2018*   | 2.13.0       |
-| Jun 2017 | **Jun 2022** |              |                 | **Debian 9 Stretch** | 2.11.0   | 2.7.13 3.5.3 |
-| Aug 2017 | *Dec 2019*   | 2.14.0       |                 | 17.10 Artful |
+| Jun 2017 | *Jun 2022*   |              |                 |              | *Debian 9 Stretch*   | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
+| Aug 2017 | *Dec 2019*   | 2.14.0       |                 |              | 17.10 Artful |
 | Oct 2017 | *Dec 2019*   | 2.15.0       |
-| Oct 2017 |              |              |                 | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 |
+| Oct 2017 |              |              |                 | 7.6          | **18.04 Bionic** |
+| Oct 2017 |              |              |                 |              | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 | 7.5 |
 | Jan 2018 | *Dec 2019*   | 2.16.0       |
-| Apr 2018 | *Dec 2019*   | 2.17.0       |                 | **18.04 Bionic**     |
-| Apr 2018 | **Apr 2028** |              |                 | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 |
-| Jun 2018 | *Dec 2019*   | 2.18.0       |
-| Jun 2018 | **Jun 2023** |              | 3.7.0           | 19.04 Disco - **20.04 Focal** / **Buster** |
-| Sep 2018 | *Dec 2019*   | 2.19.0       |                 | 18.10 Cosmic |
-| Oct 2018 |              |              |                 | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 |
-| Dec 2018 | *Dec 2019*   | **2.20.0**   |                 | 19.04 Disco / **Buster** |
-| Feb 2019 | *Dec 2019*   | 2.21.0       |
-| Apr 2019 |              |              |                 | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 |
+| Apr 2018 | *Mar 2021*   | **2.17.0**   |                 |              | **18.04 Bionic**     |
+| Apr 2018 |              |              |                 | 7.7          | 18.10 Cosmic |
+| Apr 2018 | **Apr 2028** |              |                 |              | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 | 7.6 |
+| Jun 2018 | *Mar 2021*   | 2.18.0       |
+| Jun 2018 | **Jun 2023** |              | 3.7.0           |              | 19.04 Disco - **20.04 Focal** / **Buster** |
+| Aug 2018 |              |              |                 | 7.8          |
+| Sep 2018 | *Mar 2021*   | 2.19.0       |                 |              | 18.10 Cosmic |
+| Oct 2018 |              |              |                 | 7.9          | 19.04 Disco / **Buster** |
+| Oct 2018 |              |              |                 |              | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 | 7.7 |
+| Dec 2018 | *Mar 2021*   | **2.20.0**   |                 |              | 19.04 Disco - 19.10 Eoan / **Buster** |
+| Feb 2019 | *Mar 2021*   | 2.21.0       |
+| Apr 2019 |              |              |                 | 8.0          | 19.10 Eoan |
+| Apr 2019 |              |              |                 |              | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
 | Jun 2019 |              | 2.22.0       |
-| Jul 2019 | **Jul 2024** |              |                 | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 |
-| Aug 2019 |              | 2.23.0       |
-| Oct 2019 | **Oct 2024** |              | 3.8.0           |
-| Oct 2019 |              |              |                 | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 |
-| Nov 2019 |              | 2.24.0       |
-| Jan 2020 |              | 2.25.0       |                 | **20.04 Focal** |
-| Apr 2020 | **Apr 2030** |              |                 | **20.04 Focal**      | 2.25.0   | 2.7.17 3.7.5 |
-| Oct 2020 | **Oct 2025** |              | 3.9.0           | 21.04 Hirsute / **Bullseye** |
-| Dec 2020 |              | 2.30.0       |                 | 21.04 Hirsute / **Bullseye** |
-| Apr 2021 | *Jan 2022*   |              |                 | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 |
-| Aug 2021 | **Aug 2026** |              |                 | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 |
-| **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** |
+| Jul 2019 | **Jul 2024** |              |                 |              | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
+| Aug 2019 | *Mar 2021*   | 2.23.0       |
+| Oct 2019 | **Oct 2024** |              | 3.8.0           |              | **20.04 Focal** - 20.10 Groovy |
+| Oct 2019 |              |              |                 | 8.1          |
+| Oct 2019 |              |              |                 |              | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 | 8.0 |
+| Nov 2019 | *Mar 2021*   | 2.24.0       |
+| Jan 2020 | *Mar 2021*   | 2.25.0       |                 |              | **20.04 Focal** |
+| Feb 2020 |              |              |                 | 8.2          | **20.04 Focal** |
+| Mar 2020 | *Mar 2021*   | 2.26.0       |
+| Apr 2020 | **Apr 2030** |              |                 |              | **20.04 Focal**      | 2.25.1   | 2.7.17 3.8.2 | 8.2 |
+| May 2020 | *Mar 2021*   | 2.27.0       |                 |              | 20.10 Groovy |
+| May 2020 |              |              |                 | 8.3          |
+| Jul 2020 | *Mar 2021*   | 2.28.0       |
+| Sep 2020 |              |              |                 | 8.4          | 21.04 Hirsute / **Bullseye** |
+| Oct 2020 | *Mar 2021*   | 2.29.0       |
+| Oct 2020 |              |              |                 |              | 20.10 Groovy         | 2.27.0   | 2.7.18 3.8.6 | 8.3 |
+| Oct 2020 | **Oct 2025** |              | 3.9.0           |              | 21.04 Hirsute / **Bullseye** |
+| Dec 2020 | *Mar 2021*   | 2.30.0       |                 |              | 21.04 Hirsute / **Bullseye** |
+| Mar 2021 |              | 2.31.0       |
+| Mar 2021 |              |              |                 | 8.5          |
+| Apr 2021 |              |              |                 | 8.6          |
+| Apr 2021 | *Jan 2022*   |              |                 |              | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 | 8.4 |
+| Jun 2021 |              | 2.32.0       |
+| Aug 2021 |              | 2.33.0       |
+| Aug 2021 |              |              |                 | 8.7          |
+| Aug 2021 | **Aug 2026** |              |                 |              | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
+| **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
 
 
 [contact]: ../README.md#contact
 [rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
 [rel-g]: https://en.wikipedia.org/wiki/Git#Releases
+[rel-o]: https://www.openssh.com/releasenotes.html
 [rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
 [rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
 [example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion

fetch.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module contains functions used to fetch files from various sources."""
+
+import subprocess
+import sys
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+
+def fetch_file(url, verbose=False):
+  """Fetch a file from the specified source using the appropriate protocol.
+
+  Returns:
+    The contents of the file as bytes.
+  """
+  scheme = urlparse(url).scheme
+  if scheme == 'gs':
+    cmd = ['gsutil', 'cat', url]
+    try:
+      result = subprocess.run(
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
+      return result.stdout
+    except subprocess.CalledProcessError as e:
+      print('fatal: error running "gsutil": %s' % e.stderr,
+            file=sys.stderr)
+    sys.exit(1)
+  with urlopen(url) as f:
+    return f.read()

git_command.py

@@ -169,7 +169,8 @@ class GitCommand(object):
                disable_editor=False,
                ssh_proxy=None,
                cwd=None,
-               gitdir=None):
+               gitdir=None,
+               objdir=None):
     env = self._GetBasicEnv()
 
     if disable_editor:
@@ -194,13 +195,24 @@ class GitCommand(object):
         cwd = project.worktree
       if not gitdir:
         gitdir = project.gitdir
+    # Git on Windows wants its paths only using / for reliability.
+    if platform_utils.isWindows():
+      if objdir:
+        objdir = objdir.replace('\\', '/')
+      if gitdir:
+        gitdir = gitdir.replace('\\', '/')
+
+    if objdir:
+      # Set to the place we want to save the objects.
+      env['GIT_OBJECT_DIRECTORY'] = objdir
+      if gitdir:
+        # Allow git to search the original place in case of local or unique refs
+        # that git will attempt to resolve even if we aren't fetching them.
+        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
 
     command = [GIT]
     if bare:
       if gitdir:
-        # Git on Windows wants its paths only using / for reliability.
-        if platform_utils.isWindows():
-          gitdir = gitdir.replace('\\', '/')
         env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
@@ -234,6 +246,11 @@ class GitCommand(object):
         dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
         LAST_GITDIR = env[GIT_DIR]
 
+      if 'GIT_OBJECT_DIRECTORY' in env:
+        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
+      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
+        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % env['GIT_ALTERNATE_OBJECT_DIRECTORIES']
+
       dbg += ': '
       dbg += ' '.join(command)
       if stdin == subprocess.PIPE:

git_config.py

@@ -104,6 +104,10 @@ class GitConfig(object):
           os.path.dirname(self.file),
           '.repo_' + os.path.basename(self.file) + '.json')
 
+  def ClearCache(self):
+    """Clear the in-memory cache of config."""
+    self._cache_dict = None
+
   def Has(self, name, include_defaults=True):
     """Return true if this configuration file has the key.
     """
@@ -349,7 +353,7 @@ class GitConfig(object):
       with open(self._json) as fd:
         return json.load(fd)
     except (IOError, ValueError):
-      platform_utils.remove(self._json)
+      platform_utils.remove(self._json, missing_ok=True)
       return None
 
   def _SaveJson(self, cache):
@@ -357,8 +361,7 @@ class GitConfig(object):
       with open(self._json, 'w') as fd:
         json.dump(cache, fd, indent=2)
     except (IOError, TypeError):
-      if os.path.exists(self._json):
-        platform_utils.remove(self._json)
+      platform_utils.remove(self._json, missing_ok=True)
 
   def _ReadGit(self):
     """
@@ -368,9 +371,10 @@ class GitConfig(object):
 
     """
     c = {}
-    d = self._do('--null', '--list')
-    if d is None:
+    if not os.path.exists(self.file):
       return c
+
+    d = self._do('--null', '--list')
     for line in d.rstrip('\0').split('\0'):
       if '\n' in line:
         key, val = line.split('\n', 1)
@@ -399,7 +403,7 @@ class GitConfig(object):
     if p.Wait() == 0:
       return p.stdout
     else:
-      GitError('git config %s: %s' % (str(args), p.stderr))
+      raise GitError('git config %s: %s' % (str(args), p.stderr))
 
 
 class RepoConfig(GitConfig):

git_superproject.py

@@ -32,7 +32,6 @@ from typing import NamedTuple
 from git_command import git_require, GitCommand
 from git_config import RepoConfig
 from git_refs import R_HEADS
-from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
@@ -90,16 +89,21 @@ class Superproject(object):
     self._git_event_log = git_event_log
     self._quiet = quiet
     self._print_messages = print_messages
-    self._branch = self._GetBranch()
+    self._branch = manifest.branch
     self._repodir = os.path.abspath(repodir)
     self._superproject_dir = superproject_dir
-    self._superproject_path = os.path.join(self._repodir, superproject_dir)
+    self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
+                                                          superproject_dir)
     self._manifest_path = os.path.join(self._superproject_path,
                                        _SUPERPROJECT_MANIFEST_NAME)
     git_name = ''
     if self._manifest.superproject:
-      remote_name = self._manifest.superproject['remote'].name
-      git_name = hashlib.md5(remote_name.encode('utf8')).hexdigest() + '-'
+      remote = self._manifest.superproject['remote']
+      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+      self._branch = self._manifest.superproject['revision']
+      self._remote_url = remote.url
+    else:
+      self._remote_url = None
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
@@ -113,30 +117,23 @@ class Superproject(object):
     """Returns the manifest path if the path exists or None."""
     return self._manifest_path if os.path.exists(self._manifest_path) else None
 
-  def _GetBranch(self):
-    """Returns the branch name for getting the approved manifest."""
-    p = self._manifest.manifestProject
-    b = p.GetBranch(p.CurrentBranch)
-    if not b:
-      return None
-    branch = b.merge
-    if branch and branch.startswith(R_HEADS):
-      branch = branch[len(R_HEADS):]
-    return branch
-
   def _LogMessage(self, message):
     """Logs message to stderr and _git_event_log."""
     if self._print_messages:
       print(message, file=sys.stderr)
     self._git_event_log.ErrorEvent(message, f'{message}')
 
+  def _LogMessagePrefix(self):
+    """Returns the prefix string to be logged in each log message"""
+    return f'repo superproject branch: {self._branch} url: {self._remote_url}'
+
   def _LogError(self, message):
     """Logs error message to stderr and _git_event_log."""
-    self._LogMessage(f'repo superproject error: {message}')
+    self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
 
   def _LogWarning(self, message):
     """Logs warning message to stderr and _git_event_log."""
-    self._LogMessage(f'repo superproject warning: {message}')
+    self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
 
   def _Init(self):
     """Sets up a local Git repository to get a copy of a superproject.
@@ -162,11 +159,8 @@ class Superproject(object):
       return False
     return True
 
-  def _Fetch(self, url):
-    """Fetches a local copy of a superproject for the manifest based on url.
-
-    Args:
-      url: superproject's url.
+  def _Fetch(self):
+    """Fetches a local copy of a superproject for the manifest based on |_remote_url|.
 
     Returns:
       True if fetch is successful, or False.
@@ -177,7 +171,8 @@ class Superproject(object):
     if not git_require((2, 28, 0)):
       self._LogWarning('superproject requires a git version 2.28 or later')
       return False
-    cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none']
+    cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
+           '--filter', 'blob:none']
     if self._branch:
       cmd += [self._branch + ':' + self._branch]
     p = GitCommand(None,
@@ -234,15 +229,14 @@ class Superproject(object):
     print('NOTICE: --use-superproject is in beta; report any issues to the '
           'address described in `repo version`', file=sys.stderr)
     should_exit = True
-    url = self._manifest.superproject['remote'].url
-    if not url:
+    if not self._remote_url:
       self._LogWarning(f'superproject URL is not defined in manifest: '
                        f'{self._manifest.manifestFile}')
       return SyncResult(False, should_exit)
 
     if not self._Init():
       return SyncResult(False, should_exit)
-    if not self._Fetch(url):
+    if not self._Fetch():
       return SyncResult(False, should_exit)
     if not self._quiet:
       print('%s: Initial setup for superproject completed.' % self._work_git)
@@ -260,7 +254,7 @@ class Superproject(object):
 
     data = self._LsTree()
     if not data:
-      self._LogWarning(f'warning: git ls-tree failed to return data for manifest: '
+      self._LogWarning(f'git ls-tree failed to return data for manifest: '
                        f'{self._manifest.manifestFile}')
       return CommitIdsResult(None, True)
 
@@ -317,7 +311,7 @@ class Superproject(object):
     if project.revisionId:
       return True
     # Skip the project if it comes from the local manifest.
-    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+    return project.manifest.IsFromLocalManifest(project)
 
   def UpdateProjectsRevisionId(self, projects):
     """Update revisionId of every project in projects with the commit id.
@@ -366,57 +360,36 @@ def _UseSuperprojectFromConfiguration():
   user_value = user_cfg.GetBoolean('repo.superprojectChoice')
   if user_value is not None:
     user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
-    if user_expiration is not None and (user_expiration <= 0 or user_expiration >= time_now):
+    if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now:
       # TODO(b/190688390) - Remove prompt when we are comfortable with the new
       # default value.
-      print(('You are currently enrolled in Git submodules experiment '
-             '(go/android-submodules-quickstart).  Use --no-use-superproject '
-             'to override.\n'), file=sys.stderr)
-    return user_value
+      if user_value:
+        print(('You are currently enrolled in Git submodules experiment '
+               '(go/android-submodules-quickstart).  Use --no-use-superproject '
+               'to override.\n'), file=sys.stderr)
+      else:
+        print(('You are not currently enrolled in Git submodules experiment '
+               '(go/android-submodules-quickstart).  Use --use-superproject '
+               'to override.\n'), file=sys.stderr)
+      return user_value
 
   # We don't have an unexpired choice, ask for one.
   system_cfg = RepoConfig.ForSystem()
   system_value = system_cfg.GetBoolean('repo.superprojectChoice')
   if system_value:
     # The system configuration is proposing that we should enable the
-    # use of superproject. Present this to user for confirmation if we
-    # are on a TTY, or, when we are not on a TTY, accept the system
-    # default for this time only.
+    # use of superproject. Treat the user as enrolled for two weeks.
     #
     # TODO(b/190688390) - Remove prompt when we are comfortable with the new
     # default value.
-    prompt = ('Repo can now use Git submodules (go/android-submodules-quickstart) '
-              'instead of manifests to represent the state of the Android '
-              'superproject, which results in faster syncs and better atomicity.\n\n')
-    if sys.stdout.isatty():
-      prompt += 'Would you like to opt in for two weeks (y/N)? '
-      response = input(prompt).lower()
-      time_choiceexpire = time_now + (86400 * 14)
-      if response in ('y', 'yes'):
-        userchoice = True
-      elif response in ('a', 'always'):
-        userchoice = True
-        time_choiceexpire = 0
-      elif response == 'never':
-        userchoice = False
-        time_choiceexpire = 0
-      elif response in ('n', 'no'):
-        userchoice = False
-      else:
-        # Unrecognized user response, assume the intention was no, but
-        # only for 2 hours instead of 2 weeks to balance between not
-        # being overly pushy while still retain the opportunity to
-        # enroll.
-        userchoice = False
-        time_choiceexpire = time_now + 7200
-
-      user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
-      user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
-
-      return userchoice
-    else:
-      print('Accepting once since we are not on a TTY', file=sys.stderr)
-      return True
+    userchoice = True
+    time_choiceexpire = time_now + (86400 * 14)
+    user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
+    user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
+    print('You are automatically enrolled in Git submodules experiment '
+          '(go/android-submodules-quickstart) for another two weeks.\n',
+          file=sys.stderr)
+    return True
 
   # For all other cases, we would not use superproject by default.
   return False
@@ -437,4 +410,6 @@ def UseSuperproject(opt, manifest):
     if client_value is not None:
       return client_value
     else:
+      if not manifest.superproject:
+        return False
       return _UseSuperprojectFromConfiguration()

git_trace2_event_log.py

@@ -167,6 +167,26 @@ class EventLog(object):
     repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
     self.LogConfigEvents(repo_config, 'def_param')
 
+  def GetDataEventName(self, value):
+    """Returns 'data-json' if the value is an array else returns 'data'."""
+    return 'data-json' if value[0] == '[' and value[-1] == ']' else 'data'
+
+  def LogDataConfigEvents(self, config, prefix):
+    """Append a 'data' event for each config key/value in |config| to the current log.
+
+    For each keyX and valueX of the config, "key" field of the event is '|prefix|/keyX'
+    and the "value" of the "key" field is valueX.
+
+    Args:
+      config: Configuration dictionary.
+      prefix: Prefix for each key that is logged.
+    """
+    for key, value in config.items():
+      event = self._CreateEventDict(self.GetDataEventName(value))
+      event['key'] = f'{prefix}/{key}'
+      event['value'] = value
+      self._log.append(event)
+
   def ErrorEvent(self, msg, fmt):
     """Append a 'error' event to the current log."""
     error_event = self._CreateEventDict('error')

main.py

@@ -127,6 +127,8 @@ global_options.add_option('--event-log',
                           help='filename of event log to append timeline to')
 global_options.add_option('--git-trace2-event-log', action='store',
                           help='directory to write git trace2 event log to')
+global_options.add_option('--submanifest-path', action='store',
+                          metavar='REL_PATH', help='submanifest path')
 
 
 class _Repo(object):
@@ -217,7 +219,12 @@ class _Repo(object):
     SetDefaultColoring(gopts.color)
 
     git_trace2_event_log = EventLog()
-    repo_client = RepoClient(self.repodir)
+    outer_client = RepoClient(self.repodir)
+    repo_client = outer_client
+    if gopts.submanifest_path:
+      repo_client = RepoClient(self.repodir,
+                               submanifest_path=gopts.submanifest_path,
+                               outer_client=outer_client)
     gitc_manifest = None
     gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
     if gitc_client_name:
@@ -229,6 +236,8 @@ class _Repo(object):
           repodir=self.repodir,
           client=repo_client,
           manifest=repo_client.manifest,
+          outer_client=outer_client,
+          outer_manifest=outer_client.manifest,
           gitc_manifest=gitc_manifest,
           git_event_log=git_trace2_event_log)
     except KeyError:
@@ -283,7 +292,37 @@ class _Repo(object):
     try:
       cmd.CommonValidateOptions(copts, cargs)
       cmd.ValidateOptions(copts, cargs)
-      result = cmd.Execute(copts, cargs)
+
+      this_manifest_only = copts.this_manifest_only
+      # If not specified, default to using the outer manifest.
+      outer_manifest = copts.outer_manifest is not False
+      if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
+        result = cmd.Execute(copts, cargs)
+      elif outer_manifest and repo_client.manifest.is_submanifest:
+        # The command does not support multi-manifest, we are using a
+        # submanifest, and the command line is for the outermost manifest.
+        # Re-run using the outermost manifest, which will recurse through the
+        # submanifests.
+        gopts.submanifest_path = ''
+        result = self._Run(name, gopts, argv)
+      else:
+        # No multi-manifest support. Run the command in the current
+        # (sub)manifest, and then any child submanifests.
+        result = cmd.Execute(copts, cargs)
+        for submanifest in repo_client.manifest.submanifests.values():
+          spec = submanifest.ToSubmanifestSpec(root=repo_client.outer_client)
+          gopts.submanifest_path = submanifest.repo_client.path_prefix
+          child_argv = argv[:]
+          child_argv.append('--no-outer-manifest')
+          # Not all subcommands support the 3 manifest options, so only add them
+          # if the original command includes them.
+          if hasattr(copts, 'manifest_url'):
+            child_argv.extend(['--manifest-url', spec.manifestUrl])
+          if hasattr(copts, 'manifest_name'):
+            child_argv.extend(['--manifest-name', spec.manifestName])
+          if hasattr(copts, 'manifest_branch'):
+            child_argv.extend(['--manifest-branch', spec.revision])
+          result = self._Run(name, gopts, child_argv) or result
     except (DownloadError, ManifestInvalidRevisionError,
             NoManifestException) as e:
       print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -41,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
@@ -92,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -41,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
@@ -92,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -137,6 +142,12 @@ equivalent to using \fB\-b\fR HEAD.
 The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
 used. If no manifest is specified, the manifest default.xml will be used.
 .PP
+If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded
+directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting
+up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be
+fully static and will not be re\-downloaded during subsesquent `repo init` and
+`repo sync` calls.
+.PP
 The \fB\-\-reference\fR option can be used to point to a directory that has the content
 of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as
 possible from the local reference directory when fetching from the server. This

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -161,6 +161,7 @@ CDATA #IMPLIED>
 <!ELEMENT extend\-project EMPTY>
 <!ATTLIST extend\-project name CDATA #REQUIRED>
 <!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
@@ -174,8 +175,9 @@ CDATA #IMPLIED>
 <!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
 .IP
 <!ELEMENT superproject EMPTY>
-<!ATTLIST superproject name    CDATA #REQUIRED>
-<!ATTLIST superproject remote  IDREF #IMPLIED>
+<!ATTLIST superproject name     CDATA #REQUIRED>
+<!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -385,6 +387,11 @@ original manifest.
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
 Attribute `groups`: List of additional groups to which this project belongs.
 Same syntax as the corresponding element of `project`.
 .PP
@@ -477,6 +484,10 @@ project](#element\-project) for more information.
 Attribute `remote`: Name of a previously defined remote element. If not supplied
 the remote given by the default element is used.
 .PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
 Element contactinfo
 .PP
 *** *Note*: This is currently a WIP. ***

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2021" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -43,7 +43,7 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
-.SS "The complete list of recognized repo commands are:"
+.SS "The complete list of recognized repo commands is:"
 .TP
 abandon
 Permanently abandon a development branch

manifest_xml.py

@@ -33,6 +33,9 @@ from wrapper import Wrapper
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
+SUBMANIFEST_DIR = 'submanifests'
+# Limit submanifests to an arbitrary depth for loop detection.
+MAX_SUBMANIFEST_DEPTH = 8
 
 # Add all projects from local manifest into a group.
 LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
@@ -197,10 +200,122 @@ class _XmlRemote(object):
     self.annotations.append(Annotation(name, value, keep))
 
 
+class _XmlSubmanifest:
+  """Manage the <submanifest> element specified in the manifest.
+
+  Attributes:
+    name: a string, the name for this submanifest.
+    remote: a string, the remote.name for this submanifest.
+    project: a string, the name of the manifest project.
+    revision: a string, the commitish.
+    manifestName: a string, the submanifest file name.
+    groups: a list of strings, the groups to add to all projects in the submanifest.
+    path: a string, the relative path for the submanifest checkout.
+    annotations: (derived) a list of annotations.
+    present: (derived) a boolean, whether the submanifest's manifest file is present.
+  """
+  def __init__(self,
+               name,
+               remote=None,
+               project=None,
+               revision=None,
+               manifestName=None,
+               groups=None,
+               path=None,
+               parent=None):
+    self.name = name
+    self.remote = remote
+    self.project = project
+    self.revision = revision
+    self.manifestName = manifestName
+    self.groups = groups
+    self.path = path
+    self.annotations = []
+    outer_client = parent._outer_client or parent
+    if self.remote and not self.project:
+      raise ManifestParseError(
+          f'Submanifest {name}: must specify project when remote is given.')
+    rc = self.repo_client = RepoClient(
+        parent.repodir, manifestName, parent_groups=','.join(groups) or '',
+        submanifest_path=self.relpath, outer_client=outer_client)
+
+    self.present = os.path.exists(os.path.join(self.repo_client.subdir,
+                                               MANIFEST_FILE_NAME))
+
+  def __eq__(self, other):
+    if not isinstance(other, _XmlSubmanifest):
+      return False
+    return (
+        self.name == other.name and
+        self.remote == other.remote and
+        self.project == other.project and
+        self.revision == other.revision and
+        self.manifestName == other.manifestName and
+        self.groups == other.groups and
+        self.path == other.path and
+        sorted(self.annotations) == sorted(other.annotations))
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+  def ToSubmanifestSpec(self, root):
+    """Return a SubmanifestSpec object, populating attributes"""
+    mp = root.manifestProject
+    remote = root.remotes[self.remote or root.default.remote.name]
+    # If a project was given, generate the url from the remote and project.
+    # If not, use this manifestProject's url.
+    if self.project:
+      manifestUrl = remote.ToRemoteSpec(self.project).url
+    else:
+      manifestUrl = mp.GetRemote(mp.remote.name).url
+    manifestName = self.manifestName or 'default.xml'
+    revision = self.revision or self.name
+    path = self.path or revision.split('/')[-1]
+    groups = self.groups or []
+
+    return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path,
+                           groups)
+
+  @property
+  def relpath(self):
+    """The path of this submanifest relative to the parent manifest."""
+    revision = self.revision or self.name
+    return self.path or revision.split('/')[-1]
+
+  def GetGroupsStr(self):
+    """Returns the `groups` given for this submanifest."""
+    if self.groups:
+      return ','.join(self.groups)
+    return ''
+
+  def AddAnnotation(self, name, value, keep):
+    """Add annotations to the submanifest."""
+    self.annotations.append(Annotation(name, value, keep))
+
+
+class SubmanifestSpec:
+  """The submanifest element, with all fields expanded."""
+
+  def __init__(self,
+               name,
+               manifestUrl,
+               manifestName,
+               revision,
+               path,
+               groups):
+    self.name = name
+    self.manifestUrl = manifestUrl
+    self.manifestName = manifestName
+    self.revision = revision
+    self.path = path
+    self.groups = groups or []
+
+
 class XmlManifest(object):
   """manages the repo configuration file"""
 
-  def __init__(self, repodir, manifest_file, local_manifests=None):
+  def __init__(self, repodir, manifest_file, local_manifests=None,
+               outer_client=None, parent_groups='', submanifest_path=''):
     """Initialize.
 
     Args:
@@ -210,23 +325,37 @@ class XmlManifest(object):
           be |repodir|/|MANIFEST_FILE_NAME|.
       local_manifests: Full path to the directory of local override manifests.
           This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
+      outer_client: RepoClient of the outertree.
+      parent_groups: a string, the groups to apply to this projects.
+      submanifest_path: The submanifest root relative to the repo root.
     """
     # TODO(vapier): Move this out of this class.
     self.globalConfig = GitConfig.ForUser()
 
     self.repodir = os.path.abspath(repodir)
-    self.topdir = os.path.dirname(self.repodir)
+    self._CheckLocalPath(submanifest_path)
+    self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path)
     self.manifestFile = manifest_file
     self.local_manifests = local_manifests
     self._load_local_manifests = True
+    self.parent_groups = parent_groups
+
+    if outer_client and self.isGitcClient:
+      raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`')
+
+    if submanifest_path and not outer_client:
+      # If passing a submanifest_path, there must be an outer_client.
+      raise ManifestParseError(f'Bad call to {self.__class__.__name__}')
+
+    # If self._outer_client is None, this is not a checkout that supports
+    # multi-tree.
+    self._outer_client = outer_client or self
 
     self.repoProject = MetaProject(self, 'repo',
                                    gitdir=os.path.join(repodir, 'repo/.git'),
                                    worktree=os.path.join(repodir, 'repo'))
 
-    mp = MetaProject(self, 'manifests',
-                     gitdir=os.path.join(repodir, 'manifests.git'),
-                     worktree=os.path.join(repodir, 'manifests'))
+    mp = self.SubmanifestProject(self.path_prefix)
     self.manifestProject = mp
 
     # This is a bit hacky, but we're in a chicken & egg situation: all the
@@ -270,8 +399,7 @@ class XmlManifest(object):
     self.Override(name)
 
     # Old versions of repo would generate symlinks we need to clean up.
-    if os.path.lexists(self.manifestFile):
-      platform_utils.remove(self.manifestFile)
+    platform_utils.remove(self.manifestFile, missing_ok=True)
     # This file is interpreted as if it existed inside the manifest repo.
     # That allows us to use <include> with the relative file name.
     with open(self.manifestFile, 'w') as fp:
@@ -312,6 +440,31 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         ae.setAttribute('value', a.value)
         e.appendChild(ae)
 
+  def _SubmanifestToXml(self, r, doc, root):
+    """Generate XML <submanifest/> node."""
+    e = doc.createElement('submanifest')
+    root.appendChild(e)
+    e.setAttribute('name', r.name)
+    if r.remote is not None:
+      e.setAttribute('remote', r.remote)
+    if r.project is not None:
+      e.setAttribute('project', r.project)
+    if r.manifestName is not None:
+      e.setAttribute('manifest-name', r.manifestName)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
+    if r.path is not None:
+      e.setAttribute('path', r.path)
+    if r.groups:
+      e.setAttribute('groups', r.GetGroupsStr())
+
+    for a in r.annotations:
+      if a.keep == 'true':
+        ae = doc.createElement('annotation')
+        ae.setAttribute('name', a.name)
+        ae.setAttribute('value', a.value)
+        e.appendChild(ae)
+
   def _ParseList(self, field):
     """Parse fields that contain flattened lists.
 
@@ -330,6 +483,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
+    if self.is_submanifest:
+      root.setAttribute('path', self.path_prefix)
     doc.appendChild(root)
 
     # Save out the notice.  There's a little bit of work here to give it the
@@ -384,6 +539,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
+    for r in sorted(self.submanifests):
+      self._SubmanifestToXml(self.submanifests[r], doc, root)
+    if self.submanifests:
+      root.appendChild(doc.createTextNode(''))
+
     def output_projects(parent, parent_node, projects):
       for project_name in projects:
         for project in self._projects[project_name]:
@@ -507,6 +667,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       if not d.remote or remote.orig_name != remoteName:
         remoteName = remote.orig_name
         e.setAttribute('remote', remoteName)
+      revision = remote.revision or d.revisionExpr
+      if not revision or revision != self._superproject['revision']:
+        e.setAttribute('revision', self._superproject['revision'])
       root.appendChild(e)
 
     if self._contactinfo.bugurl != Wrapper().BUG_URL:
@@ -535,6 +698,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         'project',
         'extend-project',
         'include',
+        'submanifest',
         # These are children of 'project' nodes.
         'annotation',
         'project',
@@ -572,13 +736,75 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def _output_manifest_project_extras(self, p, e):
     """Manifests can modify e if they support extra project attributes."""
 
+  @property
+  def is_multimanifest(self):
+    """Whether this is a multimanifest checkout"""
+    return bool(self.outer_client.submanifests)
+
+  @property
+  def is_submanifest(self):
+    """Whether this manifest is a submanifest"""
+    return self._outer_client and self._outer_client != self
+
+  @property
+  def outer_client(self):
+    """The instance of the outermost manifest client"""
+    self._Load()
+    return self._outer_client
+
+  @property
+  def all_manifests(self):
+    """Generator yielding all (sub)manifests."""
+    self._Load()
+    outer = self._outer_client
+    yield outer
+    for tree in outer.all_children:
+      yield tree
+
+  @property
+  def all_children(self):
+    """Generator yielding all child submanifests."""
+    self._Load()
+    for child in self._submanifests.values():
+      if child.repo_client:
+        yield child.repo_client
+        for tree in child.repo_client.all_children:
+          yield tree
+
+  @property
+  def path_prefix(self):
+    """The path of this submanifest, relative to the outermost manifest."""
+    if not self._outer_client or self == self._outer_client:
+      return ''
+    return os.path.relpath(self.topdir, self._outer_client.topdir)
+
+  @property
+  def all_paths(self):
+    """All project paths for all (sub)manifests.  See `paths`."""
+    ret = {}
+    for tree in self.all_manifests:
+      prefix = tree.path_prefix
+      ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()})
+    return ret
+
+  @property
+  def all_projects(self):
+    """All projects for all (sub)manifests.  See `projects`."""
+    return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests))
+
   @property
   def paths(self):
+    """Return all paths for this manifest.
+
+    Return:
+      A dictionary of {path: Project()}.  `path` is relative to this manifest.
+    """
     self._Load()
     return self._paths
 
   @property
   def projects(self):
+    """Return a list of all Projects in this manifest."""
     self._Load()
     return list(self._paths.values())
 
@@ -592,6 +818,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self._Load()
     return self._default
 
+  @property
+  def submanifests(self):
+    """All submanifests in this manifest."""
+    self._Load()
+    return self._submanifests
+
   @property
   def repo_hooks_project(self):
     self._Load()
@@ -648,6 +880,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def HasLocalManifests(self):
     return self._load_local_manifests and self.local_manifests
 
+  def IsFromLocalManifest(self, project):
+    """Is the project from a local manifest?"""
+    return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX)
+               for x in project.groups)
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -664,6 +901,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def HasSubmodules(self):
     return self.manifestProject.config.GetBoolean('repo.submodules')
 
+  @property
+  def EnableGitLfs(self):
+    return self.manifestProject.config.GetBoolean('repo.git-lfs')
+
+  def FindManifestByPath(self, path):
+    """Returns the manifest containing path."""
+    path = os.path.abspath(path)
+    manifest = self._outer_client or self
+    old = None
+    while manifest._submanifests and manifest != old:
+      old = manifest
+      for name in manifest._submanifests:
+        tree = manifest._submanifests[name]
+        if path.startswith(tree.repo_client.manifest.topdir):
+          manifest = tree.repo_client
+          break
+    return manifest
+
+  @property
+  def subdir(self):
+    """Returns the path for per-submanifest objects for this manifest."""
+    return self.SubmanifestInfoDir(self.path_prefix)
+
+  def SubmanifestInfoDir(self, submanifest_path, object_path=''):
+    """Return the path to submanifest-specific info for a submanifest.
+
+    Return the full path of the directory in which to put per-manifest objects.
+
+    Args:
+      submanifest_path: a string, the path of the submanifest, relative to the
+                        outermost topdir.  If empty, then repodir is returned.
+      object_path: a string, relative path to append to the submanifest info
+                   directory path.
+    """
+    if submanifest_path:
+      return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path,
+                          object_path)
+    else:
+      return os.path.join(self.repodir, object_path)
+
+  def SubmanifestProject(self, submanifest_path):
+    """Return a manifestProject for a submanifest."""
+    subdir = self.SubmanifestInfoDir(submanifest_path)
+    mp = MetaProject(self, 'manifests',
+                     gitdir=os.path.join(subdir, 'manifests.git'),
+                     worktree=os.path.join(subdir, 'manifests'))
+    return mp
+
   def GetDefaultGroupsStr(self):
     """Returns the default group string for the platform."""
     return 'default,platform-' + platform.system().lower()
@@ -681,6 +966,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self._paths = {}
     self._remotes = {}
     self._default = None
+    self._submanifests = {}
     self._repo_hooks_project = None
     self._superproject = {}
     self._contactinfo = ContactInfo(Wrapper().BUG_URL)
@@ -688,20 +974,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self.branch = None
     self._manifest_server = None
 
-  def _Load(self):
+  def _Load(self, initial_client=None, submanifest_depth=0):
+    if submanifest_depth > MAX_SUBMANIFEST_DEPTH:
+      raise ManifestParseError('maximum submanifest depth %d exceeded.' %
+                               MAX_SUBMANIFEST_DEPTH)
     if not self._loaded:
+      if self._outer_client and self._outer_client != self:
+        # This will load all clients.
+        self._outer_client._Load(initial_client=self)
+
       m = self.manifestProject
       b = m.GetBranch(m.CurrentBranch).merge
       if b is not None and b.startswith(R_HEADS):
         b = b[len(R_HEADS):]
       self.branch = b
 
+      parent_groups = self.parent_groups
+
       # The manifestFile was specified by the user which is why we allow include
       # paths to point anywhere.
       nodes = []
       nodes.append(self._ParseManifestXml(
           self.manifestFile, self.manifestProject.worktree,
-          restrict_includes=False))
+          parent_groups=parent_groups, restrict_includes=False))
 
       if self._load_local_manifests and self.local_manifests:
         try:
@@ -710,9 +1005,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
               local = os.path.join(self.local_manifests, local_file)
               # Since local manifests are entirely managed by the user, allow
               # them to point anywhere the user wants.
+              local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}'
               nodes.append(self._ParseManifestXml(
-                  local, self.repodir,
-                  parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
+                  local, self.subdir,
+                  parent_groups=f'{local_group},{parent_groups}',
                   restrict_includes=False))
         except OSError:
           pass
@@ -731,6 +1027,23 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
       self._loaded = True
 
+      # Now that we have loaded this manifest, load any submanifest  manifests
+      # as well.  We need to do this after self._loaded is set to avoid looping.
+      if self._outer_client:
+        for name in self._submanifests:
+          tree = self._submanifests[name]
+          spec = tree.ToSubmanifestSpec(self)
+          present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME))
+          if present and tree.present and not tree.repo_client:
+            if initial_client and initial_client.topdir == self.topdir:
+              tree.repo_client = self
+              tree.present = present
+            elif not os.path.exists(self.subdir):
+              tree.present = False
+          if tree.present:
+            tree.repo_client._Load(initial_client=initial_client,
+                                   submanifest_depth=submanifest_depth + 1)
+
   def _ParseManifestXml(self, path, include_root, parent_groups='',
                         restrict_includes=True):
     """Parse a manifest XML and return the computed nodes.
@@ -820,6 +1133,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if self._default is None:
       self._default = _Default()
 
+    submanifest_paths = set()
+    for node in itertools.chain(*node_list):
+      if node.nodeName == 'submanifest':
+        submanifest = self._ParseSubmanifest(node)
+        if submanifest:
+          if submanifest.name in self._submanifests:
+            if submanifest != self._submanifests[submanifest.name]:
+              raise ManifestParseError(
+                  'submanifest %s already exists with different attributes' %
+                  (submanifest.name))
+          else:
+            self._submanifests[submanifest.name] = submanifest
+            submanifest_paths.add(submanifest.relpath)
+
     for node in itertools.chain(*node_list):
       if node.nodeName == 'notice':
         if self._notice is not None:
@@ -847,11 +1174,18 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         raise ManifestParseError(
             'duplicate path %s in %s' %
             (project.relpath, self.manifestFile))
+      for tree in submanifest_paths:
+        if project.relpath.startswith(tree):
+          raise ManifestParseError(
+              'project %s conflicts with submanifest path %s' %
+              (project.relpath, tree))
       self._paths[project.relpath] = project
       projects.append(project)
       for subproject in project.subprojects:
         recursively_add_projects(subproject)
 
+    repo_hooks_project = None
+    enabled_repo_hooks = None
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
@@ -864,54 +1198,49 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                                    'project: %s' % name)
 
         path = node.getAttribute('path')
+        dest_path = node.getAttribute('dest-path')
         groups = node.getAttribute('groups')
         if groups:
           groups = self._ParseList(groups)
         revision = node.getAttribute('revision')
-        remote = node.getAttribute('remote')
-        if remote:
+        remote_name = node.getAttribute('remote')
+        if not remote_name:
+          remote = self._default.remote
+        else:
           remote = self._get_remote(node)
 
+        named_projects = self._projects[name]
+        if dest_path and not path and len(named_projects) > 1:
+          raise ManifestParseError('extend-project cannot use dest-path when '
+                                   'matching multiple projects: %s' % name)
         for p in self._projects[name]:
           if path and p.relpath != path:
             continue
           if groups:
             p.groups.extend(groups)
           if revision:
-            p.revisionExpr = revision
-            if IsId(revision):
-              p.revisionId = revision
-            else:
-              p.revisionId = None
-          if remote:
-            p.remote = remote.ToRemoteSpec(name)
-      if node.nodeName == 'repo-hooks':
-        # Get the name of the project and the (space-separated) list of enabled.
-        repo_hooks_project = self._reqatt(node, 'in-project')
-        enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
+            p.SetRevision(revision)
 
+          if remote_name:
+            p.remote = remote.ToRemoteSpec(name)
+
+          if dest_path:
+            del self._paths[p.relpath]
+            relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(
+                name, dest_path, remote.name)
+            p.UpdatePaths(relpath, worktree, gitdir, objdir)
+            self._paths[p.relpath] = p
+
+      if node.nodeName == 'repo-hooks':
         # Only one project can be the hooks project
-        if self._repo_hooks_project is not None:
+        if repo_hooks_project is not None:
           raise ManifestParseError(
               'duplicate repo-hooks in %s' %
               (self.manifestFile))
 
-        # Store a reference to the Project.
-        try:
-          repo_hooks_projects = self._projects[repo_hooks_project]
-        except KeyError:
-          raise ManifestParseError(
-              'project %s not found for repo-hooks' %
-              (repo_hooks_project))
-
-        if len(repo_hooks_projects) != 1:
-          raise ManifestParseError(
-              'internal error parsing repo-hooks in %s' %
-              (self.manifestFile))
-        self._repo_hooks_project = repo_hooks_projects[0]
-
-        # Store the enabled hooks in the Project object.
-        self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+        # Get the name of the project and the (space-separated) list of enabled.
+        repo_hooks_project = self._reqatt(node, 'in-project')
+        enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
       if node.nodeName == 'superproject':
         name = self._reqatt(node, 'name')
         # There can only be one superproject.
@@ -929,6 +1258,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           raise ManifestParseError("no remote for superproject %s within %s" %
                                    (name, self.manifestFile))
         self._superproject['remote'] = remote.ToRemoteSpec(name)
+        revision = node.getAttribute('revision') or remote.revision
+        if not revision:
+          revision = self._default.revisionExpr
+        if not revision:
+          raise ManifestParseError('no revision for superproject %s within %s' %
+                                   (name, self.manifestFile))
+        self._superproject['revision'] = revision
       if node.nodeName == 'contactinfo':
         bugurl = self._reqatt(node, 'bugurl')
         # This element can be repeated, later entries will clobber earlier ones.
@@ -944,12 +1280,30 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
           # If the manifest removes the hooks project, treat it as if it deleted
           # the repo-hooks element too.
-          if self._repo_hooks_project and (self._repo_hooks_project.name == name):
-            self._repo_hooks_project = None
+          if repo_hooks_project == name:
+            repo_hooks_project = None
         elif not XmlBool(node, 'optional', False):
           raise ManifestParseError('remove-project element specifies non-existent '
                                    'project: %s' % name)
 
+    # Store repo hooks project information.
+    if repo_hooks_project:
+      # Store a reference to the Project.
+      try:
+        repo_hooks_projects = self._projects[repo_hooks_project]
+      except KeyError:
+        raise ManifestParseError(
+            'project %s not found for repo-hooks' %
+            (repo_hooks_project))
+
+      if len(repo_hooks_projects) != 1:
+        raise ManifestParseError(
+            'internal error parsing repo-hooks in %s' %
+            (self.manifestFile))
+      self._repo_hooks_project = repo_hooks_projects[0]
+      # Store the enabled hooks in the Project object.
+      self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -1078,6 +1432,53 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     return '\n'.join(cleanLines)
 
+  def _ParseSubmanifest(self, node):
+    """Reads a <submanifest> element from the manifest file."""
+    name = self._reqatt(node, 'name')
+    remote = node.getAttribute('remote')
+    if remote == '':
+      remote = None
+    project = node.getAttribute('project')
+    if project == '':
+      project = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
+    manifestName = node.getAttribute('manifest-name')
+    if manifestName == '':
+      manifestName = None
+    groups = ''
+    if node.hasAttribute('groups'):
+      groups = node.getAttribute('groups')
+    groups = self._ParseList(groups)
+    path = node.getAttribute('path')
+    if path == '':
+      path = None
+      if revision:
+        msg = self._CheckLocalPath(revision.split('/')[-1])
+        if msg:
+          raise ManifestInvalidPathError(
+              '<submanifest> invalid "revision": %s: %s' % (revision, msg))
+      else:
+        msg = self._CheckLocalPath(name)
+        if msg:
+          raise ManifestInvalidPathError(
+              '<submanifest> invalid "name": %s: %s' % (name, msg))
+    else:
+      msg = self._CheckLocalPath(path)
+      if msg:
+        raise ManifestInvalidPathError(
+            '<submanifest> invalid "path": %s: %s' % (path, msg))
+
+    submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName,
+                                  groups, path, self)
+
+    for n in node.childNodes:
+      if n.nodeName == 'annotation':
+        self._ParseAnnotation(submanifest, n)
+
+    return submanifest
+
   def _JoinName(self, parent_name, name):
     return os.path.join(parent_name, name)
 
@@ -1141,7 +1542,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     if parent is None:
       relpath, worktree, gitdir, objdir, use_git_worktrees = \
-          self.GetProjectPaths(name, path)
+          self.GetProjectPaths(name, path, remote.name)
     else:
       use_git_worktrees = False
       relpath, worktree, gitdir, objdir = \
@@ -1187,31 +1588,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     return project
 
-  def GetProjectPaths(self, name, path):
+  def GetProjectPaths(self, name, path, remote):
+    """Return the paths for a project.
+
+    Args:
+      name: a string, the name of the project.
+      path: a string, the path of the project.
+      remote: a string, the remote.name of the project.
+    """
     # The manifest entries might have trailing slashes.  Normalize them to avoid
     # unexpected filesystem behavior since we do string concatenation below.
     path = path.rstrip('/')
     name = name.rstrip('/')
+    remote = remote.rstrip('/')
     use_git_worktrees = False
+    use_remote_name = bool(self._outer_client._submanifests)
     relpath = path
     if self.IsMirror:
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
       objdir = gitdir
     else:
+      if use_remote_name:
+        namepath = os.path.join(remote, f'{name}.git')
+      else:
+        namepath = f'{name}.git'
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
-      gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
+      gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path)
       # We allow people to mix git worktrees & non-git worktrees for now.
       # This allows for in situ migration of repo clients.
       if os.path.exists(gitdir) or not self.UseGitWorktrees:
-        objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+        objdir = os.path.join(self.subdir, 'project-objects', namepath)
       else:
         use_git_worktrees = True
-        gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
+        gitdir = os.path.join(self.repodir, 'worktrees', namepath)
         objdir = gitdir
     return relpath, worktree, gitdir, objdir, use_git_worktrees
 
-  def GetProjectsWithName(self, name):
+  def GetProjectsWithName(self, name, all_manifests=False):
+    """All projects with |name|.
+
+    Args:
+      name: a string, the name of the project.
+      all_manifests: a boolean, if True, then all manifests are searched. If
+                     False, then only this manifest is searched.
+    """
+    if all_manifests:
+      return list(itertools.chain.from_iterable(
+          x._projects.get(name, []) for x in self.all_manifests))
     return self._projects.get(name, [])
 
   def GetSubprojectName(self, parent, submodule_path):
@@ -1467,19 +1891,26 @@ class GitcManifest(XmlManifest):
 class RepoClient(XmlManifest):
   """Manages a repo client checkout."""
 
-  def __init__(self, repodir, manifest_file=None):
+  def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs):
     self.isGitcClient = False
+    submanifest_path = submanifest_path or ''
+    if submanifest_path:
+      self._CheckLocalPath(submanifest_path)
+      prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path)
+    else:
+      prefix = repodir
 
-    if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)):
+    if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)):
       print('error: %s is not supported; put local manifests in `%s` instead' %
-            (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)),
+            (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)),
             file=sys.stderr)
       sys.exit(1)
 
     if manifest_file is None:
-      manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME)
-    local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME))
-    super().__init__(repodir, manifest_file, local_manifests)
+        manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME)
+    local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME))
+    super().__init__(repodir, manifest_file, local_manifests,
+                     submanifest_path=submanifest_path, **kwargs)
 
     # TODO: Completely separate manifest logic out of the client.
     self.manifest = self

platform_utils.py

@@ -124,31 +124,30 @@ def rename(src, dst):
       else:
         raise
   else:
-    os.rename(src, dst)
+    shutil.move(src, dst)
 
 
-def remove(path):
+def remove(path, missing_ok=False):
   """Remove (delete) the file path. This is a replacement for os.remove that
   allows deleting read-only files on Windows, with support for long paths and
   for deleting directory symbolic links.
 
   Availability: Unix, Windows."""
-  if isWindows():
-    longpath = _makelongpath(path)
-    try:
-      os.remove(longpath)
-    except OSError as e:
-      if e.errno == errno.EACCES:
-        os.chmod(longpath, stat.S_IWRITE)
-        # Directory symbolic links must be deleted with 'rmdir'.
-        if islink(longpath) and isdir(longpath):
-          os.rmdir(longpath)
-        else:
-          os.remove(longpath)
+  longpath = _makelongpath(path) if isWindows() else path
+  try:
+    os.remove(longpath)
+  except OSError as e:
+    if e.errno == errno.EACCES:
+      os.chmod(longpath, stat.S_IWRITE)
+      # Directory symbolic links must be deleted with 'rmdir'.
+      if islink(longpath) and isdir(longpath):
+        os.rmdir(longpath)
       else:
-        raise
-  else:
-    os.remove(path)
+        os.remove(longpath)
+    elif missing_ok and e.errno == errno.ENOENT:
+      pass
+    else:
+      raise
 
 
 def walk(top, topdown=True, onerror=None, followlinks=False):

project.py

@@ -457,11 +457,7 @@ class RemoteSpec(object):
 
 class Project(object):
   # These objects can be shared between several working trees.
-  shareable_files = ['description', 'info']
-  shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
-  # These objects can only be used by a single working tree.
-  working_tree_files = ['config', 'packed-refs', 'shallow']
-  working_tree_dirs = ['logs', 'refs']
+  shareable_dirs = ['hooks', 'objects', 'rr-cache']
 
   def __init__(self,
                manifest,
@@ -519,21 +515,8 @@ class Project(object):
     self.client = self.manifest = manifest
     self.name = name
     self.remote = remote
-    self.gitdir = gitdir.replace('\\', '/')
-    self.objdir = objdir.replace('\\', '/')
-    if worktree:
-      self.worktree = os.path.normpath(worktree).replace('\\', '/')
-    else:
-      self.worktree = None
-    self.relpath = relpath
-    self.revisionExpr = revisionExpr
-
-    if revisionId is None \
-            and revisionExpr \
-            and IsId(revisionExpr):
-      self.revisionId = revisionExpr
-    else:
-      self.revisionId = revisionId
+    self.UpdatePaths(relpath, worktree, gitdir, objdir)
+    self.SetRevision(revisionExpr, revisionId=revisionId)
 
     self.rebase = rebase
     self.groups = groups
@@ -556,16 +539,6 @@ class Project(object):
     self.copyfiles = []
     self.linkfiles = []
     self.annotations = []
-    self.config = GitConfig.ForRepository(gitdir=self.gitdir,
-                                          defaults=self.client.globalConfig)
-
-    if self.worktree:
-      self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
-    else:
-      self.work_git = None
-    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
-    self.bare_ref = GitRefs(gitdir)
-    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
     self.old_revision = old_revision
 
@@ -573,6 +546,47 @@ class Project(object):
     # project containing repo hooks.
     self.enabled_repo_hooks = []
 
+  def RelPath(self, local=True):
+    """Return the path for the project relative to a manifest.
+
+    Args:
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
+    """
+    if local:
+      return self.relpath
+    return os.path.join(self.manifest.path_prefix, self.relpath)
+
+  def SetRevision(self, revisionExpr, revisionId=None):
+    """Set revisionId based on revision expression and id"""
+    self.revisionExpr = revisionExpr
+    if revisionId is None and revisionExpr and IsId(revisionExpr):
+      self.revisionId = self.revisionExpr
+    else:
+      self.revisionId = revisionId
+
+  def UpdatePaths(self, relpath, worktree, gitdir, objdir):
+    """Update paths used by this project"""
+    self.gitdir = gitdir.replace('\\', '/')
+    self.objdir = objdir.replace('\\', '/')
+    if worktree:
+      self.worktree = os.path.normpath(worktree).replace('\\', '/')
+    else:
+      self.worktree = None
+    self.relpath = relpath
+
+    self.config = GitConfig.ForRepository(gitdir=self.gitdir,
+                                          defaults=self.manifest.globalConfig)
+
+    if self.worktree:
+      self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir)
+    else:
+      self.work_git = None
+    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
+    self.bare_ref = GitRefs(self.gitdir)
+    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
+
   @property
   def Derived(self):
     return self.is_derived
@@ -1118,7 +1132,7 @@ class Project(object):
     self._InitRemote()
 
     if is_new:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      alt = os.path.join(self.objdir, 'objects/info/alternates')
       try:
         with open(alt) as fd:
           # This works for both absolute and relative alternate directories.
@@ -1167,7 +1181,7 @@ class Project(object):
     mp = self.manifest.manifestProject
     dissociate = mp.config.GetBoolean('repo.dissociate')
     if dissociate:
-      alternates_file = os.path.join(self.gitdir, 'objects/info/alternates')
+      alternates_file = os.path.join(self.objdir, 'objects/info/alternates')
       if os.path.exists(alternates_file):
         cmd = ['repack', '-a', '-d']
         p = GitCommand(self, cmd, bare=True, capture_stdout=bool(output_redir),
@@ -1182,10 +1196,8 @@ class Project(object):
       self._InitMRef()
     else:
       self._InitMirrorHead()
-      try:
-        platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
-      except OSError:
-        pass
+      platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
+                            missing_ok=True)
     return True
 
   def PostRepoUpgrade(self):
@@ -2040,8 +2052,11 @@ class Project(object):
 
     if current_branch_only:
       if self.revisionExpr.startswith(R_TAGS):
-        # this is a tag and its sha1 value should never change
+        # This is a tag and its commit id should never change.
         tag_name = self.revisionExpr[len(R_TAGS):]
+      elif self.upstream and self.upstream.startswith(R_TAGS):
+        # This is a tag and its commit id should never change.
+        tag_name = self.upstream[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
         if self._CheckForImmutableRevision():
@@ -2139,8 +2154,7 @@ class Project(object):
     if prune:
       cmd.append('--prune')
 
-    if submodules:
-      cmd.append('--recurse-submodules=on-demand')
+    cmd.append(f'--recurse-submodules={"on-demand" if submodules else "no"}')
 
     spec = []
     if not current_branch_only:
@@ -2189,8 +2203,10 @@ class Project(object):
     retry_cur_sleep = retry_sleep_initial_sec
     ok = prune_tried = False
     for try_n in range(retry_fetches):
-      gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy,
-                          merge_output=True, capture_stdout=quiet or bool(output_redir))
+      gitcmd = GitCommand(
+          self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects'),
+          ssh_proxy=ssh_proxy,
+          merge_output=True, capture_stdout=quiet or bool(output_redir))
       if gitcmd.stdout and not quiet and output_redir:
         output_redir.write(gitcmd.stdout)
       ret = gitcmd.Wait()
@@ -2306,16 +2322,14 @@ class Project(object):
       cmd.append(str(f))
     cmd.append('+refs/tags/*:refs/tags/*')
 
-    ok = GitCommand(self, cmd, bare=True).Wait() == 0
-    if os.path.exists(bundle_dst):
-      platform_utils.remove(bundle_dst)
-    if os.path.exists(bundle_tmp):
-      platform_utils.remove(bundle_tmp)
+    ok = GitCommand(
+        self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects')).Wait() == 0
+    platform_utils.remove(bundle_dst, missing_ok=True)
+    platform_utils.remove(bundle_tmp, missing_ok=True)
     return ok
 
   def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
-    if os.path.exists(dstPath):
-      platform_utils.remove(dstPath)
+    platform_utils.remove(dstPath, missing_ok=True)
 
     cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
     if quiet:
@@ -2439,7 +2453,7 @@ class Project(object):
     if quiet:
       cmd.append('-q')
     if GitCommand(self, cmd).Wait() != 0:
-      raise GitError('%s submodule update --init --recursive %s ' % self.name)
+      raise GitError('%s submodule update --init --recursive ' % self.name)
 
   def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
@@ -2465,6 +2479,8 @@ class Project(object):
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
+        self._UpdateHooks(quiet=quiet)
+
         if self.use_git_worktrees:
           # Enable per-worktree config file support if possible.  This is more a
           # nice-to-have feature for users rather than a hard requirement.
@@ -2477,10 +2493,9 @@ class Project(object):
           os.makedirs(self.gitdir)
 
         if init_obj_dir or init_git_dir:
-          self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
-                                copy_all=True)
+          self._ReferenceGitDir(self.objdir, self.gitdir, copy_all=True)
         try:
-          self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
+          self._CheckDirReference(self.objdir, self.gitdir)
         except GitError as e:
           if force_sync:
             print("Retrying clone after deleting %s" %
@@ -2500,39 +2515,37 @@ class Project(object):
         mp = self.manifest.manifestProject
         ref_dir = mp.config.GetString('repo.reference') or ''
 
-        if ref_dir or mirror_git:
-          if not mirror_git:
-            mirror_git = os.path.join(ref_dir, self.name + '.git')
-          repo_git = os.path.join(ref_dir, '.repo', 'projects',
-                                  self.relpath + '.git')
-          worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees',
-                                       self.name + '.git')
+        def _expanded_ref_dirs():
+          """Iterate through the possible git reference directory paths."""
+          name = self.name + '.git'
+          yield mirror_git or os.path.join(ref_dir, name)
+          for prefix in '', self.remote.name:
+            yield os.path.join(ref_dir, '.repo', 'project-objects', prefix, name)
+            yield os.path.join(ref_dir, '.repo', 'worktrees', prefix, name)
 
-          if os.path.exists(mirror_git):
-            ref_dir = mirror_git
-          elif os.path.exists(repo_git):
-            ref_dir = repo_git
-          elif os.path.exists(worktrees_git):
-            ref_dir = worktrees_git
-          else:
-            ref_dir = None
+        if ref_dir or mirror_git:
+          found_ref_dir = None
+          for path in _expanded_ref_dirs():
+            if os.path.exists(path):
+              found_ref_dir = path
+              break
+          ref_dir = found_ref_dir
 
           if ref_dir:
             if not os.path.isabs(ref_dir):
               # The alternate directory is relative to the object database.
               ref_dir = os.path.relpath(ref_dir,
                                         os.path.join(self.objdir, 'objects'))
-            _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
+            _lwrite(os.path.join(self.objdir, 'objects/info/alternates'),
                     os.path.join(ref_dir, 'objects') + '\n')
 
-        self._UpdateHooks(quiet=quiet)
-
         m = self.manifest.manifestProject.config
         for key in ['user.name', 'user.email']:
           if m.Has(key, include_defaults=False):
             self.config.SetString(key, m.GetString(key))
-        self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
-        self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
+        if not self.manifest.EnableGitLfs:
+          self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
+          self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
         self.config.SetBoolean('core.bare', True if self.manifest.IsMirror else None)
     except Exception:
       if init_obj_dir and os.path.exists(self.objdir):
@@ -2542,13 +2555,21 @@ class Project(object):
       raise
 
   def _UpdateHooks(self, quiet=False):
-    if os.path.exists(self.gitdir):
+    if os.path.exists(self.objdir):
       self._InitHooks(quiet=quiet)
 
   def _InitHooks(self, quiet=False):
-    hooks = platform_utils.realpath(self._gitdir_path('hooks'))
+    hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks'))
     if not os.path.exists(hooks):
       os.makedirs(hooks)
+
+    # Delete sample hooks.  They're noise.
+    for hook in glob.glob(os.path.join(hooks, '*.sample')):
+      try:
+        platform_utils.remove(hook, missing_ok=True)
+      except PermissionError:
+        pass
+
     for stock_hook in _ProjectHooks():
       name = os.path.basename(stock_hook)
 
@@ -2646,40 +2667,16 @@ class Project(object):
         else:
           active_git.symbolic_ref('-m', msg, ref, dst)
 
-  def _CheckDirReference(self, srcdir, destdir, share_refs):
+  def _CheckDirReference(self, srcdir, destdir):
     # Git worktrees don't use symlinks to share at all.
     if self.use_git_worktrees:
       return
 
-    symlink_files = self.shareable_files[:]
-    symlink_dirs = self.shareable_dirs[:]
-    if share_refs:
-      symlink_files += self.working_tree_files
-      symlink_dirs += self.working_tree_dirs
-    to_symlink = symlink_files + symlink_dirs
-    for name in set(to_symlink):
+    for name in self.shareable_dirs:
       # Try to self-heal a bit in simple cases.
       dst_path = os.path.join(destdir, name)
       src_path = os.path.join(srcdir, name)
 
-      if name in self.working_tree_dirs:
-        # If the dir is missing under .repo/projects/, create it.
-        if not os.path.exists(src_path):
-          os.makedirs(src_path)
-
-      elif name in self.working_tree_files:
-        # If it's a file under the checkout .git/ and the .repo/projects/ has
-        # nothing, move the file under the .repo/projects/ tree.
-        if not os.path.exists(src_path) and os.path.isfile(dst_path):
-          platform_utils.rename(dst_path, src_path)
-
-      # If the path exists under the .repo/projects/ and there's no symlink
-      # under the checkout .git/, recreate the symlink.
-      if name in self.working_tree_dirs or name in self.working_tree_files:
-        if os.path.exists(src_path) and not os.path.exists(dst_path):
-          platform_utils.symlink(
-              os.path.relpath(src_path, os.path.dirname(dst_path)), dst_path)
-
       dst = platform_utils.realpath(dst_path)
       if os.path.lexists(dst):
         src = platform_utils.realpath(src_path)
@@ -2692,23 +2689,17 @@ class Project(object):
                          ' use `repo sync --force-sync {0}` to '
                          'proceed.'.format(self.relpath))
 
-  def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
+  def _ReferenceGitDir(self, gitdir, dotgit, copy_all):
     """Update |dotgit| to reference |gitdir|, using symlinks where possible.
 
     Args:
       gitdir: The bare git repository. Must already be initialized.
       dotgit: The repository you would like to initialize.
-      share_refs: If true, |dotgit| will store its refs under |gitdir|.
-          Only one work tree can store refs under a given |gitdir|.
       copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
           This saves you the effort of initializing |dotgit| yourself.
     """
-    symlink_files = self.shareable_files[:]
     symlink_dirs = self.shareable_dirs[:]
-    if share_refs:
-      symlink_files += self.working_tree_files
-      symlink_dirs += self.working_tree_dirs
-    to_symlink = symlink_files + symlink_dirs
+    to_symlink = symlink_dirs
 
     to_copy = []
     if copy_all:
@@ -2736,14 +2727,6 @@ class Project(object):
           elif os.path.isfile(src):
             shutil.copy(src, dst)
 
-        # If the source file doesn't exist, ensure the destination
-        # file doesn't either.
-        if name in symlink_files and not os.path.lexists(src):
-          try:
-            platform_utils.remove(dst)
-          except OSError:
-            pass
-
       except OSError as e:
         if e.errno == errno.EPERM:
           raise DownloadError(self._get_symlink_error_message())
@@ -2780,50 +2763,112 @@ class Project(object):
     self._InitMRef()
 
   def _InitWorkTree(self, force_sync=False, submodules=False):
-    realdotgit = os.path.join(self.worktree, '.git')
-    tmpdotgit = realdotgit + '.tmp'
-    init_dotgit = not os.path.exists(realdotgit)
-    if init_dotgit:
-      if self.use_git_worktrees:
+    """Setup the worktree .git path.
+
+    This is the user-visible path like src/foo/.git/.
+
+    With non-git-worktrees, this will be a symlink to the .repo/projects/ path.
+    With git-worktrees, this will be a .git file using "gitdir: ..." syntax.
+
+    Older checkouts had .git/ directories.  If we see that, migrate it.
+
+    This also handles changes in the manifest.  Maybe this project was backed
+    by "foo/bar" on the server, but now it's "new/foo/bar".  We have to update
+    the path we point to under .repo/projects/ to match.
+    """
+    dotgit = os.path.join(self.worktree, '.git')
+
+    # If using an old layout style (a directory), migrate it.
+    if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
+      self._MigrateOldWorkTreeGitDir(dotgit)
+
+    init_dotgit = not os.path.exists(dotgit)
+    if self.use_git_worktrees:
+      if init_dotgit:
         self._InitGitWorktree()
         self._CopyAndLinkFiles()
-        return
-
-      dotgit = tmpdotgit
-      platform_utils.rmtree(tmpdotgit, ignore_errors=True)
-      os.makedirs(tmpdotgit)
-      self._ReferenceGitDir(self.gitdir, tmpdotgit, share_refs=True,
-                            copy_all=False)
     else:
-      dotgit = realdotgit
+      if not init_dotgit:
+        # See if the project has changed.
+        if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit):
+          platform_utils.remove(dotgit)
 
-    try:
-      self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
-    except GitError as e:
-      if force_sync and not init_dotgit:
-        try:
-          platform_utils.rmtree(dotgit)
-          return self._InitWorkTree(force_sync=False, submodules=submodules)
-        except Exception:
-          raise e
-      raise e
+      if init_dotgit or not os.path.exists(dotgit):
+        os.makedirs(self.worktree, exist_ok=True)
+        platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit)
 
-    if init_dotgit:
-      _lwrite(os.path.join(tmpdotgit, HEAD), '%s\n' % self.GetRevisionId())
+      if init_dotgit:
+        _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
-      # Now that the .git dir is fully set up, move it to its final home.
-      platform_utils.rename(tmpdotgit, realdotgit)
+        # Finish checking out the worktree.
+        cmd = ['read-tree', '--reset', '-u', '-v', HEAD]
+        if GitCommand(self, cmd).Wait() != 0:
+          raise GitError('Cannot initialize work tree for ' + self.name)
 
-      # Finish checking out the worktree.
-      cmd = ['read-tree', '--reset', '-u']
-      cmd.append('-v')
-      cmd.append(HEAD)
-      if GitCommand(self, cmd).Wait() != 0:
-        raise GitError('Cannot initialize work tree for ' + self.name)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
+        self._CopyAndLinkFiles()
 
-      if submodules:
-        self._SyncSubmodules(quiet=True)
-      self._CopyAndLinkFiles()
+  @classmethod
+  def _MigrateOldWorkTreeGitDir(cls, dotgit):
+    """Migrate the old worktree .git/ dir style to a symlink.
+
+    This logic specifically only uses state from |dotgit| to figure out where to
+    move content and not |self|.  This way if the backing project also changed
+    places, we only do the .git/ dir to .git symlink migration here.  The path
+    updates will happen independently.
+    """
+    # Figure out where in .repo/projects/ it's pointing to.
+    if not os.path.islink(os.path.join(dotgit, 'refs')):
+      raise GitError(f'{dotgit}: unsupported checkout state')
+    gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs')))
+
+    # Remove known symlink paths that exist in .repo/projects/.
+    KNOWN_LINKS = {
+        'config', 'description', 'hooks', 'info', 'logs', 'objects',
+        'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+    }
+    # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
+    SAFE_TO_CLOBBER = {
+        'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gc.log', 'gitk.cache', 'index',
+        'ORIG_HEAD',
+    }
+
+    # First see if we'd succeed before starting the migration.
+    unknown_paths = []
+    for name in platform_utils.listdir(dotgit):
+      # Ignore all temporary/backup names.  These are common with vim & emacs.
+      if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
+        continue
+
+      dotgit_path = os.path.join(dotgit, name)
+      if name in KNOWN_LINKS:
+        if not platform_utils.islink(dotgit_path):
+          unknown_paths.append(f'{dotgit_path}: should be a symlink')
+      else:
+        gitdir_path = os.path.join(gitdir, name)
+        if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path):
+          unknown_paths.append(f'{dotgit_path}: unknown file; please file a bug')
+    if unknown_paths:
+      raise GitError('Aborting migration: ' + '\n'.join(unknown_paths))
+
+    # Now walk the paths and sync the .git/ to .repo/projects/.
+    for name in platform_utils.listdir(dotgit):
+      dotgit_path = os.path.join(dotgit, name)
+
+      # Ignore all temporary/backup names.  These are common with vim & emacs.
+      if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
+        platform_utils.remove(dotgit_path)
+      elif name in KNOWN_LINKS:
+        platform_utils.remove(dotgit_path)
+      else:
+        gitdir_path = os.path.join(gitdir, name)
+        platform_utils.remove(gitdir_path, missing_ok=True)
+        platform_utils.rename(dotgit_path, gitdir_path)
+
+    # Now that the dir should be empty, clear it out, and symlink it over.
+    platform_utils.rmdir(dotgit)
+    platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit)
 
   def _get_symlink_error_message(self):
     if platform_utils.isWindows():
@@ -2833,9 +2878,6 @@ class Project(object):
               'for other options.')
     return 'filesystem must support symlinks'
 
-  def _gitdir_path(self, path):
-    return platform_utils.realpath(os.path.join(self.gitdir, path))
-
   def _revlist(self, *args, **kw):
     a = []
     a.extend(args)

release/sign-launcher.py

@@ -20,6 +20,7 @@ This is intended to be run only by the official Repo release managers.
 
 import argparse
 import os
+import re
 import subprocess
 import sys
 
@@ -49,18 +50,37 @@ def check(opts):
   util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc'])
 
 
-def postmsg(opts):
+def get_version(opts):
+  """Get the version from |launcher|."""
+  # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+  launcher = os.path.join('.', opts.launcher)
+  cmd = [launcher, '--version']
+  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+  m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout)
+  if not m:
+    sys.exit(f'{opts.launcher}: unable to detect repo version')
+  return m.group(1)
+
+
+def postmsg(opts, version):
   """Helpful info to show at the end for release manager."""
   print(f"""
 Repo launcher bucket:
   gs://git-repo-downloads/
 
-To upload this launcher directly:
-  gsutil cp -a public-read {opts.launcher} {opts.launcher}.asc gs://git-repo-downloads/
+You should first upload it with a specific version:
+  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
 
-NB: You probably want to upload it with a specific version first, e.g.:
-  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-3.0
-  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-3.0.asc
+Then to make it the public default:
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+
+NB: If a rollback is necessary, the GS bucket archives old versions, and may be
+    accessed by specifying their unique id number.
+  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
 """)
 
 
@@ -103,9 +123,10 @@ def main(argv):
     opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
     util.import_release_key(opts)
 
+  version = get_version(opts)
   sign(opts)
   check(opts)
-  postmsg(opts)
+  postmsg(opts, version)
 
   return 0
 

release/update-manpages

@@ -59,11 +59,11 @@ def main(argv):
   version = RepoSourceVersion()
   cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
     '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
     '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
   cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
     '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
     '-h', '--help-all'])
 
   with tempfile.TemporaryDirectory() as tempdir:
@@ -80,11 +80,23 @@ def main(argv):
       (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
       (r'^\.PP\nDescription', '.SH DETAILS'),
   )
-  for path in MANDIR.glob('*.1'):
-    data = path.read_text()
+  for tmp_path in MANDIR.glob('*.1.tmp'):
+    path = tmp_path.parent / tmp_path.stem
+    old_data = path.read_text() if path.exists() else ''
+
+    data = tmp_path.read_text()
+    tmp_path.unlink()
+
     for pattern, replacement in regex:
       data = re.sub(pattern, replacement, data, flags=re.M)
-    path.write_text(data)
+
+    # If the only thing that changed was the date, don't refresh.  This avoids
+    # a lot of noise when only one file actually updates.
+    old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
+    new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
+    if old_data != new_data:
+      path.write_text(data)
+
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 15)
+VERSION = (2, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -312,6 +312,10 @@ def InitParser(parser, gitc_init=False):
                    metavar='PLATFORM')
   group.add_option('--submodules', action='store_true',
                    help='sync any submodules associated with the manifest repo')
+  group.add_option('--standalone-manifest', action='store_true',
+                   help='download the manifest as a static file '
+                        'rather then create a git checkout of '
+                        'the manifest repo')
 
   # Options that only affect manifest project, and not any of the projects
   # specified in the manifest itself.
@@ -368,7 +372,7 @@ def InitParser(parser, gitc_init=False):
                    help='filter for use with --partial-clone '
                         '[default: %default]')
   group.add_option('--use-superproject', action='store_true', default=None,
-                   help='use the manifest superproject to sync projects')
+                   help='use the manifest superproject to sync projects; implies -c')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')
@@ -378,6 +382,11 @@ def InitParser(parser, gitc_init=False):
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+  group.add_option('--git-lfs', action='store_true',
+                   help='enable Git LFS support')
+  group.add_option('--no-git-lfs',
+                   dest='git_lfs', action='store_false',
+                   help='disable Git LFS support')
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -614,6 +623,7 @@ def _Init(args, gitc_init=False):
             "REPO_URL set correctly?" % url, file=sys.stderr)
 
   except CloneFailure:
+    print('fatal: double check your --repo-rev setting.', file=sys.stderr)
     if opt.quiet:
       print('fatal: repo init failed; run without --quiet to see why',
             file=sys.stderr)

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/abandon.py

@@ -69,7 +69,8 @@ It is equivalent to "git branch -D <branchname>".
     nb = args[0]
     err = defaultdict(list)
     success = defaultdict(list)
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, states):
       for (results, project) in states:
@@ -94,7 +95,7 @@ It is equivalent to "git branch -D <branchname>".
         err_msg = "error: cannot abandon %s" % br
         print(err_msg, file=sys.stderr)
         for proj in err[br]:
-          print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
+          print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
       sys.exit(1)
     elif not success:
       print('error: no project has local branch(es) : %s' % nb,
@@ -110,5 +111,5 @@ It is equivalent to "git branch -D <branchname>".
           result = "all project"
         else:
           result = "%s" % (
-              ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br]))
+              ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
         print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))

subcmds/branches.py

@@ -98,7 +98,7 @@ is shown, then the branch appears in all projects.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
@@ -147,26 +147,28 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
+      _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
+        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            relpath = b.project.relpath
             if not i.IsSplitCurrent or b.current:
-              paths.append(b.project.relpath)
+              paths.append(_RelPath(b.project))
             else:
-              non_cur_paths.append(b.project.relpath)
+              non_cur_paths.append(_RelPath(b.project))
         else:
           fmt = out.notinproject
           in_type = 'not in'
           have = set()
           for b in i.projects:
-            have.add(b.project)
+            have.add(_RelPath(b.project))
           for p in projects:
-            if p not in have:
-              paths.append(p.relpath)
+            if _RelPath(p) not in have:
+              paths.append(_RelPath(p))
 
         s = ' %s %s' % (in_type, ', '.join(paths))
         if not i.IsSplitCurrent and (width + 7 + len(s) < 80):

subcmds/checkout.py

@@ -47,7 +47,7 @@ The command is equivalent to:
     nb = args[0]
     err = []
     success = []
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, results):
       for status, project in results:

subcmds/diff.py

@@ -50,7 +50,7 @@ to the Unix 'patch' command.
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0

subcmds/diffmanifests.py

@@ -179,6 +179,9 @@ synced and their revisions won't be found.
   def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
       self.OptionParser.error('missing manifests to diff')
+    if opt.this_manifest_only is False:
+      raise self.OptionParser.error(
+          '`diffmanifest` only supports the current tree')
 
   def Execute(self, opt, args):
     self.out = _Coloring(self.client.globalConfig)

subcmds/download.py

@@ -48,7 +48,7 @@ If no project is specified try to use current directory as a project.
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
-  def _ParseChangeIds(self, args):
+  def _ParseChangeIds(self, opt, args):
     if not args:
       self.Usage()
 
@@ -77,7 +77,7 @@ If no project is specified try to use current directory as a project.
                 ps_id = max(int(match.group(1)), ps_id)
         to_get.append((project, chg_id, ps_id))
       else:
-        projects = self.GetProjects([a])
+        projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only)
         if len(projects) > 1:
           # If the cwd is one of the projects, assume they want that.
           try:
@@ -88,8 +88,8 @@ If no project is specified try to use current directory as a project.
             print('error: %s matches too many projects; please re-run inside '
                   'the project checkout.' % (a,), file=sys.stderr)
             for project in projects:
-              print('  %s/ @ %s' % (project.relpath, project.revisionExpr),
-                    file=sys.stderr)
+              print('  %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only),
+                                    project.revisionExpr), file=sys.stderr)
             sys.exit(1)
         else:
           project = projects[0]
@@ -105,7 +105,7 @@ If no project is specified try to use current directory as a project.
         self.OptionParser.error('-x and --ff are mutually exclusive options')
 
   def Execute(self, opt, args):
-    for project, change_id, ps_id in self._ParseChangeIds(args):
+    for project, change_id, ps_id in self._ParseChangeIds(opt, args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
         print('[%s] change %d/%d not found'

subcmds/forall.py

@@ -168,6 +168,7 @@ without iterating through the remaining projects.
 
   def Execute(self, opt, args):
     cmd = [opt.command[0]]
+    all_trees = not opt.this_manifest_only
 
     shell = True
     if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
@@ -213,11 +214,11 @@ without iterating through the remaining projects.
       self.manifest.Override(smart_sync_manifest_path)
 
     if opt.regex:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=all_trees)
     elif opt.inverse_regex:
-      projects = self.FindProjects(args, inverse=True)
+      projects = self.FindProjects(args, inverse=True, all_manifests=all_trees)
     else:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
@@ -290,6 +291,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
 
   setenv('REPO_PROJECT', project.name)
   setenv('REPO_PATH', project.relpath)
+  setenv('REPO_OUTERPATH', project.RelPath(local=opt.this_manifest_only))
   setenv('REPO_REMOTE', project.remote.name)
   try:
     # If we aren't in a fully synced state and we don't have the ref the manifest
@@ -320,7 +322,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     output = ''
     if ((opt.project_header and opt.verbose)
             or not opt.project_header):
-      output = 'skipping %s/' % project.relpath
+      output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only)
     return (1, output)
 
   if opt.verbose:
@@ -344,7 +346,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
       if mirror:
         project_header_path = project.name
       else:
-        project_header_path = project.relpath
+        project_header_path = project.RelPath(local=opt.this_manifest_only)
       out.project('project %s/' % project_header_path)
       out.nl()
       buf.write(output)

subcmds/gitc_init.py

@@ -24,6 +24,7 @@ import wrapper
 
 class GitcInit(init.Init, GitcAvailableCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
 %prog [options] [client name]

subcmds/grep.py

@@ -172,15 +172,16 @@ contain a line that matches both expressions:
     return (project, p.Wait(), p.stdout, p.stderr)
 
   @staticmethod
-  def _ProcessResults(full_name, have_rev, _pool, out, results):
+  def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
     git_failed = False
     bad_rev = False
     have_match = False
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     for project, rc, stdout, stderr in results:
       if rc < 0:
         git_failed = True
-        out.project('--- project %s ---' % project.relpath)
+        out.project('--- project %s ---' % _RelPath(project))
         out.nl()
         out.fail('%s', stderr)
         out.nl()
@@ -192,7 +193,7 @@ contain a line that matches both expressions:
           if have_rev and 'fatal: ambiguous argument' in stderr:
             bad_rev = True
           else:
-            out.project('--- project %s ---' % project.relpath)
+            out.project('--- project %s ---' % _RelPath(project))
             out.nl()
             out.fail('%s', stderr.strip())
             out.nl()
@@ -208,13 +209,13 @@ contain a line that matches both expressions:
           rev, line = line.split(':', 1)
           out.write("%s", rev)
           out.write(':')
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
       elif full_name:
         for line in r:
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
@@ -239,7 +240,7 @@ contain a line that matches both expressions:
       cmd_argv.append(args[0])
       args = args[1:]
 
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     full_name = False
     if len(projects) > 1:
@@ -259,7 +260,7 @@ contain a line that matches both expressions:
         opt.jobs,
         functools.partial(self._ExecuteOne, cmd_argv),
         projects,
-        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        callback=functools.partial(self._ProcessResults, full_name, have_rev, opt),
         output=out,
         ordered=True)
 

subcmds/help.py

@@ -53,7 +53,7 @@ Displays detailed usage information about a command.
     self.PrintAllCommandsBody()
 
   def PrintAllCommandsBody(self):
-    print('The complete list of recognized repo commands are:')
+    print('The complete list of recognized repo commands is:')
     commandNames = list(sorted(all_commands))
     self._PrintCommands(commandNames)
     print("See 'repo help <command>' for more information on a "

subcmds/info.py

@@ -61,6 +61,8 @@ class Info(PagedCommand):
 
     self.opt = opt
 
+    if not opt.this_manifest_only:
+      self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
     manifestGroups = (manifestConfig.GetString('manifest.groups')
@@ -80,17 +82,17 @@ class Info(PagedCommand):
     self.printSeparator()
 
     if not opt.overview:
-      self.printDiffInfo(args)
+      self._printDiffInfo(opt, args)
     else:
-      self.printCommitOverview(args)
+      self._printCommitOverview(opt, args)
 
   def printSeparator(self):
     self.text("----------------------------")
     self.out.nl()
 
-  def printDiffInfo(self, args):
+  def _printDiffInfo(self, opt, args):
     # We let exceptions bubble up to main as they'll be well structured.
-    projs = self.GetProjects(args)
+    projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     for p in projs:
       self.heading("Project: ")
@@ -179,9 +181,9 @@ class Info(PagedCommand):
       self.text(" ".join(split[1:]))
       self.out.nl()
 
-  def printCommitOverview(self, args):
+  def _printCommitOverview(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -200,7 +202,7 @@ class Info(PagedCommand):
       if project != branch.project:
         project = branch.project
         self.out.nl()
-        self.headtext(project.relpath)
+        self.headtext(project.RelPath(local=opt.this_manifest_only))
         self.out.nl()
 
       commits = branch.commits

subcmds/init.py

@@ -24,6 +24,7 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
+import fetch
 import git_superproject
 import platform_utils
 from wrapper import Wrapper
@@ -31,6 +32,7 @@ from wrapper import Wrapper
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Initialize a repo client checkout in the current directory"
   helpUsage = """
 %prog [options] [manifest url]
@@ -53,6 +55,12 @@ The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
 will be used.
 
+If the --standalone-manifest argument is set, the manifest will be downloaded
+directly from the specified --manifest-url as a static file (rather than
+setting up a manifest git checkout). With --standalone-manifest, the manifest
+will be fully static and will not be re-downloaded during subsesquent
+`repo init` and `repo sync` calls.
+
 The --reference option can be used to point to a directory that
 has the content of a --mirror sync. This will make the working
 directory use as much data as possible from the local reference
@@ -83,6 +91,17 @@ to update the working directory files.
 
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
+    m = p.add_option_group('Multi-manifest')
+    m.add_option('--outer-manifest', action='store_true',
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', default=None,
+                 help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
 
   def _RegisteredEnvironmentOptions(self):
     return {'REPO_MANIFEST_URL': 'manifest_url',
@@ -112,6 +131,27 @@ to update the working directory files.
     m = self.manifest.manifestProject
     is_new = not m.Exists
 
+    # If repo has already been initialized, we take -u with the absence of
+    # --standalone-manifest to mean "transition to a standard repo set up",
+    # which necessitates starting fresh.
+    # If --standalone-manifest is set, we always tear everything down and start
+    # anew.
+    if not is_new:
+      was_standalone_manifest = m.config.GetString('manifest.standalone')
+      if was_standalone_manifest and not opt.manifest_url:
+        print('fatal: repo was initialized with a standlone manifest, '
+              'cannot be re-initialized without --manifest-url/-u')
+        sys.exit(1)
+
+      if opt.standalone_manifest or (was_standalone_manifest and
+                                     opt.manifest_url):
+        m.config.ClearCache()
+        if m.gitdir and os.path.exists(m.gitdir):
+          platform_utils.rmtree(m.gitdir)
+        if m.worktree and os.path.exists(m.worktree):
+          platform_utils.rmtree(m.worktree)
+
+    is_new = not m.Exists
     if is_new:
       if not opt.manifest_url:
         print('fatal: manifest url is required.', file=sys.stderr)
@@ -136,6 +176,21 @@ to update the working directory files.
 
       m._InitGitDir(mirror_git=mirrored_manifest_git)
 
+    # If standalone_manifest is set, mark the project as "standalone" -- we'll
+    # still do much of the manifests.git set up, but will avoid actual syncs to
+    # a remote.
+    standalone_manifest = False
+    if opt.standalone_manifest:
+      standalone_manifest = True
+      m.config.SetString('manifest.standalone', opt.manifest_url)
+    elif not opt.manifest_url and not opt.manifest_branch:
+      # If -u is set and --standalone-manifest is not, then we're not in
+      # standalone mode. Otherwise, use config to infer what we were in the last
+      # init.
+      standalone_manifest = bool(m.config.GetString('manifest.standalone'))
+    if not standalone_manifest:
+      m.config.SetString('manifest.standalone', None)
+
     self._ConfigureDepth(opt)
 
     # Set the remote URL before the remote branch as we might need it below.
@@ -145,22 +200,23 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
-    if opt.manifest_branch:
-      if opt.manifest_branch == 'HEAD':
-        opt.manifest_branch = m.ResolveRemoteHead()
-        if opt.manifest_branch is None:
-          print('fatal: unable to resolve HEAD', file=sys.stderr)
-          sys.exit(1)
-      m.revisionExpr = opt.manifest_branch
-    else:
-      if is_new:
-        default_branch = m.ResolveRemoteHead()
-        if default_branch is None:
-          # If the remote doesn't have HEAD configured, default to master.
-          default_branch = 'refs/heads/master'
-        m.revisionExpr = default_branch
+    if not standalone_manifest:
+      if opt.manifest_branch:
+        if opt.manifest_branch == 'HEAD':
+          opt.manifest_branch = m.ResolveRemoteHead()
+          if opt.manifest_branch is None:
+            print('fatal: unable to resolve HEAD', file=sys.stderr)
+            sys.exit(1)
+        m.revisionExpr = opt.manifest_branch
       else:
-        m.PreSync()
+        if is_new:
+          default_branch = m.ResolveRemoteHead()
+          if default_branch is None:
+            # If the remote doesn't have HEAD configured, default to master.
+            default_branch = 'refs/heads/master'
+          m.revisionExpr = default_branch
+        else:
+          m.PreSync()
 
     groups = re.split(r'[,\s]+', opt.groups)
     all_platforms = ['linux', 'darwin', 'windows']
@@ -247,9 +303,28 @@ to update the working directory files.
     if opt.submodules:
       m.config.SetBoolean('repo.submodules', opt.submodules)
 
+    if opt.git_lfs is not None:
+      if opt.git_lfs:
+        git_require((2, 17, 0), fail=True, msg='Git LFS support')
+
+      m.config.SetBoolean('repo.git-lfs', opt.git_lfs)
+      if not is_new:
+        print('warning: Changing --git-lfs settings will only affect new project checkouts.\n'
+              '         Existing projects will require manual updates.\n', file=sys.stderr)
+
     if opt.use_superproject is not None:
       m.config.SetBoolean('repo.superproject', opt.use_superproject)
 
+    if standalone_manifest:
+      if is_new:
+        manifest_name = 'default.xml'
+        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
+        dest = os.path.join(m.worktree, manifest_name)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        with open(dest, 'wb') as f:
+          f.write(manifest_data)
+      return
+
     if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
                               clone_bundle=opt.clone_bundle,
                               current_branch_only=opt.current_branch_only,
@@ -423,8 +498,18 @@ to update the working directory files.
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
-    if opt.archive and opt.mirror:
-      self.OptionParser.error('--mirror and --archive cannot be used together.')
+    if opt.mirror:
+      if opt.archive:
+        self.OptionParser.error('--mirror and --archive cannot be used '
+                                'together.')
+      if opt.use_superproject is not None:
+        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                                'used together.')
+
+    if opt.standalone_manifest and (opt.manifest_branch or
+                                    opt.manifest_name != 'default.xml'):
+      self.OptionParser.error('--manifest-branch and --manifest-name cannot'
+                              ' be used with --standalone-manifest.')
 
     if args:
       if opt.manifest_url:
@@ -456,8 +541,12 @@ to update the working directory files.
     # Handle new --repo-rev requests.
     if opt.repo_rev:
       wrapper = Wrapper()
-      remote_ref, rev = wrapper.check_repo_rev(
-          rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      try:
+        remote_ref, rev = wrapper.check_repo_rev(
+            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      except wrapper.CloneFailure:
+        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
+        sys.exit(1)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
       rp.work_git.reset('--hard', rev)

subcmds/list.py

@@ -77,16 +77,17 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
     if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all,
+                                  all_manifests=not opt.this_manifest_only)
     else:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _getpath(x):
       if opt.fullpath:
         return x.worktree
       if opt.relative_to:
         return os.path.relpath(x.worktree, opt.relative_to)
-      return x.relpath
+      return x.RelPath(local=opt.this_manifest_only)
 
     lines = []
     for project in projects:

subcmds/manifest.py

@@ -15,6 +15,7 @@
 import json
 import os
 import sys
+import optparse
 
 from command import PagedCommand
 
@@ -75,7 +76,7 @@ to indicate the remote ref to push changes to via 'repo upload'.
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
-                 help='file to save the manifest to',
+                 help='file to save the manifest to. (Filename prefix for multi-tree.)',
                  metavar='-|NAME.xml')
 
   def _Output(self, opt):
@@ -83,36 +84,45 @@ to indicate the remote ref to push changes to via 'repo upload'.
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name, False)
 
-    if opt.output_file == '-':
-      fd = sys.stdout
-    else:
-      fd = open(opt.output_file, 'w')
+    for manifest in self.ManifestList(opt):
+      output_file = opt.output_file
+      if output_file == '-':
+        fd = sys.stdout
+      else:
+        if manifest.path_prefix:
+          output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}'
+        fd = open(output_file, 'w')
 
-    self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
+      manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
 
-    if opt.json:
-      print('warning: --json is experimental!', file=sys.stderr)
-      doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
-                                 peg_rev_upstream=opt.peg_rev_upstream,
-                                 peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if opt.json:
+        print('warning: --json is experimental!', file=sys.stderr)
+        doc = manifest.ToDict(peg_rev=opt.peg_rev,
+                                   peg_rev_upstream=opt.peg_rev_upstream,
+                                   peg_rev_dest_branch=opt.peg_rev_dest_branch)
+
+        json_settings = {
+            # JSON style guide says Uunicode characters are fully allowed.
+            'ensure_ascii': False,
+            # We use 2 space indent to match JSON style guide.
+            'indent': 2 if opt.pretty else None,
+            'separators': (',', ': ') if opt.pretty else (',', ':'),
+            'sort_keys': True,
+        }
+        fd.write(json.dumps(doc, **json_settings))
+      else:
+        manifest.Save(fd,
+                      peg_rev=opt.peg_rev,
+                      peg_rev_upstream=opt.peg_rev_upstream,
+                      peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if output_file != '-':
+        fd.close()
+        if manifest.path_prefix:
+          print(f'Saved {manifest.path_prefix} submanifest to {output_file}',
+                file=sys.stderr)
+        else:
+          print(f'Saved manifest to {output_file}', file=sys.stderr)
 
-      json_settings = {
-          # JSON style guide says Uunicode characters are fully allowed.
-          'ensure_ascii': False,
-          # We use 2 space indent to match JSON style guide.
-          'indent': 2 if opt.pretty else None,
-          'separators': (',', ': ') if opt.pretty else (',', ':'),
-          'sort_keys': True,
-      }
-      fd.write(json.dumps(doc, **json_settings))
-    else:
-      self.manifest.Save(fd,
-                         peg_rev=opt.peg_rev,
-                         peg_rev_upstream=opt.peg_rev_upstream,
-                         peg_rev_dest_branch=opt.peg_rev_dest_branch)
-    fd.close()
-    if opt.output_file != '-':
-      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def ValidateOptions(self, opt, args):
     if args:

subcmds/overview.py

@@ -47,7 +47,7 @@ are displayed.
 
   def Execute(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -76,7 +76,7 @@ are displayed.
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       commits = branch.commits

subcmds/prune.py

@@ -31,7 +31,7 @@ class Prune(PagedCommand):
     return project.PruneHeads()
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     # NB: Should be able to refactor this module to display summary as results
     # come back from children.
@@ -63,7 +63,7 @@ class Prune(PagedCommand):
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       print('%s %-33s ' % (

subcmds/rebase.py

@@ -69,7 +69,7 @@ branch but need to incorporate new upstream changes "underneath" them.
                       'consistent if you previously synced to a manifest)')
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
@@ -98,6 +98,7 @@ branch but need to incorporate new upstream changes "underneath" them.
     config = self.manifest.manifestProject.config
     out = RebaseColoring(config)
     out.redirect(sys.stdout)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     ret = 0
     for project in all_projects:
@@ -107,7 +108,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print("error: project %s has a detached HEAD" % project.relpath,
+          print("error: project %s has a detached HEAD" % _RelPath(project),
                 file=sys.stderr)
           return 1
         # ignore branches with detatched HEADs
@@ -117,7 +118,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       if not upbranch.LocalMerge:
         if one_project:
           print("error: project %s does not track any remote branches"
-                % project.relpath, file=sys.stderr)
+                % _RelPath(project), file=sys.stderr)
           return 1
         # ignore branches without remotes
         continue
@@ -130,7 +131,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       args.append(upbranch.LocalMerge)
 
       out.project('project %s: rebasing %s -> %s',
-                  project.relpath, cb, upbranch.LocalMerge)
+                  _RelPath(project), cb, upbranch.LocalMerge)
       out.nl()
       out.flush()
 

subcmds/stage.py

@@ -50,7 +50,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
+    all_projects = [
+        p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        if p.IsDirty()]
     if not all_projects:
       print('no projects have uncommitted modifications', file=sys.stderr)
       return
@@ -62,7 +64,8 @@ The '%prog' command stages files to prepare the next commit.
 
       for i in range(len(all_projects)):
         project = all_projects[i]
-        out.write('%3d:    %s', i + 1, project.relpath + '/')
+        out.write('%3d:    %s', i + 1,
+                  project.RelPath(local=opt.this_manifest_only) + '/')
         out.nl()
       out.nl()
 
@@ -99,7 +102,9 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      projects = [
+          p for p in all_projects
+          if a in [p.name, p.RelPath(local=opt.this_manifest_only)]]
       if len(projects) == 1:
         _AddI(projects[0])
         continue

subcmds/start.py

@@ -84,7 +84,8 @@ revision specified in the manifest.
         projects = ['.']  # start it in the local project by default
 
     all_projects = self.GetProjects(projects,
-                                    missing_ok=bool(self.gitc_manifest))
+                                    missing_ok=bool(self.gitc_manifest),
+                                    all_manifests=not opt.this_manifest_only)
 
     # This must happen after we find all_projects, since GetProjects may need
     # the local directory, which will disappear once we save the GITC manifest.
@@ -137,6 +138,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print("error: %s/: cannot start %s" % (p.relpath, nb),
+        print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb),
               file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -117,7 +117,7 @@ the following meanings:
       outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -141,9 +141,10 @@ the following meanings:
     if opt.orphans:
       proj_dirs = set()
       proj_dirs_parents = set()
-      for project in self.GetProjects(None, missing_ok=True):
-        proj_dirs.add(project.relpath)
-        (head, _tail) = os.path.split(project.relpath)
+      for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only):
+        relpath = project.RelPath(local=opt.this_manifest_only)
+        proj_dirs.add(relpath)
+        (head, _tail) = os.path.split(relpath)
         while head != "":
           proj_dirs_parents.add(head)
           (head, _tail) = os.path.split(head)

subcmds/sync.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import errno
 import functools
 import http.cookiejar as cookielib
 import io
@@ -67,6 +66,7 @@ _ONE_DAY_S = 24 * 60 * 60
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -235,24 +235,25 @@ later is required to fix a server side protocol bug.
                  dest='fetch_submodules', action='store_true',
                  help='fetch submodules from server')
     p.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
+                 help='use the manifest superproject to sync projects; implies -c')
     p.add_option('--no-use-superproject', action='store_false',
                  dest='use_superproject',
                  help='disable use of manifest superprojects')
-    p.add_option('--tags',
-                 action='store_false',
+    p.add_option('--tags', action='store_true',
                  help='fetch tags')
     p.add_option('--no-tags',
                  dest='tags', action='store_false',
-                 help="don't fetch tags")
+                 help="don't fetch tags (default)")
     p.add_option('--optimized-fetch',
                  dest='optimized_fetch', action='store_true',
                  help='only fetch projects fixed to sha1 if revision does not exist locally')
     p.add_option('--retry-fetches',
                  default=0, action='store', type='int',
                  help='number of times to retry fetches on transient errors')
-    p.add_option('--prune', dest='prune', action='store_true',
-                 help='delete refs that no longer exist on the remote')
+    p.add_option('--prune', action='store_true',
+                 help='delete refs that no longer exist on the remote (default)')
+    p.add_option('--no-prune', dest='prune', action='store_false',
+                 help='do not delete refs that no longer exist on the remote')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -448,8 +449,8 @@ later is required to fix a server side protocol bug.
       else:
         pm.update(inc=0, msg='warming up')
         chunksize = 4
-      with multiprocessing.Pool(
-          jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
+      with multiprocessing.Pool(jobs, initializer=self._FetchInitChild,
+                                initargs=(ssh_proxy,)) as pool:
         results = pool.imap_unordered(
             functools.partial(self._FetchProjectList, opt),
             projects_list,
@@ -605,7 +606,7 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
@@ -622,17 +623,29 @@ later is required to fix a server side protocol bug.
                 % (project.relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
-      gc_gitdirs[project.gitdir] = project.bare_git
-
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+      project.config.SetString('gc.autoDetach', 'false')
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
+      if project.objdir not in tidy_dirs:
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
-        bare_git.gc('--auto')
+        if run_gc:
+          bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -641,11 +654,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
-          bare_git.gc('--auto', config=config)
+          if run_gc:
+            bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -655,11 +671,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -689,7 +705,7 @@ later is required to fix a server side protocol bug.
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
-    file_path = os.path.join(self.repodir, file_name)
+    file_path = os.path.join(self.manifest.subdir, file_name)
     old_project_paths = []
 
     if os.path.exists(file_path):
@@ -745,14 +761,14 @@ later is required to fix a server side protocol bug.
     }
 
     copylinkfile_name = 'copy-link-files.json'
-    copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
+    copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name)
     old_copylinkfile_paths = {}
 
     if os.path.exists(copylinkfile_path):
       with open(copylinkfile_path, 'rb') as fp:
         try:
           old_copylinkfile_paths = json.load(fp)
-        except:
+        except Exception:
           print('error: %s is not a json formatted file.' %
                 copylinkfile_path, file=sys.stderr)
           platform_utils.remove(copylinkfile_path)
@@ -767,13 +783,9 @@ later is required to fix a server side protocol bug.
           set(new_copyfile_paths))
 
       for need_remove_file in need_remove_files:
-        try:
-          platform_utils.remove(need_remove_file)
-        except OSError as e:
-          if e.errno == errno.ENOENT:
-            # Try to remove the updated copyfile or linkfile.
-            # So, if the file is not exist, nothing need to do.
-            pass
+        # Try to remove the updated copyfile or linkfile.
+        # So, if the file is not exist, nothing need to do.
+        platform_utils.remove(need_remove_file, missing_ok=True)
 
     # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
     with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
@@ -918,6 +930,12 @@ later is required to fix a server side protocol bug.
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
         self.OptionParser.error('both -u and -p must be given')
 
+    if opt.prune is None:
+      opt.prune = True
+
+    if self.manifest.is_multimanifest and not opt.this_manifest_only and args:
+      self.OptionParser.error('partial syncs must use --this-manifest-only')
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -958,18 +976,25 @@ later is required to fix a server side protocol bug.
               file=sys.stderr)
 
     mp = self.manifest.manifestProject
-    mp.PreSync()
+    is_standalone_manifest = mp.config.GetString('manifest.standalone')
+    if not is_standalone_manifest:
+      mp.PreSync()
 
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.mp_update:
       print('Skipping update of local manifest project.')
-    else:
+    elif not is_standalone_manifest:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
     load_local_manifests = not self.manifest.HasLocalManifests
     use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
+    if use_superproject and (self.manifest.IsMirror or self.manifest.IsArchive):
+      # Don't use superproject, because we have no working tree.
+      use_superproject = False
+      if opt.use_superproject is not None:
+        print('Defaulting to no-use-superproject because there is no working tree.')
     superproject_logging_data = {
         'superproject': use_superproject,
         'haslocalmanifests': bool(self.manifest.HasLocalManifests),
@@ -1092,19 +1117,28 @@ later is required to fix a server side protocol bug.
       sys.exit(1)
 
     # Log the previous sync analysis state from the config.
-    self.git_event_log.LogConfigEvents(mp.config.GetSyncAnalysisStateData(),
-                                       'previous_sync_state')
+    self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
+                                           'previous_sync_state')
 
     # Update and log with the new sync analysis state.
     mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data)
-    self.git_event_log.LogConfigEvents(mp.config.GetSyncAnalysisStateData(),
-                                       'current_sync_state')
+    self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
+                                           'current_sync_state')
 
     if not opt.quiet:
       print('repo sync has finished successfully.')
 
 
 def _PostRepoUpgrade(manifest, quiet=False):
+  # Link the docs for the internal .repo/ layout for people
+  link = os.path.join(manifest.repodir, 'internal-fs-layout.md')
+  if not platform_utils.islink(link):
+    target = os.path.join('repo', 'docs', 'internal-fs-layout.md')
+    try:
+      platform_utils.symlink(target, link)
+    except Exception:
+      pass
+
   wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)
@@ -1171,10 +1205,7 @@ class _FetchTimes(object):
         with open(self._path) as f:
           self._times = json.load(f)
       except (IOError, ValueError):
-        try:
-          platform_utils.remove(self._path)
-        except OSError:
-          pass
+        platform_utils.remove(self._path, missing_ok=True)
         self._times = {}
 
   def Save(self):
@@ -1192,10 +1223,7 @@ class _FetchTimes(object):
       with open(self._path, 'w') as f:
         json.dump(self._times, f, indent=2)
     except (IOError, TypeError):
-      try:
-        platform_utils.remove(self._path)
-      except OSError:
-        pass
+      platform_utils.remove(self._path, missing_ok=True)
 
 # This is a replacement for xmlrpc.client.Transport using urllib2
 # and supporting persistent-http[s]. It cannot change hosts from

subcmds/upload.py

@@ -226,7 +226,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
       destination = opt.dest_branch or project.dest_branch or project.revisionExpr
       print('Upload project %s/ to remote branch %s%s:' %
-            (project.relpath, destination, ' (private)' if opt.private else ''))
+            (project.RelPath(local=opt.this_manifest_only), destination,
+             ' (private)' if opt.private else ''))
       print('  branch %s (%2d commit%s, %s):' % (
           name,
           len(commit_list),
@@ -262,7 +263,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     script.append('# Uncomment the branches to upload:')
     for project, avail in pending:
       script.append('#')
-      script.append('# project %s/:' % project.relpath)
+      script.append('# project %s/:' % project.RelPath(local=opt.this_manifest_only))
 
       b = {}
       for branch in avail:
@@ -285,7 +286,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           script.append('#         %s' % commit)
         b[name] = branch
 
-      projects[project.relpath] = project
+      projects[project.RelPath(local=opt.this_manifest_only)] = project
       branches[project.name] = b
     script.append('')
 
@@ -313,7 +314,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           _die('project for branch %s not in script', name)
         branch = branches[project.name].get(name)
         if not branch:
-          _die('branch %s not in %s', name, project.relpath)
+          _die('branch %s not in %s', name, project.RelPath(local=opt.this_manifest_only))
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
@@ -481,7 +482,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           else:
             fmt = '\n       (%s)'
           print(('[FAILED] %-15s %-15s' + fmt) % (
-              branch.project.relpath + '/',
+              branch.project.RelPath(local=opt.this_manifest_only) + '/',
               branch.name,
               str(branch.error)),
               file=sys.stderr)
@@ -490,7 +491,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     for branch in todo:
       if branch.uploaded:
         print('[OK    ] %-15s %s' % (
-            branch.project.relpath + '/',
+            branch.project.RelPath(local=opt.this_manifest_only) + '/',
             branch.name),
             file=sys.stderr)
 
@@ -524,7 +525,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     return (project, avail)
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _out, results):
       pending = []
@@ -534,7 +535,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           print('repo: error: %s: Unable to upload branch "%s". '
                 'You might be able to fix the branch by running:\n'
                 '  git branch --set-upstream-to m/%s' %
-                (project.relpath, project.CurrentBranch, self.manifest.branch),
+                (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch,
+                 project.manifest.branch),
                 file=sys.stderr)
         elif avail:
           pending.append(result)
@@ -554,15 +556,23 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
               (opt.branch,), file=sys.stderr)
       return 1
 
-    pending_proj_names = [project.name for (project, available) in pending]
-    pending_worktrees = [project.worktree for (project, available) in pending]
-    hook = RepoHook.FromSubcmd(
-        hook_type='pre-upload', manifest=self.manifest,
-        opt=opt, abort_if_user_denies=True)
-    if not hook.Run(
-        project_list=pending_proj_names,
-        worktree_list=pending_worktrees):
-      return 1
+    manifests = {project.manifest.topdir: project.manifest
+                 for (project, available) in pending}
+    ret = 0
+    for manifest in manifests.values():
+      pending_proj_names = [project.name for (project, available) in pending
+                            if project.manifest.topdir == manifest.topdir]
+      pending_worktrees = [project.worktree for (project, available) in pending
+                           if project.manifest.topdir == manifest.topdir]
+      hook = RepoHook.FromSubcmd(
+          hook_type='pre-upload', manifest=manifest,
+          opt=opt, abort_if_user_denies=True)
+      if not hook.Run(
+          project_list=pending_proj_names,
+          worktree_list=pending_worktrees):
+        ret = 1
+    if ret:
+      return ret
 
     reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
     cc = _SplitEmails(opt.cc) if opt.cc else []

tests/fixtures/test.gitconfig

@@ -11,13 +11,3 @@
 	intk = 10k
 	intm = 10m
 	intg = 10g
-[repo "syncstate.main"]
-	synctime = 2021-08-13T18:37:43.928600Z
-	version = 1
-[repo "syncstate.sys"]
-	argv = ['/usr/bin/pytest-3']
-[repo "syncstate.superproject"]
-	test = false
-[repo "syncstate.options"]
-	verbose = true
-	mpupdate = false

tests/test_git_config.py

@@ -104,25 +104,6 @@ class GitConfigReadOnlyTests(unittest.TestCase):
     for key, value in TESTS:
       self.assertEqual(value, self.config.GetInt('section.%s' % (key,)))
 
-  def test_GetSyncAnalysisStateData(self):
-    """Test config entries with a sync state analysis data."""
-    superproject_logging_data = {}
-    superproject_logging_data['test'] = False
-    options = type('options', (object,), {})()
-    options.verbose = 'true'
-    options.mp_update = 'false'
-    TESTS = (
-        ('superproject.test', 'false'),
-        ('options.verbose', 'true'),
-        ('options.mpupdate', 'false'),
-        ('main.version', '1'),
-    )
-    self.config.UpdateSyncAnalysisState(options, superproject_logging_data)
-    sync_data = self.config.GetSyncAnalysisStateData()
-    for key, value in TESTS:
-      self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
-    self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
-
 
 class GitConfigReadWriteTests(unittest.TestCase):
   """Read/write tests of the GitConfig class."""
@@ -187,6 +168,25 @@ class GitConfigReadWriteTests(unittest.TestCase):
     config = self.get_config()
     self.assertIsNone(config.GetBoolean('foo.bar'))
 
+  def test_GetSyncAnalysisStateData(self):
+    """Test config entries with a sync state analysis data."""
+    superproject_logging_data = {}
+    superproject_logging_data['test'] = False
+    options = type('options', (object,), {})()
+    options.verbose = 'true'
+    options.mp_update = 'false'
+    TESTS = (
+        ('superproject.test', 'false'),
+        ('options.verbose', 'true'),
+        ('options.mpupdate', 'false'),
+        ('main.version', '1'),
+    )
+    self.config.UpdateSyncAnalysisState(options, superproject_logging_data)
+    sync_data = self.config.GetSyncAnalysisStateData()
+    for key, value in TESTS:
+      self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
+    self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
+
 
 if __name__ == '__main__':
   unittest.main()

tests/test_git_superproject.py

@@ -157,7 +157,7 @@ class SuperprojectTestCase(unittest.TestCase):
 """)
     self._superproject = git_superproject.Superproject(manifest, self.repodir,
                                                        self.git_event_log)
-    with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'):
+    with mock.patch.object(self._superproject, '_branch', 'junk'):
       sync_result = self._superproject.Sync()
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)

tests/test_git_trace2_event_log.py

@@ -42,7 +42,7 @@ class EventLogTestCase(unittest.TestCase):
     self._event_log_module = git_trace2_event_log.EventLog(env=env)
     self._log_data = None
 
-  def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
+  def verifyCommonKeys(self, log_entry, expected_event_name=None, full_sid=True):
     """Helper function to verify common event log keys."""
     self.assertIn('event', log_entry)
     self.assertIn('sid', log_entry)
@@ -50,7 +50,8 @@ class EventLogTestCase(unittest.TestCase):
     self.assertIn('time', log_entry)
 
     # Do basic data format validation.
-    self.assertEqual(expected_event_name, log_entry['event'])
+    if expected_event_name:
+      self.assertEqual(expected_event_name, log_entry['event'])
     if full_sid:
       self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
     else:
@@ -65,6 +66,13 @@ class EventLogTestCase(unittest.TestCase):
         log_data.append(json.loads(line))
     return log_data
 
+  def remove_prefix(self, s, prefix):
+    """Return a copy string after removing |prefix| from |s|, if present or the original string."""
+    if s.startswith(prefix):
+        return s[len(prefix):]
+    else:
+        return s
+
   def test_initial_state_with_parent_sid(self):
     """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent."""
     self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX)
@@ -234,6 +242,42 @@ class EventLogTestCase(unittest.TestCase):
     self.assertEqual(len(self._log_data), 1)
     self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
 
+  def test_data_event_config(self):
+    """Test 'data' event data outputs all config keys.
+
+    Expected event log:
+    <version event>
+    <data event>
+    <data event>
+    """
+    config = {
+        'git.foo': 'bar',
+        'repo.partialclone': 'false',
+        'repo.syncstate.superproject.hassuperprojecttag': 'true',
+        'repo.syncstate.superproject.sys.argv': ['--', 'sync', 'protobuf'],
+    }
+    prefix_value = 'prefix'
+    self._event_log_module.LogDataConfigEvents(config, prefix_value)
+
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 5)
+    data_events = self._log_data[1:]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+
+    for event in data_events:
+      self.verifyCommonKeys(event)
+      # Check for 'data' event specific fields.
+      self.assertIn('key', event)
+      self.assertIn('value', event)
+      key = event['key']
+      key = self.remove_prefix(key, f'{prefix_value}/')
+      value = event['value']
+      self.assertEqual(self._event_log_module.GetDataEventName(value), event['event'])
+      self.assertTrue(key in config and value == config[key])
+
   def test_error_event(self):
     """Test and validate 'error' event data is valid.
 

tests/test_manifest_xml.py

@@ -261,6 +261,19 @@ class XmlManifestTests(ManifestParseTestCase):
   <project name="repohooks" path="src/repohooks"/>
   <repo-hooks in-project="repohooks" enabled-list="a, b"/>
 </manifest>
+""")
+    self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
+    self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
+
+  def test_repo_hooks_unordered(self):
+    """Check repo-hooks settings work even if the project def comes second."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <repo-hooks in-project="repohooks" enabled-list="a, b"/>
+  <project name="repohooks" path="src/repohooks"/>
+</manifest>
 """)
     self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
     self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
@@ -559,6 +572,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -567,6 +581,72 @@ class SuperProjectElementTests(ManifestParseTestCase):
         '<superproject name="superproject"/>'
         '</manifest>')
 
+  def test_superproject_revision(self):
+    """Check superproject settings with a different revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/main"/>'
+        '<superproject name="superproject" revision="refs/heads/stable"/>'
+        '</manifest>')
+
+  def test_superproject_revision_default_negative(self):
+    """Check superproject settings with a same revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/stable" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/stable"/>'
+        '<superproject name="superproject"/>'
+        '</manifest>')
+
+  def test_superproject_revision_remote(self):
+    """Check superproject settings with a same revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" revision="refs/heads/main" />
+  <default remote="test-remote" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote" revision="refs/heads/main"/>'
+        '<default remote="test-remote"/>'
+        '<superproject name="superproject" revision="refs/heads/stable"/>'
+        '</manifest>')
+
   def test_remote(self):
     """Check superproject settings with a remote."""
     manifest = self.getXmlManifest("""
@@ -580,6 +660,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'platform/superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -600,6 +681,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
 """)
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -715,3 +797,49 @@ class RemoveProjectElementTests(ManifestParseTestCase):
 </manifest>
 """)
     self.assertEqual(manifest.projects, [])
+
+
+class ExtendProjectElementTests(ManifestParseTestCase):
+  """Tests for <extend-project>."""
+
+  def test_extend_project_dest_path_single_match(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <extend-project name="myproject" dest-path="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].relpath, 'bar')
+
+  def test_extend_project_dest_path_multi_match(self):
+    with self.assertRaises(manifest_xml.ManifestParseError):
+      manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" path="x" />
+  <project name="myproject" path="y" />
+  <extend-project name="myproject" dest-path="bar" />
+</manifest>
+""")
+      manifest.projects
+
+  def test_extend_project_dest_path_multi_match_path_specified(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" path="x" />
+  <project name="myproject" path="y" />
+  <extend-project name="myproject" path="x" dest-path="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 2)
+    if manifest.projects[0].relpath == 'y':
+      self.assertEqual(manifest.projects[1].relpath, 'bar')
+    else:
+      self.assertEqual(manifest.projects[0].relpath, 'bar')
+      self.assertEqual(manifest.projects[1].relpath, 'y')

tests/test_platform_utils.py

@@ -0,0 +1,50 @@
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the platform_utils.py module."""
+
+import os
+import tempfile
+import unittest
+
+import platform_utils
+
+
+class RemoveTests(unittest.TestCase):
+  """Check remove() helper."""
+
+  def testMissingOk(self):
+    """Check missing_ok handling."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+      path = os.path.join(tmpdir, 'test')
+
+      # Should not fail.
+      platform_utils.remove(path, missing_ok=True)
+
+      # Should fail.
+      self.assertRaises(OSError, platform_utils.remove, path)
+      self.assertRaises(OSError, platform_utils.remove, path, missing_ok=False)
+
+      # Should not fail if it exists.
+      open(path, 'w').close()
+      platform_utils.remove(path, missing_ok=True)
+      self.assertFalse(os.path.exists(path))
+
+      open(path, 'w').close()
+      platform_utils.remove(path)
+      self.assertFalse(os.path.exists(path))
+
+      open(path, 'w').close()
+      platform_utils.remove(path, missing_ok=False)
+      self.assertFalse(os.path.exists(path))

tests/test_project.py

@@ -16,6 +16,7 @@
 
 import contextlib
 import os
+from pathlib import Path
 import shutil
 import subprocess
 import tempfile
@@ -335,3 +336,76 @@ class LinkFile(CopyLinkTestCase):
     platform_utils.symlink(self.tempdir, dest)
     lf._Link()
     self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest))
+
+
+class MigrateWorkTreeTests(unittest.TestCase):
+  """Check _MigrateOldWorkTreeGitDir handling."""
+
+  _SYMLINKS = {
+      'config', 'description', 'hooks', 'info', 'logs', 'objects',
+      'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+  }
+  _FILES = {
+      'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+      'unknown-file-should-be-migrated',
+  }
+  _CLEAN_FILES = {
+      'a-vim-temp-file~', '#an-emacs-temp-file#',
+  }
+
+  @classmethod
+  @contextlib.contextmanager
+  def _simple_layout(cls):
+    """Create a simple repo client checkout to test against."""
+    with tempfile.TemporaryDirectory() as tempdir:
+      tempdir = Path(tempdir)
+
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      gitdir.mkdir(parents=True)
+      cmd = ['git', 'init', '--bare', str(gitdir)]
+      subprocess.check_call(cmd)
+
+      dotgit = tempdir / 'src/test/.git'
+      dotgit.mkdir(parents=True)
+      for name in cls._SYMLINKS:
+        (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
+      for name in cls._FILES | cls._CLEAN_FILES:
+        (dotgit / name).write_text(name)
+
+      yield tempdir
+
+  def test_standard(self):
+    """Migrate a standard checkout that we expect."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure the dir was transformed into a symlink.
+      self.assertTrue(dotgit.is_symlink())
+      self.assertEqual(os.readlink(dotgit), '../../.repo/projects/src/test.git')
+
+      # Make sure files were moved over.
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      for name in self._FILES:
+        self.assertEqual(name, (gitdir / name).read_text())
+      # Make sure files were removed.
+      for name in self._CLEAN_FILES:
+        self.assertFalse((gitdir / name).exists())
+
+  def test_unknown(self):
+    """A checkout with unknown files should abort."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      (tempdir / '.repo/projects/src/test.git/random-file').write_text('one')
+      (dotgit / 'random-file').write_text('two')
+      with self.assertRaises(error.GitError):
+        project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure no content was actually changed.
+      self.assertTrue(dotgit.is_dir())
+      for name in self._FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._CLEAN_FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._SYMLINKS:
+        self.assertTrue((dotgit / name).is_symlink())