Sync: Fix full submodule sync while shallow specified

Git allows to clone submodules as shallow clone [1]. On the other
hand, when repo synchronize a projcet with submodules inside, it
ignores the shallow parameter.

When a project contains submodules, project.py parses the .gitmodules
file for URL and path. This parsing does not consider the shallow
option. Consequently, this parameter is not propgated to newly
created Project instance for that submodule.

[1] https://git-scm.com/docs/gitmodules#Documentation/gitmodules.txt-submoduleltnamegtshallow

Change-Id: I54fc9c69ae1b8e3cda2801202e3f0c7693b718d2
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/454261
Tested-by: Albert Akmukhametov <alb.02057@gmail.com>
Commit-Queue: Albert Akmukhametov <alb.02057@gmail.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Никита Сказкоподателев (Nask) <skazkopodatelev@gmail.com>

command.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import multiprocessing
 import optparse
 import os
@@ -70,6 +71,14 @@ class Command:
     # migrated subcommands can set it to False.
     MULTI_MANIFEST_SUPPORT = True
 
+    # Shared data across parallel execution workers.
+    _parallel_context = None
+
+    @classmethod
+    def get_parallel_context(cls):
+        assert cls._parallel_context is not None
+        return cls._parallel_context
+
     def __init__(
         self,
         repodir=None,
@@ -242,9 +251,39 @@ class Command:
         """Perform the action, after option parsing is complete."""
         raise NotImplementedError
 
-    @staticmethod
+    @classmethod
+    @contextlib.contextmanager
+    def ParallelContext(cls):
+        """Obtains the context, which is shared to ExecuteInParallel workers.
+
+        Callers can store data in the context dict before invocation of
+        ExecuteInParallel. The dict will then be shared to child workers of
+        ExecuteInParallel.
+        """
+        assert cls._parallel_context is None
+        cls._parallel_context = {}
+        try:
+            yield
+        finally:
+            cls._parallel_context = None
+
+    @classmethod
+    def _InitParallelWorker(cls, context, initializer):
+        cls._parallel_context = context
+        if initializer:
+            initializer()
+
+    @classmethod
     def ExecuteInParallel(
-        jobs, func, inputs, callback, output=None, ordered=False
+        cls,
+        jobs,
+        func,
+        inputs,
+        callback,
+        output=None,
+        ordered=False,
+        chunksize=WORKER_BATCH_SIZE,
+        initializer=None,
     ):
         """Helper for managing parallel execution boiler plate.
 
@@ -269,6 +308,9 @@ class Command:
             output: An output manager. May be progress.Progess or
                 color.Coloring.
             ordered: Whether the jobs should be processed in order.
+            chunksize: The number of jobs processed in batch by parallel
+                workers.
+            initializer: Worker initializer.
 
         Returns:
             The |callback| function's results are returned.
@@ -278,12 +320,16 @@ class Command:
             if len(inputs) == 1 or jobs == 1:
                 return callback(None, output, (func(x) for x in inputs))
             else:
-                with multiprocessing.Pool(jobs) as pool:
+                with multiprocessing.Pool(
+                    jobs,
+                    initializer=cls._InitParallelWorker,
+                    initargs=(cls._parallel_context, initializer),
+                ) as pool:
                     submit = pool.imap if ordered else pool.imap_unordered
                     return callback(
                         pool,
                         output,
-                        submit(func, inputs, chunksize=WORKER_BATCH_SIZE),
+                        submit(func, inputs, chunksize=chunksize),
                     )
         finally:
             if isinstance(output, progress.Progress):
@@ -501,7 +547,3 @@ class MirrorSafeCommand:
     """Command permits itself to run within a mirror, and does not require a
     working directory.
     """
-
-
-class GitcClientCommand:
-    """Command that requires the local client to be a GITC client."""

docs/internal-fs-layout.md

@@ -141,7 +141,7 @@ Instead, you should use standard Git workflows like [git worktree] or
     (e.g. a local mirror & a public review server) while avoiding duplicating
     the content.  However, this can run into problems if different remotes use
     the same path on their respective servers.  Best to avoid that.
-*   `subprojects/`: Like `projects/`, but for git submodules.
+*   `modules/`: Like `projects/`, but for git submodules.
 *   `subproject-objects/`: Like `project-objects/`, but for git submodules.
 *   `worktrees/`: Bare checkouts of every project synced by the manifest.  The
     filesystem layout matches the `<project name=...` setting in the manifest

docs/manifest-format.md

@@ -107,11 +107,13 @@ following DTD:
   <!ATTLIST extend-project remote CDATA #IMPLIED>
   <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
   <!ATTLIST extend-project upstream CDATA #IMPLIED>
+  <!ATTLIST extend-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name     CDATA #IMPLIED>
   <!ATTLIST remove-project path     CDATA #IMPLIED>
   <!ATTLIST remove-project optional CDATA #IMPLIED>
+  <!ATTLIST remove-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT repo-hooks EMPTY>
   <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
@@ -229,26 +231,7 @@ At most one manifest-server may be specified. The url attribute
 is used to specify the URL of a manifest server, which is an
 XML RPC service.
 
-The manifest server should implement the following RPC methods:
-
-    GetApprovedManifest(branch, target)
-
-Return a manifest in which each project is pegged to a known good revision
-for the current branch and target. This is used by repo sync when the
---smart-sync option is given.
-
-The target to use is defined by environment variables TARGET_PRODUCT
-and TARGET_BUILD_VARIANT. These variables are used to create a string
-of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
-If one of those variables or both are not present, the program will call
-GetApprovedManifest without the target parameter and the manifest server
-should choose a reasonable default target.
-
-    GetManifest(tag)
-
-Return a manifest in which each project is pegged to the revision at
-the specified tag. This is used by repo sync when the --smart-tag option
-is given.
+See the [smart sync documentation](./smart-sync.md) for more details.
 
 
 ### Element submanifest
@@ -433,6 +416,14 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `base-rev`: If specified, adds a check against the revision
+to be extended. Manifest parse will fail and give a list of mismatch extends
+if the revisions being extended have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
@@ -496,6 +487,14 @@ name. Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove-project elements with no
 matching `project` element.
 
+Attribute `base-rev`: If specified, adds a check against the revision
+to be removed. Manifest parse will fail and give a list of mismatch removes
+if the revisions being removed have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element repo-hooks
 
 NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.

docs/release-process.md

@@ -96,6 +96,9 @@ If that tag is valid, then repo will warn and use that commit instead.
 
 If that tag cannot be verified, it gives up and forces the user to resolve.
 
+If env variable `REPO_SKIP_SELF_UPDATE` is defined, this will
+bypass the self update algorithm.
+
 ### Force an update
 
 The `repo selfupdate` command can be used to force an immediate update.

docs/smart-sync.md

@@ -0,0 +1,129 @@
+# repo Smart Syncing
+
+Repo normally fetches & syncs manifests from the same URL specified during
+`repo init`, and that often fetches the latest revisions of all projects in
+the manifest.  This flow works well for tracking and developing with the
+latest code, but often it's desirable to sync to other points.  For example,
+to get a local build matching a specific release or build to reproduce bugs
+reported by other people.
+
+Repo's sync subcommand has support for fetching manifests from a server over
+an XML-RPC connection.  The local configuration and network API are defined by
+repo, but individual projects have to host their own server for the client to
+communicate with.
+
+This process is called "smart syncing" -- instead of blindly fetching the latest
+revision of all projects and getting an unknown state to develop against, the
+client passes a request to the server and is given a matching manifest that
+typically specifies specific commits for every project to fetch a known source
+state.
+
+[TOC]
+
+## Manifest Configuration
+
+The manifest specifies the server to communicate with via the
+the [`<manifest-server>` element](manifest-format.md#Element-manifest_server)
+element.  This is how the client knows what service to talk to.
+
+```xml
+  <manifest-server url="https://example.com/your/manifest/server/url" />
+```
+
+If the URL starts with `persistent-`, then the
+[`git-remote-persistent-https` helper](https://github.com/git/git/blob/HEAD/contrib/persistent-https/README)
+is used to communicate with the server.
+
+## Credentials
+
+Credentials may be specified directly in typical `username:password`
+[URI syntax](https://en.wikipedia.org/wiki/URI#Syntax) in the
+`<manifest-server>` element directly in the manifest.
+
+If they are not specified, `repo sync` has `--manifest-server-username=USERNAME`
+and `--manifest-server-password=PASSWORD` options.
+
+If those are not used, then repo will look up the host in your
+[`~/.netrc`](https://docs.python.org/3/library/netrc.html) database.
+
+When making the connection, cookies matching the host are automatically loaded
+from the cookiejar specified in
+[Git's `http.cookiefile` setting](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpcookieFile).
+
+## Manifest Server
+
+Unfortunately, there are no public reference implementations.  Google has an
+internal one for Android, but it is written using Google's internal systems,
+so wouldn't be that helpful as a reference.
+
+That said, the XML-RPC API is pretty simple, so any standard XML-RPC server
+example would do.  Google's internal server uses Python's
+[xmlrpc.server.SimpleXMLRPCDispatcher](https://docs.python.org/3/library/xmlrpc.server.html).
+
+## Network API
+
+The manifest server should implement the following RPC methods.
+
+### GetApprovedManifest
+
+> `GetApprovedManifest(branch: str, target: Optional[str]) -> str`
+
+The meaning of `branch` and `target` is not strictly defined.  The server may
+interpret them however it wants.  The recommended interpretation is that the
+`branch` matches the manifest branch, and `target` is an identifier for your
+project that matches something users would build.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+### GetManifest
+
+> `GetManifest(tag: str) -> str`
+
+The meaning of `tag` is not strictly defined.  Projects are encouraged to use
+a system where the tag matches a unique source state.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+## Client Options
+
+Once repo has successfully downloaded the manifest from the server, it saves a
+copy into `.repo/manifests/smart_sync_override.xml` so users can examine it.
+The next time `repo sync` is run, this file is automatically replaced or removed
+based on the current set of options.
+
+### --smart-sync
+
+Repo will call `GetApprovedManifest(branch[, target])`.
+
+The `branch` is determined by the current manifest branch as specified by
+`--manifest-branch=BRANCH` when running `repo init`.
+
+The `target` is defined by environment variables in the order below.  If none
+of them match, then `target` is omitted.  These variables were decided as they
+match the settings Android build environments automatically setup.
+
+1.  `${SYNC_TARGET}`: If defined, the value is used directly.
+2.  `${TARGET_PRODUCT}-${TARGET_RELEASE}-${TARGET_BUILD_VARIANT}`: If these
+    variables are all defined, then they are merged with `-` and used.
+3.  `${TARGET_PRODUCT}-${TARGET_BUILD_VARIANT}`: If these variables are all
+    defined, then they are merged with `-` and used.
+
+### --smart-tag=TAG
+
+Repo will call `GetManifest(TAG)`.

error.py

@@ -111,10 +111,6 @@ class GitAuthError(RepoExitError):
     """Cannot talk to remote due to auth issue."""
 
 
-class GitcUnsupportedError(RepoExitError):
-    """Gitc no longer supported."""
-
-
 class UploadError(RepoError):
     """A bundle upload to Gerrit did not succeed."""
 

event_log.py

@@ -168,8 +168,10 @@ class EventLog:
                 f.write("\n")
 
 
-# An integer id that is unique across this invocation of the program.
-_EVENT_ID = multiprocessing.Value("i", 1)
+# An integer id that is unique across this invocation of the program, to be set
+# by the first Add event. We can't set it here since it results in leaked
+# resources (see: https://issues.gerritcodereview.com/353656374).
+_EVENT_ID = None
 
 
 def _NextEventId():
@@ -178,6 +180,12 @@ def _NextEventId():
     Returns:
         A unique, to this invocation of the program, integer id.
     """
+    global _EVENT_ID
+    if _EVENT_ID is None:
+        # There is a small chance of race condition - two parallel processes
+        # setting up _EVENT_ID. However, we expect TASK_COMMAND to happen before
+        # mp kicks in.
+        _EVENT_ID = multiprocessing.Value("i", 1)
     with _EVENT_ID.get_lock():
         val = _EVENT_ID.value
         _EVENT_ID.value += 1

git_config.py

@@ -90,6 +90,20 @@ class GitConfig:
 
     @staticmethod
     def _getUserConfig():
+        """Get the user-specific config file.
+
+        Prefers the XDG config location if available, with fallback to
+        ~/.gitconfig
+
+        This matches git behavior:
+        https://git-scm.com/docs/git-config#FILES
+        """
+        xdg_config_home = os.getenv(
+            "XDG_CONFIG_HOME", os.path.expanduser("~/.config")
+        )
+        xdg_config_file = os.path.join(xdg_config_home, "git", "config")
+        if os.path.exists(xdg_config_file):
+            return xdg_config_file
         return os.path.expanduser("~/.gitconfig")
 
     @classmethod

git_trace2_event_log_base.py

@@ -130,10 +130,10 @@ class BaseEventLog:
             "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         }
 
-    def StartEvent(self):
+    def StartEvent(self, argv):
         """Append a 'start' event to the current log."""
         start_event = self._CreateEventDict("start")
-        start_event["argv"] = sys.argv
+        start_event["argv"] = argv
         self._log.append(start_event)
 
     def ExitEvent(self, result):
@@ -159,9 +159,11 @@ class BaseEventLog:
             name: Name of the primary command (ex: repo, git)
             subcommands: List of the sub-commands (ex: version, init, sync)
         """
-        command_event = self._CreateEventDict("command")
+        command_event = self._CreateEventDict("cmd_name")
+        name = f"{name}-"
+        name += "-".join(subcommands)
         command_event["name"] = name
-        command_event["subcommands"] = subcommands
+        command_event["hierarchy"] = name
         self._log.append(command_event)
 
     def LogConfigEvents(self, config, event_dict_name):

main.py

@@ -45,7 +45,6 @@ from command import InteractiveCommand
 from command import MirrorSafeCommand
 from editor import Editor
 from error import DownloadError
-from error import GitcUnsupportedError
 from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
@@ -308,10 +307,6 @@ class _Repo:
                 outer_client=outer_client,
             )
 
-        if Wrapper().gitc_parse_clientdir(os.getcwd()):
-            logger.error("GITC is not supported.")
-            raise GitcUnsupportedError()
-
         try:
             cmd = self.commands[name](
                 repodir=self.repodir,
@@ -357,7 +352,7 @@ class _Repo:
         start = time.time()
         cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
         cmd.event_log.SetParent(cmd_event)
-        git_trace2_event_log.StartEvent()
+        git_trace2_event_log.StartEvent(["repo", name] + argv)
         git_trace2_event_log.CommandEvent(name="repo", subcommands=[name])
 
         def execute_command_helper():

man/repo-gc.1

@@ -0,0 +1,43 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "December 2024" "repo gc" "Repo Manual"
+.SH NAME
+repo \- repo gc - manual page for repo gc
+.SH SYNOPSIS
+.B repo
+\fI\,gc\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Cleaning up internal repo state.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-n\fR, \fB\-\-dry\-run\fR
+do everything except actually delete
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help gc` to view the detailed manual.

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "April 2024" "repo manifest" "Repo Manual"
+.TH REPO "1" "December 2024" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -192,11 +192,13 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
 <!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
 <!ATTLIST extend\-project upstream CDATA #IMPLIED>
+<!ATTLIST extend\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name     CDATA #IMPLIED>
 <!ATTLIST remove\-project path     CDATA #IMPLIED>
 <!ATTLIST remove\-project optional CDATA #IMPLIED>
+<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT repo\-hooks EMPTY>
 <!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
@@ -495,6 +497,14 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project. Same syntax as the corresponding element of `project`.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+extended. Manifest parse will fail and give a list of mismatch extends if the
+revisions being extended have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -556,6 +566,14 @@ Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove\-project elements with no
 matching `project` element.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+removed. Manifest parse will fail and give a list of mismatch removes if the
+revisions being removed have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element repo\-hooks
 .PP
 NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "April 2024" "repo" "Repo Manual"
+.TH REPO "1" "December 2024" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -79,6 +79,9 @@ Download and checkout a change
 forall
 Run a shell command in each project
 .TP
+gc
+Cleaning up internal repo state.
+.TP
 grep
 Print lines matching a pattern
 .TP

manifest_xml.py

@@ -1445,6 +1445,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
         repo_hooks_project = None
         enabled_repo_hooks = None
+        failed_revision_changes = []
         for node in itertools.chain(*node_list):
             if node.nodeName == "project":
                 project = self._ParseProject(node)
@@ -1471,6 +1472,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     remote = self._get_remote(node)
                 dest_branch = node.getAttribute("dest-branch")
                 upstream = node.getAttribute("upstream")
+                base_revision = node.getAttribute("base-rev")
 
                 named_projects = self._projects[name]
                 if dest_path and not path and len(named_projects) > 1:
@@ -1484,6 +1486,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     if groups:
                         p.groups.extend(groups)
                     if revision:
+                        if base_revision:
+                            if p.revisionExpr != base_revision:
+                                failed_revision_changes.append(
+                                    "extend-project name %s mismatch base "
+                                    "%s vs revision %s"
+                                    % (name, base_revision, p.revisionExpr)
+                                )
                         p.SetRevision(revision)
 
                     if remote_name:
@@ -1558,6 +1567,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             if node.nodeName == "remove-project":
                 name = node.getAttribute("name")
                 path = node.getAttribute("path")
+                base_revision = node.getAttribute("base-rev")
 
                 # Name or path needed.
                 if not name and not path:
@@ -1571,6 +1581,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                 for projname, projects in list(self._projects.items()):
                     for p in projects:
                         if name == projname and not path:
+                            if base_revision:
+                                if p.revisionExpr != base_revision:
+                                    failed_revision_changes.append(
+                                        "remove-project name %s mismatch base "
+                                        "%s vs revision %s"
+                                        % (name, base_revision, p.revisionExpr)
+                                    )
                             del self._paths[p.relpath]
                             if not removed_project:
                                 del self._projects[name]
@@ -1578,6 +1595,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         elif path == p.relpath and (
                             name == projname or not name
                         ):
+                            if base_revision:
+                                if p.revisionExpr != base_revision:
+                                    failed_revision_changes.append(
+                                        "remove-project path %s mismatch base "
+                                        "%s vs revision %s"
+                                        % (
+                                            p.relpath,
+                                            base_revision,
+                                            p.revisionExpr,
+                                        )
+                                    )
                             self._projects[projname].remove(p)
                             del self._paths[p.relpath]
                             removed_project = p.name
@@ -1597,6 +1625,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         "project: %s" % node.toxml()
                     )
 
+        if failed_revision_changes:
+            raise ManifestParseError(
+                "revision base check failed, rebase patches and update "
+                "base revs for: ",
+                failed_revision_changes,
+            )
+
         # Store repo hooks project information.
         if repo_hooks_project:
             # Store a reference to the Project.
@@ -2021,7 +2056,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         path = path.rstrip("/")
         name = name.rstrip("/")
         relpath = self._JoinRelpath(parent.relpath, path)
-        gitdir = os.path.join(parent.gitdir, "subprojects", "%s.git" % path)
+        subprojects = os.path.join(parent.gitdir, "subprojects", f"{path}.git")
+        modules = os.path.join(parent.gitdir, "modules", path)
+        if platform_utils.isdir(subprojects):
+            gitdir = subprojects
+        else:
+            gitdir = modules
         objdir = os.path.join(
             parent.gitdir, "subproject-objects", "%s.git" % name
         )

progress.py

@@ -100,6 +100,7 @@ class Progress:
         self._show = not delay
         self._units = units
         self._elide = elide and _TTY
+        self._quiet = quiet
 
         # Only show the active jobs section if we run more than one in parallel.
         self._show_jobs = False
@@ -114,13 +115,7 @@ class Progress:
         )
         self._update_thread.daemon = True
 
-        # When quiet, never show any output.  It's a bit hacky, but reusing the
-        # existing logic that delays initial output keeps the rest of the class
-        # clean.  Basically we set the start time to years in the future.
-        if quiet:
-            self._show = False
-            self._start += 2**32
-        elif show_elapsed:
+        if not quiet and show_elapsed:
             self._update_thread.start()
 
     def _update_loop(self):
@@ -160,7 +155,7 @@ class Progress:
             msg = self._last_msg
         self._last_msg = msg
 
-        if not _TTY or IsTraceToStderr():
+        if not _TTY or IsTraceToStderr() or self._quiet:
             return
 
         elapsed_sec = time.time() - self._start
@@ -202,7 +197,7 @@ class Progress:
 
     def end(self):
         self._update_event.set()
-        if not _TTY or IsTraceToStderr() or not self._show:
+        if not _TTY or IsTraceToStderr() or self._quiet:
             return
 
         duration = duration_str(time.time() - self._start)

project.py

@@ -576,7 +576,6 @@ class Project:
         dest_branch=None,
         optimized_fetch=False,
         retry_fetches=0,
-        old_revision=None,
     ):
         """Init a Project object.
 
@@ -609,7 +608,6 @@ class Project:
                 only fetch from the remote if the sha1 is not present locally.
             retry_fetches: Retry remote fetches n times upon receiving transient
                 error with exponential backoff and jitter.
-            old_revision: saved git commit id for open GITC projects.
         """
         self.client = self.manifest = manifest
         self.name = name
@@ -639,12 +637,15 @@ class Project:
         self.linkfiles = []
         self.annotations = []
         self.dest_branch = dest_branch
-        self.old_revision = old_revision
 
         # This will be filled in if a project is later identified to be the
         # project containing repo hooks.
         self.enabled_repo_hooks = []
 
+        # This will be updated later if the project has submodules and
+        # if they will be synced.
+        self.has_subprojects = False
+
     def RelPath(self, local=True):
         """Return the path for the project relative to a manifest.
 
@@ -1563,6 +1564,11 @@ class Project:
             return
 
         self._InitWorkTree(force_sync=force_sync, submodules=submodules)
+        # TODO(https://git-scm.com/docs/git-worktree#_bugs): Re-evaluate if
+        # submodules can be init when using worktrees once its support is
+        # complete.
+        if self.has_subprojects and not self.use_git_worktrees:
+            self._InitSubmodules()
         all_refs = self.bare_ref.all
         self.CleanPublishedCache(all_refs)
         revid = self.GetRevisionId(all_refs)
@@ -2191,24 +2197,27 @@ class Project:
 
         def get_submodules(gitdir, rev):
             # Parse .gitmodules for submodule sub_paths and sub_urls.
-            sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
+            sub_paths, sub_urls, sub_shallows = parse_gitmodules(gitdir, rev)
             if not sub_paths:
                 return []
             # Run `git ls-tree` to read SHAs of submodule object, which happen
             # to be revision of submodule repository.
             sub_revs = git_ls_tree(gitdir, rev, sub_paths)
             submodules = []
-            for sub_path, sub_url in zip(sub_paths, sub_urls):
+            for sub_path, sub_url, sub_shallow in zip(
+                sub_paths, sub_urls, sub_shallows
+            ):
                 try:
                     sub_rev = sub_revs[sub_path]
                 except KeyError:
                     # Ignore non-exist submodules.
                     continue
-                submodules.append((sub_rev, sub_path, sub_url))
+                submodules.append((sub_rev, sub_path, sub_url, sub_shallow))
             return submodules
 
         re_path = re.compile(r"^submodule\.(.+)\.path=(.*)$")
         re_url = re.compile(r"^submodule\.(.+)\.url=(.*)$")
+        re_shallow = re.compile(r"^submodule\.(.+)\.shallow=(.*)$")
 
         def parse_gitmodules(gitdir, rev):
             cmd = ["cat-file", "blob", "%s:.gitmodules" % rev]
@@ -2222,9 +2231,9 @@ class Project:
                     gitdir=gitdir,
                 )
             except GitError:
-                return [], []
+                return [], [], []
             if p.Wait() != 0:
-                return [], []
+                return [], [], []
 
             gitmodules_lines = []
             fd, temp_gitmodules_path = tempfile.mkstemp()
@@ -2241,16 +2250,17 @@ class Project:
                     gitdir=gitdir,
                 )
                 if p.Wait() != 0:
-                    return [], []
+                    return [], [], []
                 gitmodules_lines = p.stdout.split("\n")
             except GitError:
-                return [], []
+                return [], [], []
             finally:
                 platform_utils.remove(temp_gitmodules_path)
 
             names = set()
             paths = {}
             urls = {}
+            shallows = {}
             for line in gitmodules_lines:
                 if not line:
                     continue
@@ -2264,10 +2274,16 @@ class Project:
                     names.add(m.group(1))
                     urls[m.group(1)] = m.group(2)
                     continue
+                m = re_shallow.match(line)
+                if m:
+                    names.add(m.group(1))
+                    shallows[m.group(1)] = m.group(2)
+                    continue
             names = sorted(names)
             return (
                 [paths.get(name, "") for name in names],
                 [urls.get(name, "") for name in names],
+                [shallows.get(name, "") for name in names],
             )
 
         def git_ls_tree(gitdir, rev, paths):
@@ -2296,7 +2312,9 @@ class Project:
 
         try:
             rev = self.GetRevisionId()
-        except GitError:
+        except (GitError, ManifestInvalidRevisionError):
+            # The git repo may be outdated (i.e. not fetched yet) and querying
+            # its submodules using the revision may not work; so return here.
             return []
         return get_submodules(self.gitdir, rev)
 
@@ -2306,7 +2324,7 @@ class Project:
             # If git repo does not exist yet, querying its submodules will
             # mess up its states; so return here.
             return result
-        for rev, path, url in self._GetSubmodules():
+        for rev, path, url, shallow in self._GetSubmodules():
             name = self.manifest.GetSubprojectName(self, path)
             (
                 relpath,
@@ -2328,6 +2346,7 @@ class Project:
                 review=self.remote.review,
                 revision=self.remote.revision,
             )
+            clone_depth = 1 if shallow.lower() == "true" else None
             subproject = Project(
                 manifest=self.manifest,
                 name=name,
@@ -2344,10 +2363,13 @@ class Project:
                 sync_s=self.sync_s,
                 sync_tags=self.sync_tags,
                 parent=self,
+                clone_depth=clone_depth,
                 is_derived=True,
             )
             result.append(subproject)
             result.extend(subproject.GetDerivedSubprojects())
+        if result:
+            self.has_subprojects = True
         return result
 
     def EnableRepositoryExtension(self, key, value="true", version=1):
@@ -2998,6 +3020,17 @@ class Project:
                 project=self.name,
             )
 
+    def _InitSubmodules(self, quiet=True):
+        """Initialize the submodules for the project."""
+        cmd = ["submodule", "init"]
+        if quiet:
+            cmd.append("-q")
+        if GitCommand(self, cmd).Wait() != 0:
+            raise GitError(
+                f"{self.name} submodule init",
+                project=self.name,
+            )
+
     def _Rebase(self, upstream, onto=None):
         cmd = ["rebase"]
         if onto is not None:
@@ -3373,24 +3406,29 @@ class Project:
             setting = fp.read()
             assert setting.startswith("gitdir:")
             git_worktree_path = setting.split(":", 1)[1].strip()
-        # Some platforms (e.g. Windows) won't let us update dotgit in situ
-        # because of file permissions.  Delete it and recreate it from scratch
-        # to avoid.
-        platform_utils.remove(dotgit)
-        # Use relative path from checkout->worktree & maintain Unix line endings
-        # on all OS's to match git behavior.
-        with open(dotgit, "w", newline="\n") as fp:
-            print(
-                "gitdir:",
-                os.path.relpath(git_worktree_path, self.worktree),
-                file=fp,
-            )
-        # Use relative path from worktree->checkout & maintain Unix line endings
-        # on all OS's to match git behavior.
-        with open(
-            os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
-        ) as fp:
-            print(os.path.relpath(dotgit, git_worktree_path), file=fp)
+
+        # `gitdir` maybe be either relative or absolute depending on the
+        # behavior of the local copy of git, so only convert the path to
+        # relative if it needs to be converted.
+        if os.path.isabs(git_worktree_path):
+            # Some platforms (e.g. Windows) won't let us update dotgit in situ
+            # because of file permissions.  Delete it and recreate it from
+            # scratch to avoid.
+            platform_utils.remove(dotgit)
+            # Use relative path from checkout->worktree & maintain Unix line
+            # endings on all OS's to match git behavior.
+            with open(dotgit, "w", newline="\n") as fp:
+                print(
+                    "gitdir:",
+                    os.path.relpath(git_worktree_path, self.worktree),
+                    file=fp,
+                )
+            # Use relative path from worktree->checkout & maintain Unix line
+            # endings on all OS's to match git behavior.
+            with open(
+                os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
+            ) as fp:
+                print(os.path.relpath(dotgit, git_worktree_path), file=fp)
 
         self._InitMRef()
 
@@ -3411,6 +3449,11 @@ class Project:
         """
         dotgit = os.path.join(self.worktree, ".git")
 
+        # If bare checkout of the submodule is stored under the subproject dir,
+        # migrate it.
+        if self.parent:
+            self._MigrateOldSubmoduleDir()
+
         # If using an old layout style (a directory), migrate it.
         if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
             self._MigrateOldWorkTreeGitDir(dotgit, project=self.name)
@@ -3421,34 +3464,76 @@ class Project:
                 self._InitGitWorktree()
                 self._CopyAndLinkFiles()
         else:
+            # Remove old directory symbolic links for submodules.
+            if self.parent and platform_utils.islink(dotgit):
+                platform_utils.remove(dotgit)
+                init_dotgit = True
+
             if not init_dotgit:
                 # See if the project has changed.
-                if os.path.realpath(self.gitdir) != os.path.realpath(dotgit):
-                    platform_utils.remove(dotgit)
+                self._removeBadGitDirLink(dotgit)
 
             if init_dotgit or not os.path.exists(dotgit):
-                os.makedirs(self.worktree, exist_ok=True)
-                platform_utils.symlink(
-                    os.path.relpath(self.gitdir, self.worktree), dotgit
-                )
+                self._createDotGit(dotgit)
 
             if init_dotgit:
                 _lwrite(
-                    os.path.join(dotgit, HEAD), "%s\n" % self.GetRevisionId()
+                    os.path.join(self.gitdir, HEAD), f"{self.GetRevisionId()}\n"
                 )
 
                 # Finish checking out the worktree.
                 cmd = ["read-tree", "--reset", "-u", "-v", HEAD]
-                if GitCommand(self, cmd).Wait() != 0:
-                    raise GitError(
-                        "Cannot initialize work tree for " + self.name,
-                        project=self.name,
-                    )
+                try:
+                    if GitCommand(self, cmd).Wait() != 0:
+                        raise GitError(
+                            "Cannot initialize work tree for " + self.name,
+                            project=self.name,
+                        )
+                except Exception as e:
+                    # Something went wrong with read-tree (perhaps fetching
+                    # missing blobs), so remove .git to avoid half initialized
+                    # workspace from which repo can't recover on its own.
+                    platform_utils.remove(dotgit)
+                    raise e
 
                 if submodules:
                     self._SyncSubmodules(quiet=True)
                 self._CopyAndLinkFiles()
 
+    def _createDotGit(self, dotgit):
+        """Initialize .git path.
+
+        For submodule projects, create a '.git' file using the gitfile
+        mechanism, and for the rest, create a symbolic link.
+        """
+        os.makedirs(self.worktree, exist_ok=True)
+        if self.parent:
+            _lwrite(
+                dotgit,
+                f"gitdir: {os.path.relpath(self.gitdir, self.worktree)}\n",
+            )
+        else:
+            platform_utils.symlink(
+                os.path.relpath(self.gitdir, self.worktree), dotgit
+            )
+
+    def _removeBadGitDirLink(self, dotgit):
+        """Verify .git is initialized correctly, otherwise delete it."""
+        if self.parent and os.path.isfile(dotgit):
+            with open(dotgit) as fp:
+                setting = fp.read()
+            if not setting.startswith("gitdir:"):
+                raise GitError(
+                    f"'.git' in {self.worktree} must start with 'gitdir:'",
+                    project=self.name,
+                )
+            gitdir = setting.split(":", 1)[1].strip()
+            dotgit_path = os.path.normpath(os.path.join(self.worktree, gitdir))
+        else:
+            dotgit_path = os.path.realpath(dotgit)
+        if os.path.realpath(self.gitdir) != dotgit_path:
+            platform_utils.remove(dotgit)
+
     @classmethod
     def _MigrateOldWorkTreeGitDir(cls, dotgit, project=None):
         """Migrate the old worktree .git/ dir style to a symlink.
@@ -3537,6 +3622,28 @@ class Project:
             dotgit,
         )
 
+    def _MigrateOldSubmoduleDir(self):
+        """Move the old bare checkout in 'subprojects' to 'modules'
+        as bare checkouts of submodules are now in 'modules' dir.
+        """
+        subprojects = os.path.join(self.parent.gitdir, "subprojects")
+        if not platform_utils.isdir(subprojects):
+            return
+
+        modules = os.path.join(self.parent.gitdir, "modules")
+        old = self.gitdir
+        new = os.path.splitext(self.gitdir.replace(subprojects, modules))[0]
+
+        if all(map(platform_utils.isdir, [old, new])):
+            platform_utils.rmtree(old, ignore_errors=True)
+        else:
+            os.makedirs(modules, exist_ok=True)
+            platform_utils.rename(old, new)
+        self.gitdir = new
+        self.UpdatePaths(self.relpath, self.worktree, self.gitdir, self.objdir)
+        if platform_utils.isdir(subprojects) and not os.listdir(subprojects):
+            platform_utils.rmtree(subprojects, ignore_errors=True)
+
     def _get_symlink_error_message(self):
         if platform_utils.isWindows():
             return (

repo

@@ -124,7 +124,7 @@ if not REPO_REV:
 BUG_URL = "https://issues.gerritcodereview.com/issues/new?component=1370071"
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 48)
+VERSION = (2, 50)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -215,8 +215,6 @@ repodir = ".repo"  # name of repo's private directory
 S_repo = "repo"  # special repo repository
 S_manifests = "manifests"  # special manifest repository
 REPO_MAIN = S_repo + "/main.py"  # main script
-GITC_CONFIG_FILE = "/gitc/.config"
-GITC_FS_ROOT_DIR = "/gitc/manifest-rw/"
 
 
 import collections
@@ -235,12 +233,9 @@ home_dot_repo = os.path.join(repo_config_dir, ".repoconfig")
 gpg_dir = os.path.join(home_dot_repo, "gnupg")
 
 
-def GetParser(gitc_init=False):
+def GetParser():
     """Setup the CLI parser."""
-    if gitc_init:
-        sys.exit("repo: fatal: GITC not supported.")
-    else:
-        usage = "repo init [options] [-u] url"
+    usage = "repo init [options] [-u] url"
 
     parser = optparse.OptionParser(usage=usage)
     InitParser(parser)
@@ -557,49 +552,6 @@ def run_command(cmd, **kwargs):
     return ret
 
 
-_gitc_manifest_dir = None
-
-
-def get_gitc_manifest_dir():
-    global _gitc_manifest_dir
-    if _gitc_manifest_dir is None:
-        _gitc_manifest_dir = ""
-        try:
-            with open(GITC_CONFIG_FILE) as gitc_config:
-                for line in gitc_config:
-                    match = re.match("gitc_dir=(?P<gitc_manifest_dir>.*)", line)
-                    if match:
-                        _gitc_manifest_dir = match.group("gitc_manifest_dir")
-        except OSError:
-            pass
-    return _gitc_manifest_dir
-
-
-def gitc_parse_clientdir(gitc_fs_path):
-    """Parse a path in the GITC FS and return its client name.
-
-    Args:
-        gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
-
-    Returns:
-        The GITC client name.
-    """
-    if gitc_fs_path == GITC_FS_ROOT_DIR:
-        return None
-    if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
-        manifest_dir = get_gitc_manifest_dir()
-        if manifest_dir == "":
-            return None
-        if manifest_dir[-1] != "/":
-            manifest_dir += "/"
-        if gitc_fs_path == manifest_dir:
-            return None
-        if not gitc_fs_path.startswith(manifest_dir):
-            return None
-        return gitc_fs_path.split(manifest_dir)[1].split("/")[0]
-    return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split("/")[0]
-
-
 class CloneFailure(Exception):
 
     """Indicate the remote clone of repo itself failed."""
@@ -638,9 +590,9 @@ def check_repo_rev(dst, rev, repo_verify=True, quiet=False):
     return (remote_ref, rev)
 
 
-def _Init(args, gitc_init=False):
+def _Init(args):
     """Installs repo by cloning it over the network."""
-    parser = GetParser(gitc_init=gitc_init)
+    parser = GetParser()
     opt, args = parser.parse_args(args)
     if args:
         if not opt.manifest_url:
@@ -1164,7 +1116,7 @@ class _Options:
 def _ExpandAlias(name):
     """Look up user registered aliases."""
     # We don't resolve aliases for existing subcommands.  This matches git.
-    if name in {"gitc-init", "help", "init"}:
+    if name in {"help", "init"}:
         return name, []
 
     alias = _GetRepoConfig(f"alias.{name}")
@@ -1292,10 +1244,6 @@ class Requirements:
 
 
 def _Usage():
-    gitc_usage = ""
-    if get_gitc_manifest_dir():
-        gitc_usage = "  gitc-init Initialize a GITC Client.\n"
-
     print(
         """usage: repo COMMAND [ARGS]
 
@@ -1304,9 +1252,7 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-"""
-        + gitc_usage
-        + """  help      Display detailed help on a command
+  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """
@@ -1317,8 +1263,8 @@ For access to the full online help, install repo ("repo init").
 
 def _Help(args):
     if args:
-        if args[0] in {"init", "gitc-init"}:
-            parser = GetParser(gitc_init=args[0] == "gitc-init")
+        if args[0] in {"init"}:
+            parser = GetParser()
             parser.print_help()
             sys.exit(0)
         else:
@@ -1335,10 +1281,11 @@ def _Help(args):
 
 def _Version():
     """Show version information."""
+    git_version = ParseGitVersion()
     print("<repo not installed>")
     print(f"repo launcher version {'.'.join(str(x) for x in VERSION)}")
     print(f"       (from {__file__})")
-    print(f"git {ParseGitVersion().full}")
+    print(f"git {git_version.full}" if git_version else "git not installed")
     print(f"Python {sys.version}")
     uname = platform.uname()
     print(f"OS {uname.system} {uname.release} ({uname.version})")
@@ -1371,11 +1318,11 @@ def _RunSelf(wrapper_path):
     my_main = os.path.join(my_dir, "main.py")
     my_git = os.path.join(my_dir, ".git")
 
-    if os.path.isfile(my_main) and os.path.isdir(my_git):
+    if os.path.isfile(my_main):
         for name in ["git_config.py", "project.py", "subcmds"]:
             if not os.path.exists(os.path.join(my_dir, name)):
                 return None, None
-        return my_main, my_git
+        return my_main, my_git if os.path.isdir(my_git) else None
     return None, None
 
 
@@ -1406,23 +1353,11 @@ def main(orig_args):
     # We run this early as we run some git commands ourselves.
     SetGitTrace2ParentSid()
 
-    repo_main, rel_repo_dir = None, None
-    # Don't use the local repo copy, make sure to switch to the gitc client first.
-    if cmd != "gitc-init":
-        repo_main, rel_repo_dir = _FindRepo()
+    repo_main, rel_repo_dir = _FindRepo()
 
     wrapper_path = os.path.abspath(__file__)
     my_main, my_git = _RunSelf(wrapper_path)
 
-    cwd = os.getcwd()
-    if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-        print(
-            "error: repo cannot be used in the GITC local manifest directory."
-            "\nIf you want to work on this GITC client please rerun this "
-            "command from the corresponding client under /gitc/",
-            file=sys.stderr,
-        )
-        sys.exit(1)
     if not repo_main:
         # Only expand aliases here since we'll be parsing the CLI ourselves.
         # If we had repo_main, alias expansion would happen in main.py.
@@ -1437,11 +1372,11 @@ def main(orig_args):
             _Version()
         if not cmd:
             _NotInstalled()
-        if cmd == "init" or cmd == "gitc-init":
+        if cmd == "init":
             if my_git:
                 _SetDefaultsTo(my_git)
             try:
-                _Init(args, gitc_init=(cmd == "gitc-init"))
+                _Init(args)
             except CloneFailure:
                 path = os.path.join(repodir, S_repo)
                 print(

run_tests.vpython3

@@ -5,61 +5,55 @@
 # List of available wheels:
 #   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
 
-python_version: "3.8"
+python_version: "3.11"
 
 wheel: <
   name: "infra/python/wheels/pytest-py3"
-  version: "version:6.2.2"
+  version: "version:8.3.4"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/py-py2_py3"
-  version: "version:1.10.0"
+  version: "version:1.11.0"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/iniconfig-py3"
   version: "version:1.1.1"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/packaging-py3"
   version: "version:23.0"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/pluggy-py3"
-  version: "version:0.13.1"
+  version: "version:1.5.0"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/toml-py3"
   version: "version:0.10.1"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/pyparsing-py3"
   version: "version:3.0.7"
 >
 
-# Required by pytest==6.2.2
+# Required by pytest==8.3.4
 wheel: <
   name: "infra/python/wheels/attrs-py2_py3"
   version: "version:21.4.0"
 >
 
-# Required by packaging==16.8
-wheel: <
-  name: "infra/python/wheels/six-py2_py3"
-  version: "version:1.16.0"
->
-
 wheel: <
   name: "infra/python/wheels/black-py3"
   version: "version:23.1.0"

subcmds/abandon.py

@@ -70,8 +70,10 @@ It is equivalent to "git branch -D <branchname>".
         else:
             args.insert(0, "'All local branches'")
 
-    def _ExecuteOne(self, all_branches, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, all_branches, nb, project_idx):
         """Abandon one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         if all_branches:
             branches = project.GetBranches()
         else:
@@ -89,7 +91,7 @@ It is equivalent to "git branch -D <branchname>".
             if status is not None:
                 ret[name] = status
 
-        return (ret, project, errors)
+        return (ret, project_idx, errors)
 
     def Execute(self, opt, args):
         nb = args[0].split()
@@ -102,7 +104,8 @@ It is equivalent to "git branch -D <branchname>".
         _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
         def _ProcessResults(_pool, pm, states):
-            for results, project, errors in states:
+            for results, project_idx, errors in states:
+                project = all_projects[project_idx]
                 for branch, status in results.items():
                     if status:
                         success[branch].append(project)
@@ -111,15 +114,18 @@ It is equivalent to "git branch -D <branchname>".
                 aggregate_errors.extend(errors)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, opt.all, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                f"Abandon {nb}", len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, opt.all, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Abandon {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
 
         width = max(
             itertools.chain(

subcmds/branches.py

@@ -98,6 +98,22 @@ is shown, then the branch appears in all projects.
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
+    @classmethod
+    def _ExpandProjectToBranches(cls, project_idx):
+        """Expands a project into a list of branch names & associated info.
+
+        Args:
+            project_idx: project.Project index
+
+        Returns:
+            List[Tuple[str, git_config.Branch, int]]
+        """
+        branches = []
+        project = cls.get_parallel_context()["projects"][project_idx]
+        for name, b in project.GetBranches().items():
+            branches.append((name, b, project_idx))
+        return branches
+
     def Execute(self, opt, args):
         projects = self.GetProjects(
             args, all_manifests=not opt.this_manifest_only
@@ -107,17 +123,20 @@ is shown, then the branch appears in all projects.
         project_cnt = len(projects)
 
         def _ProcessResults(_pool, _output, results):
-            for name, b in itertools.chain.from_iterable(results):
+            for name, b, project_idx in itertools.chain.from_iterable(results):
+                b.project = projects[project_idx]
                 if name not in all_branches:
                     all_branches[name] = BranchInfo(name)
                 all_branches[name].add(b)
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            expand_project_to_branches,
-            projects,
-            callback=_ProcessResults,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                self._ExpandProjectToBranches,
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
 
         names = sorted(all_branches)
 
@@ -148,7 +167,10 @@ is shown, then the branch appears in all projects.
             else:
                 published = " "
 
-            hdr("%c%c %-*s" % (current, published, width, name))
+            # A branch name can contain a percent sign, so we need to escape it.
+            # Escape after f-string formatting to properly account for leading
+            # spaces.
+            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
             out.write(" |")
 
             _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
@@ -191,19 +213,3 @@ is shown, then the branch appears in all projects.
             else:
                 out.write(" in all projects")
             out.nl()
-
-
-def expand_project_to_branches(project):
-    """Expands a project into a list of branch names & associated information.
-
-    Args:
-        project: project.Project
-
-    Returns:
-        List[Tuple[str, git_config.Branch]]
-    """
-    branches = []
-    for name, b in project.GetBranches().items():
-        b.project = project
-        branches.append((name, b))
-    return branches

subcmds/checkout.py

@@ -20,7 +20,6 @@ from command import DEFAULT_LOCAL_JOBS
 from error import GitError
 from error import RepoExitError
 from progress import Progress
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -30,7 +29,7 @@ logger = RepoLogger(__file__)
 class CheckoutBranchResult(NamedTuple):
     # Whether the Project is on the branch (i.e. branch exists and no errors)
     result: bool
-    project: Project
+    project_idx: int
     error: Exception
 
 
@@ -62,15 +61,17 @@ The command is equivalent to:
         if not args:
             self.Usage()
 
-    def _ExecuteOne(self, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, nb, project_idx):
         """Checkout one project."""
         error = None
         result = None
+        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             result = project.CheckoutBranch(nb)
         except GitError as e:
             error = e
-        return CheckoutBranchResult(result, project, error)
+        return CheckoutBranchResult(result, project_idx, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -83,22 +84,25 @@ The command is equivalent to:
 
         def _ProcessResults(_pool, pm, results):
             for result in results:
+                project = all_projects[result.project_idx]
                 if result.error is not None:
                     err.append(result.error)
-                    err_projects.append(result.project)
+                    err_projects.append(project)
                 elif result.result:
-                    success.append(result.project)
+                    success.append(project)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                f"Checkout {nb}", len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Checkout {nb}", len(all_projects), quiet=opt.quiet
+                ),
+            )
 
         if err_projects:
             for p in err_projects:

subcmds/diff.py

@@ -40,7 +40,8 @@ to the Unix 'patch' command.
             help="paths are relative to the repository root",
         )
 
-    def _ExecuteOne(self, absolute, local, project):
+    @classmethod
+    def _ExecuteOne(cls, absolute, local, project_idx):
         """Obtains the diff for a specific project.
 
         Args:
@@ -48,12 +49,13 @@ to the Unix 'patch' command.
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project: Project to get status of.
+            project_idx: Project index to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
         return (ret, buf.getvalue())
 
@@ -71,12 +73,15 @@ to the Unix 'patch' command.
                     ret = 1
             return ret
 
-        return self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(
-                self._ExecuteOne, opt.absolute, opt.this_manifest_only
-            ),
-            all_projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            return self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne, opt.absolute, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )

subcmds/forall.py

@@ -15,7 +15,6 @@
 import errno
 import functools
 import io
-import multiprocessing
 import os
 import re
 import signal
@@ -26,7 +25,6 @@ from color import Coloring
 from command import Command
 from command import DEFAULT_LOCAL_JOBS
 from command import MirrorSafeCommand
-from command import WORKER_BATCH_SIZE
 from error import ManifestInvalidRevisionError
 from repo_logging import RepoLogger
 
@@ -241,7 +239,6 @@ without iterating through the remaining projects.
                     cmd.insert(cmd.index(cn) + 1, "--color")
 
         mirror = self.manifest.IsMirror
-        rc = 0
 
         smart_sync_manifest_name = "smart_sync_override.xml"
         smart_sync_manifest_path = os.path.join(
@@ -264,35 +261,44 @@ without iterating through the remaining projects.
 
         os.environ["REPO_COUNT"] = str(len(projects))
 
+        def _ProcessResults(_pool, _output, results):
+            rc = 0
+            first = True
+            for r, output in results:
+                if output:
+                    if first:
+                        first = False
+                    elif opt.project_header:
+                        print()
+                    # To simplify the DoWorkWrapper, take care of automatic
+                    # newlines.
+                    end = "\n"
+                    if output[-1] == "\n":
+                        end = ""
+                    print(output, end=end)
+                rc = rc or r
+                if r != 0 and opt.abort_on_errors:
+                    raise Exception("Aborting due to previous error")
+            return rc
+
         try:
             config = self.manifest.manifestProject.config
-            with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
-                results_it = pool.imap(
+            with self.ParallelContext():
+                self.get_parallel_context()["projects"] = projects
+                rc = self.ExecuteInParallel(
+                    opt.jobs,
                     functools.partial(
-                        DoWorkWrapper, mirror, opt, cmd, shell, config
+                        self.DoWorkWrapper, mirror, opt, cmd, shell, config
                     ),
-                    enumerate(projects),
-                    chunksize=WORKER_BATCH_SIZE,
+                    range(len(projects)),
+                    callback=_ProcessResults,
+                    ordered=True,
+                    initializer=self.InitWorker,
+                    chunksize=1,
                 )
-                first = True
-                for r, output in results_it:
-                    if output:
-                        if first:
-                            first = False
-                        elif opt.project_header:
-                            print()
-                        # To simplify the DoWorkWrapper, take care of automatic
-                        # newlines.
-                        end = "\n"
-                        if output[-1] == "\n":
-                            end = ""
-                        print(output, end=end)
-                    rc = rc or r
-                    if r != 0 and opt.abort_on_errors:
-                        raise Exception("Aborting due to previous error")
         except (KeyboardInterrupt, WorkerKeyboardInterrupt):
             # Catch KeyboardInterrupt raised inside and outside of workers
-            rc = rc or errno.EINTR
+            rc = errno.EINTR
         except Exception as e:
             # Catch any other exceptions raised
             logger.error(
@@ -300,35 +306,35 @@ without iterating through the remaining projects.
                 type(e).__name__,
                 e,
             )
-            rc = rc or getattr(e, "errno", 1)
+            rc = getattr(e, "errno", 1)
         if rc != 0:
             sys.exit(rc)
 
+    @classmethod
+    def InitWorker(cls):
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    @classmethod
+    def DoWorkWrapper(cls, mirror, opt, cmd, shell, config, project_idx):
+        """A wrapper around the DoWork() method.
+
+        Catch the KeyboardInterrupt exceptions here and re-raise them as a
+        different, ``Exception``-based exception to stop it flooding the console
+        with stacktraces and making the parent hang indefinitely.
+
+        """
+        project = cls.get_parallel_context()["projects"][project_idx]
+        try:
+            return DoWork(project, mirror, opt, cmd, shell, project_idx, config)
+        except KeyboardInterrupt:
+            print("%s: Worker interrupted" % project.name)
+            raise WorkerKeyboardInterrupt()
+
 
 class WorkerKeyboardInterrupt(Exception):
     """Keyboard interrupt exception for worker processes."""
 
 
-def InitWorker():
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-
-def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
-    """A wrapper around the DoWork() method.
-
-    Catch the KeyboardInterrupt exceptions here and re-raise them as a
-    different, ``Exception``-based exception to stop it flooding the console
-    with stacktraces and making the parent hang indefinitely.
-
-    """
-    cnt, project = args
-    try:
-        return DoWork(project, mirror, opt, cmd, shell, cnt, config)
-    except KeyboardInterrupt:
-        print("%s: Worker interrupted" % project.name)
-        raise WorkerKeyboardInterrupt()
-
-
 def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env = os.environ.copy()
 

subcmds/gc.py

@@ -0,0 +1,294 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import List, Set
+
+from command import Command
+from git_command import GitCommand
+import platform_utils
+from progress import Progress
+from project import Project
+
+
+class Gc(Command):
+    COMMON = True
+    helpSummary = "Cleaning up internal repo and Git state."
+    helpUsage = """
+%prog
+"""
+
+    def _Options(self, p):
+        p.add_option(
+            "-n",
+            "--dry-run",
+            dest="dryrun",
+            default=False,
+            action="store_true",
+            help="do everything except actually delete",
+        )
+        p.add_option(
+            "-y",
+            "--yes",
+            default=False,
+            action="store_true",
+            help="answer yes to all safe prompts",
+        )
+        p.add_option(
+            "--repack",
+            default=False,
+            action="store_true",
+            help="repack all projects that use partial clone with "
+            "filter=blob:none",
+        )
+
+    def _find_git_to_delete(
+        self, to_keep: Set[str], start_dir: str
+    ) -> Set[str]:
+        """Searches no longer needed ".git" directories.
+
+        Scans the file system starting from `start_dir` and removes all
+        directories that end with ".git" that are not in the `to_keep` set.
+        """
+        to_delete = set()
+        for root, dirs, _ in platform_utils.walk(start_dir):
+            for directory in dirs:
+                if not directory.endswith(".git"):
+                    continue
+
+                path = os.path.join(root, directory)
+                if path not in to_keep:
+                    to_delete.add(path)
+
+        return to_delete
+
+    def delete_unused_projects(self, projects: List[Project], opt):
+        print(f"Scanning filesystem under {self.repodir}...")
+
+        project_paths = set()
+        project_object_paths = set()
+
+        for project in projects:
+            project_paths.add(project.gitdir)
+            project_object_paths.add(project.objdir)
+
+        to_delete = self._find_git_to_delete(
+            project_paths, os.path.join(self.repodir, "projects")
+        )
+
+        to_delete.update(
+            self._find_git_to_delete(
+                project_object_paths,
+                os.path.join(self.repodir, "project-objects"),
+            )
+        )
+
+        if not to_delete:
+            print("Nothing to clean up.")
+            return 0
+
+        print("Identified the following projects are no longer used:")
+        print("\n".join(to_delete))
+        print("")
+        if not opt.yes:
+            print(
+                "If you proceed, any local commits in those projects will be "
+                "destroyed!"
+            )
+            ask = input("Proceed? [y/N] ")
+            if ask.lower() != "y":
+                return 1
+
+        pm = Progress(
+            "Deleting",
+            len(to_delete),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for path in to_delete:
+            if opt.dryrun:
+                print(f"\nWould have deleted ${path}")
+            else:
+                tmp_path = os.path.join(
+                    os.path.dirname(path),
+                    f"to_be_deleted_{os.path.basename(path)}",
+                )
+                platform_utils.rename(path, tmp_path)
+                platform_utils.rmtree(tmp_path)
+            pm.update(msg=path)
+        pm.end()
+
+        return 0
+
+    def _generate_promisor_files(self, pack_dir: str):
+        """Generates promisor files for all pack files in the given directory.
+
+        Promisor files are empty files with the same name as the corresponding
+        pack file but with the ".promisor" extension. They are used by Git.
+        """
+        for root, _, files in platform_utils.walk(pack_dir):
+            for file in files:
+                if not file.endswith(".pack"):
+                    continue
+                with open(os.path.join(root, f"{file[:-4]}promisor"), "w"):
+                    pass
+
+    def repack_projects(self, projects: List[Project], opt):
+        repack_projects = []
+        # Find all projects eligible for repacking:
+        # - can't be shared
+        # - have a specific fetch filter
+        for project in projects:
+            if project.config.GetBoolean("extensions.preciousObjects"):
+                continue
+            if not project.clone_depth:
+                continue
+            if project.manifest.CloneFilterForDepth != "blob:none":
+                continue
+
+            repack_projects.append(project)
+
+        if opt.dryrun:
+            print(f"Would have repacked {len(repack_projects)} projects.")
+            return 0
+
+        pm = Progress(
+            "Repacking (this will take a while)",
+            len(repack_projects),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for project in repack_projects:
+            pm.update(msg=f"{project.name}")
+
+            pack_dir = os.path.join(project.gitdir, "tmp_repo_repack")
+            if os.path.isdir(pack_dir):
+                platform_utils.rmtree(pack_dir)
+            os.mkdir(pack_dir)
+
+            # Prepare workspace for repacking - remove all unreachable refs and
+            # their objects.
+            GitCommand(
+                project,
+                ["reflog", "expire", "--expire-unreachable=all"],
+                verify_command=True,
+            ).Wait()
+            pm.update(msg=f"{project.name} | gc", inc=0)
+            GitCommand(
+                project,
+                ["gc"],
+                verify_command=True,
+            ).Wait()
+
+            # Get all objects that are reachable from the remote, and pack them.
+            pm.update(msg=f"{project.name} | generating list of objects", inc=0)
+            remote_objects_cmd = GitCommand(
+                project,
+                [
+                    "rev-list",
+                    "--objects",
+                    f"--remotes={project.remote.name}",
+                    "--filter=blob:none",
+                    "--tags",
+                ],
+                capture_stdout=True,
+                verify_command=True,
+            )
+
+            # Get all local objects and pack them.
+            local_head_objects_cmd = GitCommand(
+                project,
+                ["rev-list", "--objects", "HEAD^{tree}"],
+                capture_stdout=True,
+                verify_command=True,
+            )
+            local_objects_cmd = GitCommand(
+                project,
+                [
+                    "rev-list",
+                    "--objects",
+                    "--all",
+                    "--reflog",
+                    "--indexed-objects",
+                    "--not",
+                    f"--remotes={project.remote.name}",
+                    "--tags",
+                ],
+                capture_stdout=True,
+                verify_command=True,
+            )
+
+            remote_objects_cmd.Wait()
+
+            pm.update(msg=f"{project.name} | remote repack", inc=0)
+            GitCommand(
+                project,
+                ["pack-objects", os.path.join(pack_dir, "pack")],
+                input=remote_objects_cmd.stdout,
+                capture_stderr=True,
+                capture_stdout=True,
+                verify_command=True,
+            ).Wait()
+
+            # create promisor file for each pack file
+            self._generate_promisor_files(pack_dir)
+
+            local_head_objects_cmd.Wait()
+            local_objects_cmd.Wait()
+
+            pm.update(msg=f"{project.name} | local repack", inc=0)
+            GitCommand(
+                project,
+                ["pack-objects", os.path.join(pack_dir, "pack")],
+                input=local_head_objects_cmd.stdout + local_objects_cmd.stdout,
+                capture_stderr=True,
+                capture_stdout=True,
+                verify_command=True,
+            ).Wait()
+
+            # Swap the old pack directory with the new one.
+            platform_utils.rename(
+                os.path.join(project.objdir, "objects", "pack"),
+                os.path.join(project.objdir, "objects", "pack_old"),
+            )
+            platform_utils.rename(
+                pack_dir,
+                os.path.join(project.objdir, "objects", "pack"),
+            )
+            platform_utils.rmtree(
+                os.path.join(project.objdir, "objects", "pack_old")
+            )
+
+        pm.end()
+        return 0
+
+    def Execute(self, opt, args):
+        projects: List[Project] = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+
+        ret = self.delete_unused_projects(projects, opt)
+        if ret != 0:
+            return ret
+
+        if not opt.repack:
+            return
+
+        return self.repack_projects(projects, opt)

subcmds/grep.py

@@ -23,7 +23,6 @@ from error import GitError
 from error import InvalidArgumentsError
 from error import SilentRepoExitError
 from git_command import GitCommand
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -40,7 +39,7 @@ class GrepColoring(Coloring):
 class ExecuteOneResult(NamedTuple):
     """Result from an execute instance."""
 
-    project: Project
+    project_idx: int
     rc: int
     stdout: str
     stderr: str
@@ -262,8 +261,10 @@ contain a line that matches both expressions:
             help="Show only file names not containing matching lines",
         )
 
-    def _ExecuteOne(self, cmd_argv, project):
+    @classmethod
+    def _ExecuteOne(cls, cmd_argv, project_idx):
         """Process one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             p = GitCommand(
                 project,
@@ -274,7 +275,7 @@ contain a line that matches both expressions:
                 verify_command=True,
             )
         except GitError as e:
-            return ExecuteOneResult(project, -1, None, str(e), e)
+            return ExecuteOneResult(project_idx, -1, None, str(e), e)
 
         try:
             error = None
@@ -282,10 +283,12 @@ contain a line that matches both expressions:
         except GitError as e:
             rc = 1
             error = e
-        return ExecuteOneResult(project, rc, p.stdout, p.stderr, error)
+        return ExecuteOneResult(project_idx, rc, p.stdout, p.stderr, error)
 
     @staticmethod
-    def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
+    def _ProcessResults(
+        full_name, have_rev, opt, projects, _pool, out, results
+    ):
         git_failed = False
         bad_rev = False
         have_match = False
@@ -293,9 +296,10 @@ contain a line that matches both expressions:
         errors = []
 
         for result in results:
+            project = projects[result.project_idx]
             if result.rc < 0:
                 git_failed = True
-                out.project("--- project %s ---" % _RelPath(result.project))
+                out.project("--- project %s ---" % _RelPath(project))
                 out.nl()
                 out.fail("%s", result.stderr)
                 out.nl()
@@ -311,9 +315,7 @@ contain a line that matches both expressions:
                     ):
                         bad_rev = True
                     else:
-                        out.project(
-                            "--- project %s ---" % _RelPath(result.project)
-                        )
+                        out.project("--- project %s ---" % _RelPath(project))
                         out.nl()
                         out.fail("%s", result.stderr.strip())
                         out.nl()
@@ -331,13 +333,13 @@ contain a line that matches both expressions:
                     rev, line = line.split(":", 1)
                     out.write("%s", rev)
                     out.write(":")
-                    out.project(_RelPath(result.project))
+                    out.project(_RelPath(project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
             elif full_name:
                 for line in r:
-                    out.project(_RelPath(result.project))
+                    out.project(_RelPath(project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
@@ -381,16 +383,19 @@ contain a line that matches both expressions:
             cmd_argv.extend(opt.revision)
         cmd_argv.append("--")
 
-        git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, cmd_argv),
-            projects,
-            callback=functools.partial(
-                self._ProcessResults, full_name, have_rev, opt
-            ),
-            output=out,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, cmd_argv),
+                range(len(projects)),
+                callback=functools.partial(
+                    self._ProcessResults, full_name, have_rev, opt, projects
+                ),
+                output=out,
+                ordered=True,
+                chunksize=1,
+            )
 
         if git_failed:
             raise GrepCommandError(

subcmds/prune.py

@@ -27,8 +27,10 @@ class Prune(PagedCommand):
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-    def _ExecuteOne(self, project):
+    @classmethod
+    def _ExecuteOne(cls, project_idx):
         """Process one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         return project.PruneHeads()
 
     def Execute(self, opt, args):
@@ -41,13 +43,15 @@ class Prune(PagedCommand):
         def _ProcessResults(_pool, _output, results):
             return list(itertools.chain.from_iterable(results))
 
-        all_branches = self.ExecuteInParallel(
-            opt.jobs,
-            self._ExecuteOne,
-            projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            all_branches = self.ExecuteInParallel(
+                opt.jobs,
+                self._ExecuteOne,
+                range(len(projects)),
+                callback=_ProcessResults,
+                ordered=True,
+            )
 
         if not all_branches:
             return

subcmds/start.py

@@ -21,7 +21,6 @@ from error import RepoExitError
 from git_command import git
 from git_config import IsImmutable
 from progress import Progress
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -29,7 +28,7 @@ logger = RepoLogger(__file__)
 
 
 class ExecuteOneResult(NamedTuple):
-    project: Project
+    project_idx: int
     error: Exception
 
 
@@ -80,18 +79,20 @@ revision specified in the manifest.
         if not git.check_ref_format("heads/%s" % nb):
             self.OptionParser.error("'%s' is not a valid name" % nb)
 
-    def _ExecuteOne(self, revision, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, revision, nb, default_revisionExpr, project_idx):
         """Start one project."""
         # If the current revision is immutable, such as a SHA1, a tag or
         # a change, then we can't push back to it. Substitute with
         # dest_branch, if defined; or with manifest default revision instead.
         branch_merge = ""
         error = None
+        project = cls.get_parallel_context()["projects"][project_idx]
         if IsImmutable(project.revisionExpr):
             if project.dest_branch:
                 branch_merge = project.dest_branch
             else:
-                branch_merge = self.manifest.default.revisionExpr
+                branch_merge = default_revisionExpr
 
         try:
             project.StartBranch(
@@ -100,7 +101,7 @@ revision specified in the manifest.
         except Exception as e:
             logger.error("error: unable to checkout %s: %s", project.name, e)
             error = e
-        return ExecuteOneResult(project, error)
+        return ExecuteOneResult(project_idx, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -120,19 +121,28 @@ revision specified in the manifest.
         def _ProcessResults(_pool, pm, results):
             for result in results:
                 if result.error:
-                    err_projects.append(result.project)
+                    project = all_projects[result.project_idx]
+                    err_projects.append(project)
                     err.append(result.error)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, opt.revision, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                f"Starting {nb}", len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne,
+                    opt.revision,
+                    nb,
+                    self.manifest.default.revisionExpr,
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Starting {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
 
         if err_projects:
             for p in err_projects:

subcmds/status.py

@@ -88,7 +88,8 @@ the following meanings:
             "projects",
         )
 
-    def _StatusHelper(self, quiet, local, project):
+    @classmethod
+    def _StatusHelper(cls, quiet, local, project_idx):
         """Obtains the status for a specific project.
 
         Obtains the status for a project, redirecting the output to
@@ -99,12 +100,13 @@ the following meanings:
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project: Project to get status of.
+            project_idx: Project index to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeStatus(
             quiet=quiet, output_redir=buf, local=local
         )
@@ -143,15 +145,18 @@ the following meanings:
                     ret += 1
             return ret
 
-        counter = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(
-                self._StatusHelper, opt.quiet, opt.this_manifest_only
-            ),
-            all_projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            counter = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._StatusHelper, opt.quiet, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )
 
         if not opt.quiet and len(all_projects) == counter:
             print("nothing to commit (working directory clean)")

subcmds/sync.py

@@ -141,7 +141,7 @@ class _FetchOneResult(NamedTuple):
 
     Attributes:
       success (bool): True if successful.
-      project (Project): The fetched project.
+      project_idx (int): The fetched project index.
       start (float): The starting time.time().
       finish (float): The ending time.time().
       remote_fetched (bool): True if the remote was actually queried.
@@ -149,7 +149,7 @@ class _FetchOneResult(NamedTuple):
 
     success: bool
     errors: List[Exception]
-    project: Project
+    project_idx: int
     start: float
     finish: float
     remote_fetched: bool
@@ -182,14 +182,14 @@ class _CheckoutOneResult(NamedTuple):
 
     Attributes:
       success (bool): True if successful.
-      project (Project): The project.
+      project_idx (int): The project index.
       start (float): The starting time.time().
       finish (float): The ending time.time().
     """
 
     success: bool
     errors: List[Exception]
-    project: Project
+    project_idx: int
     start: float
     finish: float
 
@@ -592,7 +592,8 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS) :]
         return branch
 
-    def _GetCurrentBranchOnly(self, opt, manifest):
+    @classmethod
+    def _GetCurrentBranchOnly(cls, opt, manifest):
         """Returns whether current-branch or use-superproject options are
         enabled.
 
@@ -710,7 +711,8 @@ later is required to fix a server side protocol bug.
         if need_unload:
             m.outer_client.manifest.Unload()
 
-    def _FetchProjectList(self, opt, projects):
+    @classmethod
+    def _FetchProjectList(cls, opt, projects):
         """Main function of the fetch worker.
 
         The projects we're given share the same underlying git object store, so
@@ -722,21 +724,23 @@ later is required to fix a server side protocol bug.
             opt: Program options returned from optparse.  See _Options().
             projects: Projects to fetch.
         """
-        return [self._FetchOne(opt, x) for x in projects]
+        return [cls._FetchOne(opt, x) for x in projects]
 
-    def _FetchOne(self, opt, project):
+    @classmethod
+    def _FetchOne(cls, opt, project_idx):
         """Fetch git objects for a single project.
 
         Args:
             opt: Program options returned from optparse.  See _Options().
-            project: Project object for the project to fetch.
+            project_idx: Project index for the project to fetch.
 
         Returns:
             Whether the fetch was successful.
         """
+        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         k = f"{project.name} @ {project.relpath}"
-        self._sync_dict[k] = start
+        cls.get_parallel_context()["sync_dict"][k] = start
         success = False
         remote_fetched = False
         errors = []
@@ -746,7 +750,7 @@ later is required to fix a server side protocol bug.
                 quiet=opt.quiet,
                 verbose=opt.verbose,
                 output_redir=buf,
-                current_branch_only=self._GetCurrentBranchOnly(
+                current_branch_only=cls._GetCurrentBranchOnly(
                     opt, project.manifest
                 ),
                 force_sync=opt.force_sync,
@@ -756,7 +760,7 @@ later is required to fix a server side protocol bug.
                 optimized_fetch=opt.optimized_fetch,
                 retry_fetches=opt.retry_fetches,
                 prune=opt.prune,
-                ssh_proxy=self.ssh_proxy,
+                ssh_proxy=cls.get_parallel_context()["ssh_proxy"],
                 clone_filter=project.manifest.CloneFilter,
                 partial_clone_exclude=project.manifest.PartialCloneExclude,
                 clone_filter_for_depth=project.manifest.CloneFilterForDepth,
@@ -788,24 +792,20 @@ later is required to fix a server side protocol bug.
                 type(e).__name__,
                 e,
             )
-            del self._sync_dict[k]
             errors.append(e)
             raise
+        finally:
+            del cls.get_parallel_context()["sync_dict"][k]
 
         finish = time.time()
-        del self._sync_dict[k]
         return _FetchOneResult(
-            success, errors, project, start, finish, remote_fetched
+            success, errors, project_idx, start, finish, remote_fetched
         )
 
-    @classmethod
-    def _FetchInitChild(cls, ssh_proxy):
-        cls.ssh_proxy = ssh_proxy
-
     def _GetSyncProgressMessage(self):
         earliest_time = float("inf")
         earliest_proj = None
-        items = self._sync_dict.items()
+        items = self.get_parallel_context()["sync_dict"].items()
         for project, t in items:
             if t < earliest_time:
                 earliest_time = t
@@ -813,7 +813,7 @@ later is required to fix a server side protocol bug.
 
         if not earliest_proj:
             # This function is called when sync is still running but in some
-            # cases (by chance), _sync_dict can contain no entries. Return some
+            # cases (by chance), sync_dict can contain no entries. Return some
             # text to indicate that sync is still working.
             return "..working.."
 
@@ -821,6 +821,16 @@ later is required to fix a server side protocol bug.
         jobs = jobs_str(len(items))
         return f"{jobs} | {elapsed_str(elapsed)} {earliest_proj}"
 
+    @classmethod
+    def InitWorker(cls):
+        # Force connect to the manager server now.
+        # This is good because workers are initialized one by one. Without this,
+        # multiple workers may connect to the manager when handling the first
+        # job at the same time. Then the connection may fail if too many
+        # connections are pending and execeeded the socket listening backlog,
+        # especially on MacOS.
+        len(cls.get_parallel_context()["sync_dict"])
+
     def _Fetch(self, projects, opt, err_event, ssh_proxy, errors):
         ret = True
 
@@ -835,7 +845,6 @@ later is required to fix a server side protocol bug.
             elide=True,
         )
 
-        self._sync_dict = multiprocessing.Manager().dict()
         sync_event = _threading.Event()
 
         def _MonitorSyncLoop():
@@ -846,21 +855,13 @@ later is required to fix a server side protocol bug.
 
         sync_progress_thread = _threading.Thread(target=_MonitorSyncLoop)
         sync_progress_thread.daemon = True
-        sync_progress_thread.start()
 
-        objdir_project_map = dict()
-        for project in projects:
-            objdir_project_map.setdefault(project.objdir, []).append(project)
-        projects_list = list(objdir_project_map.values())
-
-        jobs = min(opt.jobs_network, len(projects_list))
-
-        def _ProcessResults(results_sets):
+        def _ProcessResults(pool, pm, results_sets):
             ret = True
             for results in results_sets:
                 for result in results:
                     success = result.success
-                    project = result.project
+                    project = projects[result.project_idx]
                     start = result.start
                     finish = result.finish
                     self._fetch_times.Set(project, finish - start)
@@ -884,45 +885,50 @@ later is required to fix a server side protocol bug.
                         fetched.add(project.gitdir)
                     pm.update()
                 if not ret and opt.fail_fast:
+                    if pool:
+                        pool.close()
                     break
             return ret
 
-        # We pass the ssh proxy settings via the class.  This allows
-        # multiprocessing to pickle it up when spawning children.  We can't pass
-        # it as an argument to _FetchProjectList below as multiprocessing is
-        # unable to pickle those.
-        Sync.ssh_proxy = None
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            self.get_parallel_context()[
+                "sync_dict"
+            ] = multiprocessing.Manager().dict()
 
-        # NB: Multiprocessing is heavy, so don't spin it up for one job.
-        if jobs == 1:
-            self._FetchInitChild(ssh_proxy)
-            if not _ProcessResults(
-                self._FetchProjectList(opt, x) for x in projects_list
-            ):
-                ret = False
-        else:
+            objdir_project_map = dict()
+            for index, project in enumerate(projects):
+                objdir_project_map.setdefault(project.objdir, []).append(index)
+            projects_list = list(objdir_project_map.values())
+
+            jobs = max(1, min(opt.jobs_network, len(projects_list)))
+
+            # We pass the ssh proxy settings via the class.  This allows
+            # multiprocessing to pickle it up when spawning children.  We can't
+            # pass it as an argument to _FetchProjectList below as
+            # multiprocessing is unable to pickle those.
+            self.get_parallel_context()["ssh_proxy"] = ssh_proxy
+
+            sync_progress_thread.start()
             if not opt.quiet:
                 pm.update(inc=0, msg="warming up")
-            with multiprocessing.Pool(
-                jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)
-            ) as pool:
-                results = pool.imap_unordered(
+            try:
+                ret = self.ExecuteInParallel(
+                    jobs,
                     functools.partial(self._FetchProjectList, opt),
                     projects_list,
-                    chunksize=_chunksize(len(projects_list), jobs),
+                    callback=_ProcessResults,
+                    output=pm,
+                    # Use chunksize=1 to avoid the chance that some workers are
+                    # idle while other workers still have more than one job in
+                    # their chunk queue.
+                    chunksize=1,
+                    initializer=self.InitWorker,
                 )
-                if not _ProcessResults(results):
-                    ret = False
-                    pool.close()
+            finally:
+                sync_event.set()
+                sync_progress_thread.join()
 
-        # Cleanup the reference now that we're done with it, and we're going to
-        # release any resources it points to.  If we don't, later
-        # multiprocessing usage (e.g. checkouts) will try to pickle and then
-        # crash.
-        del Sync.ssh_proxy
-
-        sync_event.set()
-        pm.end()
         self._fetch_times.Save()
         self._local_sync_state.Save()
 
@@ -963,7 +969,9 @@ later is required to fix a server side protocol bug.
         if not success:
             err_event.set()
 
-        _PostRepoFetch(rp, opt.repo_verify)
+        # Call self update, unless requested not to
+        if os.environ.get("REPO_SKIP_SELF_UPDATE", "0") == "0":
+            _PostRepoFetch(rp, opt.repo_verify)
         if opt.network_only:
             # Bail out now; the rest touches the working tree.
             if err_event.is_set():
@@ -1008,14 +1016,15 @@ later is required to fix a server side protocol bug.
 
         return _FetchMainResult(all_projects)
 
+    @classmethod
     def _CheckoutOne(
-        self,
+        cls,
         detach_head,
         force_sync,
         force_checkout,
         force_rebase,
         verbose,
-        project,
+        project_idx,
     ):
         """Checkout work tree for one project
 
@@ -1027,11 +1036,12 @@ later is required to fix a server side protocol bug.
             force_checkout: Force checking out of the repo content.
             force_rebase: Force rebase.
             verbose: Whether to show verbose messages.
-            project: Project object for the project to checkout.
+            project_idx: Project index for the project to checkout.
 
         Returns:
             Whether the fetch was successful.
         """
+        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         syncbuf = SyncBuffer(
             project.manifest.manifestProject.config, detach_head=detach_head
@@ -1048,6 +1058,8 @@ later is required to fix a server side protocol bug.
                 verbose=verbose,
             )
             success = syncbuf.Finish()
+        except KeyboardInterrupt:
+            logger.error("Keyboard interrupt while processing %s", project.name)
         except GitError as e:
             logger.error(
                 "error.GitError: Cannot checkout %s: %s", project.name, e
@@ -1065,7 +1077,7 @@ later is required to fix a server side protocol bug.
         if not success:
             logger.error("error: Cannot checkout %s", project.name)
         finish = time.time()
-        return _CheckoutOneResult(success, errors, project, start, finish)
+        return _CheckoutOneResult(success, errors, project_idx, start, finish)
 
     def _Checkout(self, all_projects, opt, err_results, checkout_errors):
         """Checkout projects listed in all_projects
@@ -1083,7 +1095,9 @@ later is required to fix a server side protocol bug.
             ret = True
             for result in results:
                 success = result.success
-                project = result.project
+                project = self.get_parallel_context()["projects"][
+                    result.project_idx
+                ]
                 start = result.start
                 finish = result.finish
                 self.event_log.AddSync(
@@ -1110,22 +1124,28 @@ later is required to fix a server side protocol bug.
             return ret
 
         for projects in _SafeCheckoutOrder(all_projects):
-            proc_res = self.ExecuteInParallel(
-                opt.jobs_checkout,
-                functools.partial(
-                    self._CheckoutOne,
-                    opt.detach_head,
-                    opt.force_sync,
-                    opt.force_checkout,
-                    opt.rebase,
-                    opt.verbose,
-                ),
-                projects,
-                callback=_ProcessResults,
-                output=Progress(
-                    "Checking out", len(all_projects), quiet=opt.quiet
-                ),
-            )
+            with self.ParallelContext():
+                self.get_parallel_context()["projects"] = projects
+                proc_res = self.ExecuteInParallel(
+                    opt.jobs_checkout,
+                    functools.partial(
+                        self._CheckoutOne,
+                        opt.detach_head,
+                        opt.force_sync,
+                        opt.force_checkout,
+                        opt.rebase,
+                        opt.verbose,
+                    ),
+                    range(len(projects)),
+                    callback=_ProcessResults,
+                    output=Progress(
+                        "Checking out", len(all_projects), quiet=opt.quiet
+                    ),
+                    # Use chunksize=1 to avoid the chance that some workers are
+                    # idle while other workers still have more than one job in
+                    # their chunk queue.
+                    chunksize=1,
+                )
 
         self._local_sync_state.Save()
         return proc_res and not err_results
@@ -1424,7 +1444,10 @@ later is required to fix a server side protocol bug.
             for need_remove_file in need_remove_files:
                 # Try to remove the updated copyfile or linkfile.
                 # So, if the file is not exist, nothing need to do.
-                platform_utils.remove(need_remove_file, missing_ok=True)
+                platform_utils.remove(
+                    os.path.join(self.client.topdir, need_remove_file),
+                    missing_ok=True,
+                )
 
         # Create copy-link-files.json, save dest path of "copyfile" and
         # "linkfile".
@@ -1479,6 +1502,7 @@ later is required to fix a server side protocol bug.
         if manifest_server.startswith("persistent-"):
             manifest_server = manifest_server[len("persistent-") :]
 
+        # Changes in behavior should update docs/smart-sync.md accordingly.
         try:
             server = xmlrpc.client.Server(manifest_server, transport=transport)
             if opt.smart_sync:
@@ -1975,6 +1999,8 @@ def _PostRepoFetch(rp, repo_verify=True, verbose=False):
             # We also have to make sure this will switch to an older commit if
             # that's the latest tag in order to support release rollback.
             try:
+                # Refresh index since reset --keep won't do it.
+                rp.work_git.update_index("-q", "--refresh")
                 rp.work_git.reset("--keep", new_rev)
             except GitError as e:
                 raise RepoUnhandledExceptionError(e)

subcmds/upload.py

@@ -603,19 +603,22 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             full_dest = destination
             if not full_dest.startswith(R_HEADS):
                 full_dest = R_HEADS + full_dest
+            full_revision = branch.project.revisionExpr
+            if not full_revision.startswith(R_HEADS):
+                full_revision = R_HEADS + full_revision
 
             # If the merge branch of the local branch is different from
             # the project's revision AND destination, this might not be
             # intentional.
             if (
                 merge_branch
-                and merge_branch != branch.project.revisionExpr
+                and merge_branch != full_revision
                 and merge_branch != full_dest
             ):
                 print(
                     f"For local branch {branch.name}: merge branch "
                     f"{merge_branch} does not match destination branch "
-                    f"{destination}"
+                    f"{destination} and revision {branch.project.revisionExpr}"
                 )
                 print("skipping upload.")
                 print(
@@ -713,16 +716,17 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         merge_branch = p.stdout.strip()
         return merge_branch
 
-    @staticmethod
-    def _GatherOne(opt, project):
+    @classmethod
+    def _GatherOne(cls, opt, project_idx):
         """Figure out the upload status for |project|."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         if opt.current_branch:
             cbr = project.CurrentBranch
             up_branch = project.GetUploadableBranch(cbr)
             avail = [up_branch] if up_branch else None
         else:
             avail = project.GetUploadableBranches(opt.branch)
-        return (project, avail)
+        return (project_idx, avail)
 
     def Execute(self, opt, args):
         projects = self.GetProjects(
@@ -732,7 +736,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         def _ProcessResults(_pool, _out, results):
             pending = []
             for result in results:
-                project, avail = result
+                project_idx, avail = result
+                project = projects[project_idx]
                 if avail is None:
                     logger.error(
                         'repo: error: %s: Unable to upload branch "%s". '
@@ -743,15 +748,17 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                         project.manifest.branch,
                     )
                 elif avail:
-                    pending.append(result)
+                    pending.append((project, avail))
             return pending
 
-        pending = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._GatherOne, opt),
-            projects,
-            callback=_ProcessResults,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            pending = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._GatherOne, opt),
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
 
         if not pending:
             if opt.branch is None:

tests/fixtures/gitc_config

@@ -1 +0,0 @@
-gitc_dir=/test/usr/local/google/gitc

tests/test_git_trace2_event_log.py

@@ -150,7 +150,7 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <start event>
         """
-        self._event_log_module.StartEvent()
+        self._event_log_module.StartEvent([])
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
             self._log_data = self.readLog(log_path)
@@ -213,10 +213,8 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <command event>
         """
-        name = "repo"
-        subcommands = ["init" "this"]
         self._event_log_module.CommandEvent(
-            name="repo", subcommands=subcommands
+            name="repo", subcommands=["init", "this"]
         )
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
@@ -225,12 +223,10 @@ class EventLogTestCase(unittest.TestCase):
         self.assertEqual(len(self._log_data), 2)
         command_event = self._log_data[1]
         self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
-        self.verifyCommonKeys(command_event, expected_event_name="command")
+        self.verifyCommonKeys(command_event, expected_event_name="cmd_name")
         # Check for 'command' event specific fields.
         self.assertIn("name", command_event)
-        self.assertIn("subcommands", command_event)
-        self.assertEqual(command_event["name"], name)
-        self.assertEqual(command_event["subcommands"], subcommands)
+        self.assertEqual(command_event["name"], "repo-init-this")
 
     def test_def_params_event_repo_config(self):
         """Test 'def_params' event data outputs only repo config keys.
@@ -382,17 +378,17 @@ class EventLogTestCase(unittest.TestCase):
             socket_path = os.path.join(tempdir, "server.sock")
             server_ready = threading.Condition()
             # Start "server" listening on Unix domain socket at socket_path.
+            server_thread = threading.Thread(
+                target=serverLoggingThread,
+                args=(socket_path, server_ready, received_traces),
+            )
             try:
-                server_thread = threading.Thread(
-                    target=serverLoggingThread,
-                    args=(socket_path, server_ready, received_traces),
-                )
                 server_thread.start()
 
                 with server_ready:
                     server_ready.wait(timeout=120)
 
-                self._event_log_module.StartEvent()
+                self._event_log_module.StartEvent([])
                 path = self._event_log_module.Write(
                     path=f"af_unix:{socket_path}"
                 )

tests/test_manifest_xml.py

@@ -1049,6 +1049,91 @@ class RemoveProjectElementTests(ManifestParseTestCase):
         self.assertTrue(found_proj1_path1)
         self.assertTrue(found_proj2)
 
+    def test_base_revision_checks_on_patching(self):
+        manifest_fail_wrong_tag = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="tag.002" />
+  <project name="project1" path="tests/path1" />
+  <extend-project name="project1" revision="new_hash" base-rev="tag.001" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_wrong_tag.ToXml()
+
+        manifest_fail_remove = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <remove-project name="project1" base-rev="wrong_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_remove.ToXml()
+
+        manifest_fail_extend = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <extend-project name="project1" revision="new_hash" base-rev="wrong_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_extend.ToXml()
+
+        manifest_fail_unknown = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" />
+  <extend-project name="project1" revision="new_hash" base-rev="any_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_unknown.ToXml()
+
+        manifest_ok = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <project name="project2" path="tests/path2" revision="hash2" />
+  <project name="project3" path="tests/path3" revision="hash3" />
+  <project name="project4" path="tests/path4" revision="hash4" />
+
+  <remove-project name="project1" />
+  <remove-project name="project2" base-rev="hash2" />
+  <project name="project2" path="tests/path2" revision="new_hash2" />
+  <extend-project name="project3" base-rev="hash3" revision="new_hash3" />
+  <extend-project name="project3" base-rev="new_hash3" revision="newer_hash3" />
+  <remove-project path="tests/path4" base-rev="hash4" />
+</manifest>
+"""
+        )
+        found_proj2 = False
+        found_proj3 = False
+        for proj in manifest_ok.projects:
+            if proj.name == "project2":
+                found_proj2 = True
+            if proj.name == "project3":
+                found_proj3 = True
+            self.assertNotEqual(proj.name, "project1")
+            self.assertNotEqual(proj.name, "project4")
+        self.assertTrue(found_proj2)
+        self.assertTrue(found_proj3)
+        self.assertTrue(len(manifest_ok.projects) == 2)
+
 
 class ExtendProjectElementTests(ManifestParseTestCase):
     """Tests for <extend-project>."""

tests/test_subcmds_forall.py

@@ -0,0 +1,156 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the forall subcmd."""
+
+from io import StringIO
+import os
+from shutil import rmtree
+import subprocess
+import tempfile
+import unittest
+from unittest import mock
+
+import git_command
+import manifest_xml
+import project
+import subcmds
+
+
+class AllCommands(unittest.TestCase):
+    """Check registered all_commands."""
+
+    def setUp(self):
+        """Common setup."""
+        self.tempdirobj = tempfile.TemporaryDirectory(prefix="forall_tests")
+        self.tempdir = self.tempdirobj.name
+        self.repodir = os.path.join(self.tempdir, ".repo")
+        self.manifest_dir = os.path.join(self.repodir, "manifests")
+        self.manifest_file = os.path.join(
+            self.repodir, manifest_xml.MANIFEST_FILE_NAME
+        )
+        self.local_manifest_dir = os.path.join(
+            self.repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME
+        )
+        os.mkdir(self.repodir)
+        os.mkdir(self.manifest_dir)
+
+    def tearDown(self):
+        """Common teardown."""
+        rmtree(self.tempdir, ignore_errors=True)
+
+    def initTempGitTree(self, git_dir):
+        """Create a new empty git checkout for testing."""
+
+        # Tests need to assume, that main is default branch at init,
+        # which is not supported in config until 2.28.
+        cmd = ["git", "init", "-q"]
+        if git_command.git_require((2, 28, 0)):
+            cmd += ["--initial-branch=main"]
+        else:
+            # Use template dir for init
+            templatedir = os.path.join(self.tempdirobj.name, ".test-template")
+            os.makedirs(templatedir)
+            with open(os.path.join(templatedir, "HEAD"), "w") as fp:
+                fp.write("ref: refs/heads/main\n")
+            cmd += ["--template", templatedir]
+        cmd += [git_dir]
+        subprocess.check_call(cmd)
+
+    def getXmlManifestWith8Projects(self):
+        """Create and return a setup of 8 projects with enough dummy
+        files and setup to execute forall."""
+
+        # Set up a manifest git dir for parsing to work
+        gitdir = os.path.join(self.repodir, "manifests.git")
+        os.mkdir(gitdir)
+        with open(os.path.join(gitdir, "config"), "w") as fp:
+            fp.write(
+                """[remote "origin"]
+                    url = https://localhost:0/manifest
+                    verbose = false
+                """
+            )
+
+        # Add the manifest data
+        manifest_data = """
+                <manifest>
+                    <remote name="origin" fetch="http://localhost" />
+                    <default remote="origin" revision="refs/heads/main" />
+                    <project name="project1" path="tests/path1" />
+                    <project name="project2" path="tests/path2" />
+                    <project name="project3" path="tests/path3" />
+                    <project name="project4" path="tests/path4" />
+                    <project name="project5" path="tests/path5" />
+                    <project name="project6" path="tests/path6" />
+                    <project name="project7" path="tests/path7" />
+                    <project name="project8" path="tests/path8" />
+                </manifest>
+            """
+        with open(self.manifest_file, "w", encoding="utf-8") as fp:
+            fp.write(manifest_data)
+
+        # Set up 8 empty projects to match the manifest
+        for x in range(1, 9):
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "projects/tests/path" + str(x) + ".git"
+                )
+            )
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "project-objects/project" + str(x) + ".git"
+                )
+            )
+            git_path = os.path.join(self.tempdir, "tests/path" + str(x))
+            self.initTempGitTree(git_path)
+
+        return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
+
+    # Use mock to capture stdout from the forall run
+    @unittest.mock.patch("sys.stdout", new_callable=StringIO)
+    def test_forall_all_projects_called_once(self, mock_stdout):
+        """Test that all projects get a command run once each."""
+
+        manifest_with_8_projects = self.getXmlManifestWith8Projects()
+
+        cmd = subcmds.forall.Forall()
+        cmd.manifest = manifest_with_8_projects
+
+        # Use echo project names as the test of forall
+        opts, args = cmd.OptionParser.parse_args(["-c", "echo $REPO_PROJECT"])
+        opts.verbose = False
+
+        # Mock to not have the Execute fail on remote check
+        with mock.patch.object(
+            project.Project, "GetRevisionId", return_value="refs/heads/main"
+        ):
+            # Run the forall command
+            cmd.Execute(opts, args)
+
+            # Verify that we got every project name in the prints
+            for x in range(1, 9):
+                self.assertIn("project" + str(x), mock_stdout.getvalue())
+
+            # Split the captured output into lines to count them
+            line_count = 0
+            for line in mock_stdout.getvalue().split("\n"):
+                # A commented out print to stderr as a reminder
+                # that stdout is mocked, include sys and uncomment if needed
+                # print(line, file=sys.stderr)
+                if len(line) > 0:
+                    line_count += 1
+
+            # Verify that we didn't get more lines than expected
+            assert line_count == 8

tests/test_wrapper.py

@@ -72,84 +72,11 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
 
     def test_init_parser(self):
         """Make sure 'init' GetParser works."""
-        parser = self.wrapper.GetParser(gitc_init=False)
+        parser = self.wrapper.GetParser()
         opts, args = parser.parse_args([])
         self.assertEqual([], args)
         self.assertIsNone(opts.manifest_url)
 
-    def test_gitc_init_parser(self):
-        """Make sure 'gitc-init' GetParser raises."""
-        with self.assertRaises(SystemExit):
-            self.wrapper.GetParser(gitc_init=True)
-
-    def test_get_gitc_manifest_dir_no_gitc(self):
-        """
-        Test reading a missing gitc config file
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "")
-
-    def test_get_gitc_manifest_dir(self):
-        """
-        Test reading the gitc config file and parsing the directory
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "/test/usr/local/google/gitc")
-
-    def test_gitc_parse_clientdir_no_gitc(self):
-        """
-        Test parsing the gitc clientdir without gitc running
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-
-    def test_gitc_parse_clientdir(self):
-        """
-        Test parsing the gitc clientdir
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/extra"),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/extra"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/"), None
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/test/usr/local/google/gitc/"),
-            None,
-        )
-
 
 class SetGitTrace2ParentSid(RepoWrapperTestCase):
     """Check SetGitTrace2ParentSid behavior."""