sync: Add --superproject-rev flag to sync to specific revision

Allow syncing the outer manifest to a state defined by a specific
superproject revision. It updates the superproject, reads the manifest
commit from .supermanifest, and checks out the outer manifest project
to that commit.

Submanifests are then processed normally, allowing them to be updated
to the revisions specified in the new outer manifest state.

Bug: 416589884
Change-Id: I304c37a2b8794f9b74cb7e5e209a8a93762bdb52
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/576321
Commit-Queue: Gavin Mak <gavinmak@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

.flake8

@@ -1,15 +1,17 @@
 [flake8]
-max-line-length=100
+max-line-length = 80
-ignore=
+per-file-ignores =
-    # E111: Indentation is not a multiple of four
+    # E501: line too long
-    E111,
+    tests/test_git_superproject.py: E501
-    # E114: Indentation is not a multiple of four (comment)
+extend-ignore =
-    E114,
+    # E203: Whitespace before ':'
+    # See https://github.com/PyCQA/pycodestyle/issues/373
+    E203,
     # E402: Module level import not at top of file
     E402,
     # E731: do not assign a lambda expression, use a def
     E731,
-    # W503: Line break before binary operator
+exclude =
-    W503,
+    .venv,
-    # W504: Line break after binary operator
+    venv,
-    W504
+    .tox,

.github/workflows/black.yml

@@ -0,0 +1,16 @@
+# GitHub actions workflow.
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
+# https://black.readthedocs.io/en/stable/integrations/github_actions.html
+
+name: Format
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: psf/black@stable

.github/workflows/close-pull-request.yml

@@ -0,0 +1,22 @@
+# GitHub actions workflow.
+# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+
+# https://github.com/superbrothers/close-pull-request
+name: Close Pull Request
+
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: superbrothers/close-pull-request@v3
+      with:
+        comment: >
+          Thanks for your contribution!
+          Unfortunately, we don't use GitHub pull requests to manage code
+          contributions to this repository.
+          Instead, please see [README.md](../blob/HEAD/CONTRIBUTING.md)
+          which provides full instructions on how to get involved.

.github/workflows/flake8-postsubmit.yml

@@ -0,0 +1,23 @@
+# GitHub actions workflow.
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
+# https://github.com/marketplace/actions/python-flake8
+
+name: Flake8
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  lint:
+    name: Python Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Run flake8
+        uses: julianwachholz/flake8-action@v2
+        with:
+          checkName: "Python Lint"

.github/workflows/test-ci.yml

@@ -13,19 +13,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        # ubuntu-20.04 is the last version that supports python 3.6
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
+        python -m pip install pytest
-    - name: Test with tox
+    - name: Run tests
-      run: tox
+      run: python -m pytest

.gitignore

@@ -6,6 +6,7 @@ __pycache__
 /dist
 .repopickle_*
 /repoc
+/.cipd_bin
 /.tox
 /.venv
 

.pydevproject

@@ -5,6 +5,5 @@
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/git-repo</path>
 </pydev_pathproperty>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

CONTRIBUTING.md

@@ -1,19 +1,19 @@
+# Submitting Changes
+
+Here's a short overview of the process.
+
+*   Make small logical changes.
+*   [Provide a meaningful commit message][commit-message-style].
+*   Make sure all code is under the Apache License, 2.0.
+*   Publish your changes for review.
+    *   `git push origin HEAD:refs/for/main`
+*   Make corrections if requested.
+*   [Verify your changes on Gerrit.](#verify)
+*   [Send to the commit queue for testing & merging.](#cq)
+
 [TOC]
 
-# Short Version
+## Long Version
-
- - Make small logical changes.
- - [Provide a meaningful commit message][commit-message-style].
- - Check for coding errors and style nits with flake8.
- - Make sure all code is under the Apache License, 2.0.
- - Publish your changes for review.
- - Make corrections if requested.
- - Verify your changes on gerrit so they can be submitted.
-
-   `git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/main`
-
-
-# Long Version
 
 I wanted a file describing how to submit patches for repo,
 so I started with the one found in the core Git distribution
@@ -24,7 +24,7 @@ However there are some differences, so please review and familiarize
 yourself with the following relevant bits.
 
 
-## Make separate commits for logically separate changes.
+## Make separate commits for logically separate changes
 
 Unless your patch is really trivial, you should not be sending out a patch that
 was generated between your working tree and your commit head.
@@ -39,17 +39,21 @@ If your description starts to get too long, that's a sign that you
 probably need to split up your commit to finer grained pieces.
 
 
-## Check for coding errors and style violations with flake8
+## Linting and formatting code
 
-Run `flake8` on changed modules:
+Lint any changes by running:
+```sh
+$ flake8
+```
 
-    flake8 file.py
+And format with:
+```sh
+$ black file.py
+```
 
-Note that repo generally follows [Google's Python Style Guide] rather than
+Repo uses [black](https://black.readthedocs.io/) with line length of 80 as its
-[PEP 8], with a couple of notable exceptions:
+formatter and flake8 as its linter. Repo also follows
-
+[Google's Python Style Guide].
-* Indentation is at 2 columns rather than 4
-* The maximum line length is 100 columns rather than 80
 
 There should be no new errors or warnings introduced.
 
@@ -64,15 +68,11 @@ the entire project in the included `.flake8` file.
 [PEP 8]: https://www.python.org/dev/peps/pep-0008/
 [flake8 documentation]: https://flake8.pycqa.org/en/3.1.1/user/ignoring-errors.html#in-line-ignoring-errors
 
+
 ## Running tests
 
-We use [pytest](https://pytest.org/) and [tox](https://tox.readthedocs.io/) for
+We use [pytest](https://pytest.org/) for running tests.  You should make sure to
-running tests.  You should make sure to install those first.
+install that first.
-
-To run the full suite against all supported Python versions, simply execute:
-```sh
-$ tox -p auto
-```
 
 We have [`./run_tests`](./run_tests) which is a simple wrapper around `pytest`:
 ```sh
@@ -118,7 +118,7 @@ your patch.  It is virtually impossible to remove a patch once it
 has been applied and pushed out.
 
 
-## Sending your patches.
+## Sending your patches
 
 Do not email your patches to anyone.
 
@@ -134,7 +134,7 @@ they have right to redistribute your work under the Apache License:
 
 Ensure you have obtained an HTTP password to authenticate:
 
-  https://gerrit-review.googlesource.com/new-password
+  https://www.googlesource.com/new-password
 
 Ensure that you have the local commit hook installed to automatically
 add a ChangeId to your commits:
@@ -166,12 +166,16 @@ commit. If you make the requested changes you will need to amend your commit
 and push it to the review server again.
 
 
-## Verify your changes on gerrit
+## Verify your changes on Gerrit {#verify}
 
 After you receive a Code-Review+2 from the maintainer, select the Verified
-button on the gerrit page for the change. This verifies that you have tested
+button on the Gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
-The maintainer will then submit your changes to the repository.
 
+## Merge your changes via the commit queue {#cq}
+
+Once a change is ready to be merged, select the Commit-Queue+2 setting on the
+Gerrit page for it. This tells the CI system to test the change, and if it
+passes all the checks, automatically merges it.
 
 [commit-message-style]: https://chris.beams.io/posts/git-commit/

README.md

@@ -8,13 +8,13 @@ that you can put anywhere in your path.
 
 * Homepage: <https://gerrit.googlesource.com/git-repo/>
 * Mailing list: [repo-discuss on Google Groups][repo-discuss]
-* Bug reports: <https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo>
+* Bug reports: <https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071>
 * Source: <https://gerrit.googlesource.com/git-repo/>
 * Overview: <https://source.android.com/source/developing.html>
 * Docs: <https://source.android.com/source/using-repo.html>
 * [repo Manifest Format](./docs/manifest-format.md)
 * [repo Hooks](./docs/repo-hooks.md)
-* [Submitting patches](./SUBMITTING_PATCHES.md)
+* [Contributing](./CONTRIBUTING.md)
 * Running Repo in [Microsoft Windows](./docs/windows.md)
 * GitHub mirror: <https://github.com/GerritCodeReview/git-repo>
 * Postsubmit tests: <https://github.com/GerritCodeReview/git-repo/actions>
@@ -49,7 +49,44 @@ $ curl https://storage.googleapis.com/git-repo-downloads/repo > ~/.bin/repo
 $ chmod a+rx ~/.bin/repo
 ```
 
+## Shell Completion
 
-[new-bug]: https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue
+Repo includes completion scripts for Bash and Zsh.
-[issue tracker]: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
+
+### Bash
+
+To enable completion in Bash, source `completion.bash` in your `~/.bashrc`:
+
+```sh
+source /path/to/git-repo/completion.bash
+```
+
+### Zsh
+
+To enable completion in Zsh, you can either:
+
+1.  Copy or symlink `completion.zsh` to a file named `_repo` in a directory in your `$fpath`:
+    ```sh
+    mkdir -p ~/.zsh/completion
+    # You can copy the file:
+    cp /path/to/git-repo/completion.zsh ~/.zsh/completion/_repo
+    # Or symlink it:
+    ln -s /path/to/git-repo/completion.zsh ~/.zsh/completion/_repo
+    ```
+    Then add that directory to your `fpath` in `~/.zshrc` before `compinit`:
+    ```zsh
+    fpath=(~/.zsh/completion $fpath)
+    autoload -Uz compinit
+    compinit
+    ```
+
+2.  Or source the file directly and call `compdef` in your `~/.zshrc`:
+    ```zsh
+    source /path/to/git-repo/completion.zsh
+    compdef _repo repo
+    ```
+
+
+[new-bug]: https://issues.gerritcodereview.com/issues/new?component=1370071
+[issue tracker]: https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071
 [repo-discuss]: https://groups.google.com/forum/#!forum/repo-discuss

cipd_manifest.txt

@@ -0,0 +1,32 @@
+# Copyright (C) 2026 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains version pins of a few tools used by run_tests.
+
+# Pin resolved versions in the repo, to reduce trust in the CIPD backend.
+#
+# Most of these tools are generated via builders at
+# https://ci.chromium.org/p/infra/g/infra/console
+#
+# For these, the git revision is the one of
+# https://chromium.googlesource.com/infra/infra.git.
+#
+# To regenerate them (after modifying this file):
+#   cipd ensure-file-resolve -ensure-file cipd_manifest.txt
+$ResolvedVersions cipd_manifest.versions
+
+# Supported platforms.  Feel free to add more as long as the tools work.
+$VerifiedPlatform linux-amd64 linux-arm64 mac-amd64 mac-arm64
+
+infra/3pp/tools/help2man/${platform} version:3@1.49.3

cipd_manifest.versions

@@ -0,0 +1,18 @@
+# This file is auto-generated by 'cipd ensure-file-resolve'.
+# Do not modify manually. All changes will be overwritten.
+
+infra/3pp/tools/help2man/linux-amd64
+	version:3@1.49.3
+	82ySLC4--mvjnmuwj_j_2odHLj5zFaNFpWvfCyMunY0C
+
+infra/3pp/tools/help2man/linux-arm64
+	version:3@1.49.3
+	szfWfjI4RyCT6gIB48CeUOGG6_Un4wfbIb0Vvs5M9MUC
+
+infra/3pp/tools/help2man/mac-amd64
+	version:3@1.49.3
+	t7iyQx5TnGbuJfj6zMTvjPdBl-Cb_w5papUh80NI2S8C
+
+infra/3pp/tools/help2man/mac-arm64
+	version:3@1.49.3
+	t5eD-eLKRpnSifZdbi072U8bnPfGye1pIMmQ07nxJ7kC

color.py

@@ -14,199 +14,206 @@
 
 import os
 import sys
+from typing import Optional
 
 import pager
 
-COLORS = {None: -1,
-          'normal': -1,
-          'black': 0,
-          'red': 1,
-          'green': 2,
-          'yellow': 3,
-          'blue': 4,
-          'magenta': 5,
-          'cyan': 6,
-          'white': 7}
 
-ATTRS = {None: -1,
+COLORS = {
-         'bold': 1,
+    None: -1,
-         'dim': 2,
+    "normal": -1,
-         'ul': 4,
+    "black": 0,
-         'blink': 5,
+    "red": 1,
-         'reverse': 7}
+    "green": 2,
+    "yellow": 3,
+    "blue": 4,
+    "magenta": 5,
+    "cyan": 6,
+    "white": 7,
+}
+
+ATTRS = {None: -1, "bold": 1, "dim": 2, "ul": 4, "blink": 5, "reverse": 7}
 
 RESET = "\033[m"
 
 
 def is_color(s):
-  return s in COLORS
+    return s in COLORS
 
 
 def is_attr(s):
-  return s in ATTRS
+    return s in ATTRS
 
 
 def _Color(fg=None, bg=None, attr=None):
-  fg = COLORS[fg]
+    fg = COLORS[fg]
-  bg = COLORS[bg]
+    bg = COLORS[bg]
-  attr = ATTRS[attr]
+    attr = ATTRS[attr]
 
-  if attr >= 0 or fg >= 0 or bg >= 0:
+    if attr >= 0 or fg >= 0 or bg >= 0:
-    need_sep = False
+        need_sep = False
-    code = "\033["
+        code = "\033["
 
-    if attr >= 0:
+        if attr >= 0:
-      code += chr(ord('0') + attr)
+            code += chr(ord("0") + attr)
-      need_sep = True
+            need_sep = True
 
-    if fg >= 0:
+        if fg >= 0:
-      if need_sep:
+            if need_sep:
-        code += ';'
+                code += ";"
-      need_sep = True
+            need_sep = True
 
-      if fg < 8:
+            if fg < 8:
-        code += '3%c' % (ord('0') + fg)
+                code += "3%c" % (ord("0") + fg)
-      else:
+            else:
-        code += '38;5;%d' % fg
+                code += "38;5;%d" % fg
 
-    if bg >= 0:
+        if bg >= 0:
-      if need_sep:
+            if need_sep:
-        code += ';'
+                code += ";"
 
-      if bg < 8:
+            if bg < 8:
-        code += '4%c' % (ord('0') + bg)
+                code += "4%c" % (ord("0") + bg)
-      else:
+            else:
-        code += '48;5;%d' % bg
+                code += "48;5;%d" % bg
-    code += 'm'
+        code += "m"
-  else:
+    else:
-    code = ''
+        code = ""
-  return code
+    return code
 
 
 DEFAULT = None
 
 
-def SetDefaultColoring(state):
+def SetDefaultColoring(state: Optional[str]) -> None:
-  """Set coloring behavior to |state|.
+    """Set coloring behavior to |state|.
 
-  This is useful for overriding config options via the command line.
+    This is useful for overriding config options via the command line.
-  """
+    """
-  if state is None:
+    if state is None:
-    # Leave it alone -- return quick!
+        # Leave it alone -- return quick!
-    return
+        return
 
-  global DEFAULT
+    global DEFAULT
-  state = state.lower()
+    state = state.lower()
-  if state in ('auto',):
+    if state in ("auto",):
-    DEFAULT = state
+        DEFAULT = state
-  elif state in ('always', 'yes', 'true', True):
+    elif state in ("always", "yes", "true"):
-    DEFAULT = 'always'
+        DEFAULT = "always"
-  elif state in ('never', 'no', 'false', False):
+    elif state in ("never", "no", "false"):
-    DEFAULT = 'never'
+        DEFAULT = "never"
 
 
-class Coloring(object):
+class Coloring:
-  def __init__(self, config, section_type):
+    def __init__(self, config, section_type):
-    self._section = 'color.%s' % section_type
+        self._section = "color.%s" % section_type
-    self._config = config
+        self._config = config
-    self._out = sys.stdout
+        self._out = sys.stdout
 
-    on = DEFAULT
+        on = DEFAULT
-    if on is None:
+        if on is None:
-      on = self._config.GetString(self._section)
+            on = self._config.GetString(self._section)
-      if on is None:
+            if on is None:
-        on = self._config.GetString('color.ui')
+                on = self._config.GetString("color.ui")
 
-    if on == 'auto':
+        if on == "auto":
-      if pager.active or os.isatty(1):
+            if pager.active or os.isatty(1):
-        self._on = True
+                self._on = True
-      else:
+            else:
-        self._on = False
+                self._on = False
-    elif on in ('true', 'always'):
+        elif on in ("true", "always"):
-      self._on = True
+            self._on = True
-    else:
-      self._on = False
-
-  def redirect(self, out):
-    self._out = out
-
-  @property
-  def is_on(self):
-    return self._on
-
-  def write(self, fmt, *args):
-    self._out.write(fmt % args)
-
-  def flush(self):
-    self._out.flush()
-
-  def nl(self):
-    self._out.write('\n')
-
-  def printer(self, opt=None, fg=None, bg=None, attr=None):
-    s = self
-    c = self.colorer(opt, fg, bg, attr)
-
-    def f(fmt, *args):
-      s._out.write(c(fmt, *args))
-    return f
-
-  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
-    s = self
-    c = self.nofmt_colorer(opt, fg, bg, attr)
-
-    def f(fmt):
-      s._out.write(c(fmt))
-    return f
-
-  def colorer(self, opt=None, fg=None, bg=None, attr=None):
-    if self._on:
-      c = self._parse(opt, fg, bg, attr)
-
-      def f(fmt, *args):
-        output = fmt % args
-        return ''.join([c, output, RESET])
-      return f
-    else:
-
-      def f(fmt, *args):
-        return fmt % args
-      return f
-
-  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
-    if self._on:
-      c = self._parse(opt, fg, bg, attr)
-
-      def f(fmt):
-        return ''.join([c, fmt, RESET])
-      return f
-    else:
-      def f(fmt):
-        return fmt
-      return f
-
-  def _parse(self, opt, fg, bg, attr):
-    if not opt:
-      return _Color(fg, bg, attr)
-
-    v = self._config.GetString('%s.%s' % (self._section, opt))
-    if v is None:
-      return _Color(fg, bg, attr)
-
-    v = v.strip().lower()
-    if v == "reset":
-      return RESET
-    elif v == '':
-      return _Color(fg, bg, attr)
-
-    have_fg = False
-    for a in v.split(' '):
-      if is_color(a):
-        if have_fg:
-          bg = a
         else:
-          fg = a
+            self._on = False
-      elif is_attr(a):
-        attr = a
 
-    return _Color(fg, bg, attr)
+    def redirect(self, out):
+        self._out = out
+
+    @property
+    def is_on(self):
+        return self._on
+
+    def write(self, fmt, *args):
+        self._out.write(fmt % args)
+
+    def flush(self):
+        self._out.flush()
+
+    def nl(self):
+        self._out.write("\n")
+
+    def printer(self, opt=None, fg=None, bg=None, attr=None):
+        s = self
+        c = self.colorer(opt, fg, bg, attr)
+
+        def f(fmt, *args):
+            s._out.write(c(fmt, *args))
+
+        return f
+
+    def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+        s = self
+        c = self.nofmt_colorer(opt, fg, bg, attr)
+
+        def f(fmt):
+            s._out.write(c(fmt))
+
+        return f
+
+    def colorer(self, opt=None, fg=None, bg=None, attr=None):
+        if self._on:
+            c = self._parse(opt, fg, bg, attr)
+
+            def f(fmt, *args):
+                output = fmt % args
+                return "".join([c, output, RESET])
+
+            return f
+        else:
+
+            def f(fmt, *args):
+                return fmt % args
+
+            return f
+
+    def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+        if self._on:
+            c = self._parse(opt, fg, bg, attr)
+
+            def f(fmt):
+                return "".join([c, fmt, RESET])
+
+            return f
+        else:
+
+            def f(fmt):
+                return fmt
+
+            return f
+
+    def _parse(self, opt, fg, bg, attr):
+        if not opt:
+            return _Color(fg, bg, attr)
+
+        v = self._config.GetString(f"{self._section}.{opt}")
+        if v is None:
+            return _Color(fg, bg, attr)
+
+        v = v.strip().lower()
+        if v == "reset":
+            return RESET
+        elif v == "":
+            return _Color(fg, bg, attr)
+
+        have_fg = False
+        for a in v.split(" "):
+            if is_color(a):
+                if have_fg:
+                    bg = a
+                else:
+                    have_fg = True
+                    fg = a
+            elif is_attr(a):
+                attr = a
+
+        return _Color(fg, bg, attr)

command.py

@@ -12,20 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import multiprocessing
-import os
 import optparse
+import os
 import re
-import sys
 
-from event_log import EventLog
-from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
+from error import NoSuchProjectError
+from error import RepoExitError
+from event_log import EventLog
 import progress
 
 
 # Are we generating man-pages?
-GENERATE_MANPAGES = os.environ.get('_REPO_GENERATE_MANPAGES_') == ' indeed! '
+GENERATE_MANPAGES = os.environ.get("_REPO_GENERATE_MANPAGES_") == " indeed! "
 
 
 # Number of projects to submit to a single worker process at a time.
@@ -42,403 +43,529 @@ WORKER_BATCH_SIZE = 32
 DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8)
 
 
-class Command(object):
+class UsageError(RepoExitError):
-  """Base class for any command line action in repo.
+    """Exception thrown with invalid command usage."""
-  """
 
-  # Singleton for all commands to track overall repo command execution and
-  # provide event summary to callers.  Only used by sync subcommand currently.
-  #
-  # NB: This is being replaced by git trace2 events.  See git_trace2_event_log.
-  event_log = EventLog()
 
-  # Whether this command is a "common" one, i.e. whether the user would commonly
+class Command:
-  # use it or it's a more uncommon command.  This is used by the help command to
+    """Base class for any command line action in repo."""
-  # show short-vs-full summaries.
-  COMMON = False
 
-  # Whether this command supports running in parallel.  If greater than 0,
+    # Singleton for all commands to track overall repo command execution and
-  # it is the number of parallel jobs to default to.
+    # provide event summary to callers. Only used by sync subcommand currently.
-  PARALLEL_JOBS = None
+    #
+    # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
+    event_log = EventLog()
 
-  # Whether this command supports Multi-manifest.  If False, then main.py will
+    # Whether this command is a "common" one, i.e. whether the user would
-  # iterate over the manifests and invoke the command once per (sub)manifest.
+    # commonly use it or it's a more uncommon command. This is used by the help
-  # This is only checked after calling ValidateOptions, so that partially
+    # command to show short-vs-full summaries.
-  # migrated subcommands can set it to False.
+    COMMON = False
-  MULTI_MANIFEST_SUPPORT = True
 
-  def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
+    # Whether this command supports running in parallel. If greater than 0,
-               git_event_log=None, outer_client=None, outer_manifest=None):
+    # it is the number of parallel jobs to default to.
-    self.repodir = repodir
+    PARALLEL_JOBS = None
-    self.client = client
-    self.outer_client = outer_client or client
-    self.manifest = manifest
-    self.gitc_manifest = gitc_manifest
-    self.git_event_log = git_event_log
-    self.outer_manifest = outer_manifest
 
-    # Cache for the OptionParser property.
+    # Whether this command supports Multi-manifest. If False, then main.py will
-    self._optparse = None
+    # iterate over the manifests and invoke the command once per (sub)manifest.
+    # This is only checked after calling ValidateOptions, so that partially
+    # migrated subcommands can set it to False.
+    MULTI_MANIFEST_SUPPORT = True
 
-  def WantPager(self, _opt):
+    # Shared data across parallel execution workers.
-    return False
+    _parallel_context = None
 
-  def ReadEnvironmentOptions(self, opts):
+    @classmethod
-    """ Set options from environment variables. """
+    def get_parallel_context(cls):
+        assert cls._parallel_context is not None
+        return cls._parallel_context
 
-    env_options = self._RegisteredEnvironmentOptions()
+    def __init__(
+        self,
+        repodir=None,
+        client=None,
+        manifest=None,
+        git_event_log=None,
+        outer_client=None,
+        outer_manifest=None,
+    ):
+        self.repodir = repodir
+        self.client = client
+        self.outer_client = outer_client or client
+        self.manifest = manifest
+        self.git_event_log = git_event_log
+        self.outer_manifest = outer_manifest
 
-    for env_key, opt_key in env_options.items():
+        # Cache for the OptionParser property.
-      # Get the user-set option value if any
+        self._optparse = None
-      opt_value = getattr(opts, opt_key)
 
-      # If the value is set, it means the user has passed it as a command
+    def WantPager(self, _opt):
-      # line option, and we should use that.  Otherwise we can try to set it
+        return False
-      # with the value from the corresponding environment variable.
-      if opt_value is not None:
-        continue
 
-      env_value = os.environ.get(env_key)
+    @staticmethod
-      if env_value is not None:
+    def is_multiprocessing_active() -> bool:
-        setattr(opts, opt_key, env_value)
+        """Whether the current process is a worker in a pool."""
+        return multiprocessing.current_process().name != "MainProcess"
 
-    return opts
+    def ReadEnvironmentOptions(self, opts):
+        """Set options from environment variables."""
 
-  @property
+        env_options = self._RegisteredEnvironmentOptions()
-  def OptionParser(self):
-    if self._optparse is None:
-      try:
-        me = 'repo %s' % self.NAME
-        usage = self.helpUsage.strip().replace('%prog', me)
-      except AttributeError:
-        usage = 'repo %s' % self.NAME
-      epilog = 'Run `repo help %s` to view the detailed manual.' % self.NAME
-      self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
-      self._CommonOptions(self._optparse)
-      self._Options(self._optparse)
-    return self._optparse
 
-  def _CommonOptions(self, p, opt_v=True):
+        for env_key, opt_key in env_options.items():
-    """Initialize the option parser with common options.
+            # Get the user-set option value if any
+            opt_value = getattr(opts, opt_key)
 
-    These will show up for *all* subcommands, so use sparingly.
+            # If the value is set, it means the user has passed it as a command
-    NB: Keep in sync with repo:InitParser().
+            # line option, and we should use that. Otherwise we can try to set
-    """
+            # it with the value from the corresponding environment variable.
-    g = p.add_option_group('Logging options')
+            if opt_value is not None:
-    opts = ['-v'] if opt_v else []
+                continue
-    g.add_option(*opts, '--verbose',
-                 dest='output_mode', action='store_true',
-                 help='show all output')
-    g.add_option('-q', '--quiet',
-                 dest='output_mode', action='store_false',
-                 help='only show errors')
 
-    if self.PARALLEL_JOBS is not None:
+            env_value = os.environ.get(env_key)
-      default = 'based on number of CPU cores'
+            if env_value is not None:
-      if not GENERATE_MANPAGES:
+                setattr(opts, opt_key, env_value)
-        # Only include active cpu count if we aren't generating man pages.
-        default = f'%default; {default}'
-      p.add_option(
-          '-j', '--jobs',
-          type=int, default=self.PARALLEL_JOBS,
-          help=f'number of jobs to run in parallel (default: {default})')
 
-    m = p.add_option_group('Multi-manifest options')
+        return opts
-    m.add_option('--outer-manifest', action='store_true', default=None,
-                 help='operate starting at the outermost manifest')
-    m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', help='do not operate on outer manifests')
-    m.add_option('--this-manifest-only', action='store_true', default=None,
-                 help='only operate on this (sub)manifest')
-    m.add_option('--no-this-manifest-only', '--all-manifests',
-                 dest='this_manifest_only', action='store_false',
-                 help='operate on this manifest and its submanifests')
 
-  def _Options(self, p):
+    @property
-    """Initialize the option parser with subcommand-specific options."""
+    def OptionParser(self):
+        if self._optparse is None:
+            try:
+                me = "repo %s" % self.NAME
+                usage = self.helpUsage.strip().replace("%prog", me)
+            except AttributeError:
+                usage = "repo %s" % self.NAME
+            epilog = (
+                "Run `repo help %s` to view the detailed manual." % self.NAME
+            )
+            self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
+            self._CommonOptions(self._optparse)
+            self._Options(self._optparse)
+        return self._optparse
 
-  def _RegisteredEnvironmentOptions(self):
+    def _CommonOptions(self, p, opt_v=True):
-    """Get options that can be set from environment variables.
+        """Initialize the option parser with common options.
 
-    Return a dictionary mapping environment variable name
+        These will show up for *all* subcommands, so use sparingly.
-    to option key name that it can override.
+        NB: Keep in sync with repo:InitParser().
+        """
+        g = p.add_option_group("Logging options")
+        opts = ["-v"] if opt_v else []
+        g.add_option(
+            *opts,
+            "--verbose",
+            dest="output_mode",
+            action="store_true",
+            help="show all output",
+        )
+        g.add_option(
+            "-q",
+            "--quiet",
+            dest="output_mode",
+            action="store_false",
+            help="only show errors",
+        )
 
-    Example: {'REPO_MY_OPTION': 'my_option'}
+        if self.PARALLEL_JOBS is not None:
+            default = "based on number of CPU cores"
+            if not GENERATE_MANPAGES:
+                # Only include active cpu count if we aren't generating man
+                # pages.
+                default = f"%default; {default}"
+            p.add_option(
+                "-j",
+                "--jobs",
+                type=int,
+                default=self.PARALLEL_JOBS,
+                help=f"number of jobs to run in parallel (default: {default})",
+            )
 
-    Will allow the option with key value 'my_option' to be set
+        m = p.add_option_group("Multi-manifest options")
-    from the value in the environment variable named 'REPO_MY_OPTION'.
+        m.add_option(
+            "--outer-manifest",
+            action="store_true",
+            default=None,
+            help="operate starting at the outermost manifest",
+        )
+        m.add_option(
+            "--no-outer-manifest",
+            dest="outer_manifest",
+            action="store_false",
+            help="do not operate on outer manifests",
+        )
+        m.add_option(
+            "--this-manifest-only",
+            action="store_true",
+            default=None,
+            help="only operate on this (sub)manifest",
+        )
+        m.add_option(
+            "--no-this-manifest-only",
+            "--all-manifests",
+            dest="this_manifest_only",
+            action="store_false",
+            help="operate on this manifest and its submanifests",
+        )
 
-    Note: This does not work properly for options that are explicitly
+    def _Options(self, p):
-    set to None by the user, or options that are defined with a
+        """Initialize the option parser with subcommand-specific options."""
-    default value other than None.
 
-    """
+    def _RegisteredEnvironmentOptions(self):
-    return {}
+        """Get options that can be set from environment variables.
 
-  def Usage(self):
+        Return a dictionary mapping environment variable name
-    """Display usage and terminate.
+        to option key name that it can override.
-    """
-    self.OptionParser.print_usage()
-    sys.exit(1)
 
-  def CommonValidateOptions(self, opt, args):
+        Example: {'REPO_MY_OPTION': 'my_option'}
-    """Validate common options."""
-    opt.quiet = opt.output_mode is False
-    opt.verbose = opt.output_mode is True
-    if opt.outer_manifest is None:
-      # By default, treat multi-manifest instances as a single manifest from
-      # the user's perspective.
-      opt.outer_manifest = True
 
-  def ValidateOptions(self, opt, args):
+        Will allow the option with key value 'my_option' to be set
-    """Validate the user options & arguments before executing.
+        from the value in the environment variable named 'REPO_MY_OPTION'.
 
-    This is meant to help break the code up into logical steps.  Some tips:
+        Note: This does not work properly for options that are explicitly
-    * Use self.OptionParser.error to display CLI related errors.
+        set to None by the user, or options that are defined with a
-    * Adjust opt member defaults as makes sense.
+        default value other than None.
-    * Adjust the args list, but do so inplace so the caller sees updates.
-    * Try to avoid updating self state.  Leave that to Execute.
-    """
 
-  def Execute(self, opt, args):
+        """
-    """Perform the action, after option parsing is complete.
+        return {}
-    """
-    raise NotImplementedError
 
-  @staticmethod
+    def Usage(self):
-  def ExecuteInParallel(jobs, func, inputs, callback, output=None, ordered=False):
+        """Display usage and terminate."""
-    """Helper for managing parallel execution boiler plate.
+        self.OptionParser.print_usage()
+        raise UsageError()
 
-    For subcommands that can easily split their work up.
+    def CommonValidateOptions(self, opt, args):
+        """Validate common options."""
+        opt.quiet = opt.output_mode is False
+        opt.verbose = opt.output_mode is True
+        if opt.outer_manifest is None:
+            # By default, treat multi-manifest instances as a single manifest
+            # from the user's perspective.
+            opt.outer_manifest = True
 
-    Args:
+    def ValidateOptions(self, opt, args):
-      jobs: How many parallel processes to use.
+        """Validate the user options & arguments before executing.
-      func: The function to apply to each of the |inputs|.  Usually a
-          functools.partial for wrapping additional arguments.  It will be run
-          in a separate process, so it must be pickalable, so nested functions
-          won't work.  Methods on the subcommand Command class should work.
-      inputs: The list of items to process.  Must be a list.
-      callback: The function to pass the results to for processing.  It will be
-          executed in the main thread and process the results of |func| as they
-          become available.  Thus it may be a local nested function.  Its return
-          value is passed back directly.  It takes three arguments:
-          - The processing pool (or None with one job).
-          - The |output| argument.
-          - An iterator for the results.
-      output: An output manager.  May be progress.Progess or color.Coloring.
-      ordered: Whether the jobs should be processed in order.
 
-    Returns:
+        This is meant to help break the code up into logical steps. Some tips:
-      The |callback| function's results are returned.
+        * Use self.OptionParser.error to display CLI related errors.
-    """
+        * Adjust opt member defaults as makes sense.
-    try:
+        * Adjust the args list, but do so inplace so the caller sees updates.
-      # NB: Multiprocessing is heavy, so don't spin it up for one job.
+        * Try to avoid updating self state. Leave that to Execute.
-      if len(inputs) == 1 or jobs == 1:
+        """
-        return callback(None, output, (func(x) for x in inputs))
-      else:
-        with multiprocessing.Pool(jobs) as pool:
-          submit = pool.imap if ordered else pool.imap_unordered
-          return callback(pool, output, submit(func, inputs, chunksize=WORKER_BATCH_SIZE))
-    finally:
-      if isinstance(output, progress.Progress):
-        output.end()
 
-  def _ResetPathToProjectMap(self, projects):
+    def Execute(self, opt, args):
-    self._by_path = dict((p.worktree, p) for p in projects)
+        """Perform the action, after option parsing is complete."""
+        raise NotImplementedError
 
-  def _UpdatePathToProjectMap(self, project):
+    @classmethod
-    self._by_path[project.worktree] = project
+    @contextlib.contextmanager
+    def ParallelContext(cls):
+        """Obtains the context, which is shared to ExecuteInParallel workers.
 
-  def _GetProjectByPath(self, manifest, path):
+        Callers can store data in the context dict before invocation of
-    project = None
+        ExecuteInParallel. The dict will then be shared to child workers of
-    if os.path.exists(path):
+        ExecuteInParallel.
-      oldpath = None
+        """
-      while (path and
+        assert cls._parallel_context is None
-             path != oldpath and
+        cls._parallel_context = {}
-             path != manifest.topdir):
         try:
-          project = self._by_path[path]
+            yield
-          break
+        finally:
-        except KeyError:
+            cls._parallel_context = None
-          oldpath = path
+
-          path = os.path.dirname(path)
+    @classmethod
-      if not project and path == manifest.topdir:
+    def _InitParallelWorker(cls, context, initializer):
+        cls._parallel_context = context
+        if initializer:
+            initializer()
+
+    @classmethod
+    def ExecuteInParallel(
+        cls,
+        jobs,
+        func,
+        inputs,
+        callback,
+        output=None,
+        ordered=False,
+        chunksize=WORKER_BATCH_SIZE,
+        initializer=None,
+    ):
+        """Helper for managing parallel execution boiler plate.
+
+        For subcommands that can easily split their work up.
+
+        Args:
+            jobs: How many parallel processes to use.
+            func: The function to apply to each of the |inputs|. Usually a
+                functools.partial for wrapping additional arguments. It will be
+                run in a separate process, so it must be pickalable, so nested
+                functions won't work. Methods on the subcommand Command class
+                should work.
+            inputs: The list of items to process. Must be a list.
+            callback: The function to pass the results to for processing. It
+                will be executed in the main thread and process the results of
+                |func| as they become available. Thus it may be a local nested
+                function. Its return value is passed back directly. It takes
+                three arguments:
+                - The processing pool (or None with one job).
+                - The |output| argument.
+                - An iterator for the results.
+            output: An output manager. May be progress.Progess or
+                color.Coloring.
+            ordered: Whether the jobs should be processed in order.
+            chunksize: The number of jobs processed in batch by parallel
+                workers.
+            initializer: Worker initializer.
+
+        Returns:
+            The |callback| function's results are returned.
+        """
         try:
-          project = self._by_path[path]
+            # NB: Multiprocessing is heavy, so don't spin it up for one job.
-        except KeyError:
+            if len(inputs) == 1 or jobs == 1:
-          pass
+                return callback(None, output, (func(x) for x in inputs))
-    else:
+            else:
-      try:
+                with multiprocessing.Pool(
-        project = self._by_path[path]
+                    jobs,
-      except KeyError:
+                    initializer=cls._InitParallelWorker,
-        pass
+                    initargs=(cls._parallel_context, initializer),
-    return project
+                ) as pool:
+                    submit = pool.imap if ordered else pool.imap_unordered
+                    return callback(
+                        pool,
+                        output,
+                        submit(func, inputs, chunksize=chunksize),
+                    )
+        finally:
+            if isinstance(output, progress.Progress):
+                output.end()
 
-  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+    def _ResetPathToProjectMap(self, projects):
-                  submodules_ok=False, all_manifests=False):
+        self._by_path = {p.worktree: p for p in projects}
-    """A list of projects that match the arguments.
 
-    Args:
+    def _UpdatePathToProjectMap(self, project):
-      args: a list of (case-insensitive) strings, projects to search for.
+        self._by_path[project.worktree] = project
-      manifest: an XmlManifest, the manifest to use, or None for default.
-      groups: a string, the manifest groups in use.
-      missing_ok: a boolean, whether to allow missing projects.
-      submodules_ok: a boolean, whether to allow submodules.
-      all_manifests: a boolean, if True then all manifests and submanifests are
-                     used.  If False, then only the local (sub)manifest is used.
 
-    Returns:
+    def _GetProjectByPath(self, manifest, path):
-      A list of matching Project instances.
+        project = None
-    """
+        if os.path.exists(path):
-    if all_manifests:
+            oldpath = None
-      if not manifest:
+            while path and path != oldpath and path != manifest.topdir:
-        manifest = self.manifest.outer_client
+                try:
-      all_projects_list = manifest.all_projects
+                    project = self._by_path[path]
-    else:
+                    break
-      if not manifest:
+                except KeyError:
-        manifest = self.manifest
+                    oldpath = path
-      all_projects_list = manifest.projects
+                    path = os.path.dirname(path)
-    result = []
+            if not project and path == manifest.topdir:
+                try:
+                    project = self._by_path[path]
+                except KeyError:
+                    pass
+        else:
+            try:
+                project = self._by_path[path]
+            except KeyError:
+                pass
+        return project
 
-    if not groups:
+    def GetProjects(
-      groups = manifest.GetGroupsStr()
+        self,
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+        args,
+        manifest=None,
+        groups="",
+        missing_ok=False,
+        submodules_ok=False,
+        all_manifests=False,
+    ):
+        """A list of projects that match the arguments.
 
-    if not args:
+        Args:
-      derived_projects = {}
+            args: a list of (case-insensitive) strings, projects to search for.
-      for project in all_projects_list:
+            manifest: an XmlManifest, the manifest to use, or None for default.
-        if submodules_ok or project.sync_s:
+            groups: a string, the manifest groups in use.
-          derived_projects.update((p.name, p)
+            missing_ok: a boolean, whether to allow missing projects.
-                                  for p in project.GetDerivedSubprojects())
+            submodules_ok: a boolean, whether to allow submodules.
-      all_projects_list.extend(derived_projects.values())
+            all_manifests: a boolean, if True then all manifests and
-      for project in all_projects_list:
+                submanifests are used. If False, then only the local
-        if (missing_ok or project.Exists) and project.MatchesGroups(groups):
+                (sub)manifest is used.
-          result.append(project)
-    else:
-      self._ResetPathToProjectMap(all_projects_list)
 
-      for arg in args:
+        Returns:
-        # We have to filter by manifest groups in case the requested project is
+            A list of matching Project instances.
-        # checked out multiple times or differently based on them.
+        """
-        projects = [project for project in manifest.GetProjectsWithName(
+        if all_manifests:
-                        arg, all_manifests=all_manifests)
+            if not manifest:
-                    if project.MatchesGroups(groups)]
+                manifest = self.manifest.outer_client
+            all_projects_list = manifest.all_projects
+        else:
+            if not manifest:
+                manifest = self.manifest
+            all_projects_list = manifest.projects
+        result = []
 
-        if not projects:
+        if not groups:
-          path = os.path.abspath(arg).replace('\\', '/')
+            groups = manifest.GetManifestGroupsStr()
-          tree = manifest
+        groups = [x for x in re.split(r"[,\s]+", groups) if x]
-          if all_manifests:
-            # Look for the deepest matching submanifest.
-            for tree in reversed(list(manifest.all_manifests)):
-              if path.startswith(tree.topdir):
-                break
-          project = self._GetProjectByPath(tree, path)
 
-          # If it's not a derived project, update path->project mapping and
+        if not args:
-          # search again, as arg might actually point to a derived subproject.
+            derived_projects = {}
-          if (project and not project.Derived and (submodules_ok or
+            for project in all_projects_list:
-                                                   project.sync_s)):
+                if submodules_ok or project.sync_s:
-            search_again = False
+                    derived_projects.update(
-            for subproject in project.GetDerivedSubprojects():
+                        (p.RelPath(local=False), p)
-              self._UpdatePathToProjectMap(subproject)
+                        for p in project.GetDerivedSubprojects()
-              search_again = True
+                    )
-            if search_again:
+            all_projects_list.extend(derived_projects.values())
-              project = self._GetProjectByPath(manifest, path) or project
+            for project in all_projects_list:
+                if (missing_ok or project.Exists) and project.MatchesGroups(
+                    groups
+                ):
+                    result.append(project)
+        else:
+            self._ResetPathToProjectMap(all_projects_list)
 
-          if project:
+            for arg in args:
-            projects = [project]
+                # We have to filter by manifest groups in case the requested
+                # project is checked out multiple times or differently based on
+                # them.
+                projects = [
+                    project
+                    for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests
+                    )
+                    if project.MatchesGroups(groups)
+                ]
 
-        if not projects:
+                if not projects:
-          raise NoSuchProjectError(arg)
+                    path = os.path.abspath(arg).replace("\\", "/")
+                    tree = manifest
+                    if all_manifests:
+                        # Look for the deepest matching submanifest.
+                        for tree in reversed(list(manifest.all_manifests)):
+                            if path.startswith(tree.topdir):
+                                break
+                    project = self._GetProjectByPath(tree, path)
 
-        for project in projects:
+                    # If it's not a derived project, update path->project
-          if not missing_ok and not project.Exists:
+                    # mapping and search again, as arg might actually point to
-            raise NoSuchProjectError('%s (%s)' % (
+                    # a derived subproject.
-                arg, project.RelPath(local=not all_manifests)))
+                    if (
-          if not project.MatchesGroups(groups):
+                        project
-            raise InvalidProjectGroupsError(arg)
+                        and not project.Derived
+                        and (submodules_ok or project.sync_s)
+                    ):
+                        search_again = False
+                        for subproject in project.GetDerivedSubprojects():
+                            self._UpdatePathToProjectMap(subproject)
+                            search_again = True
+                        if search_again:
+                            project = (
+                                self._GetProjectByPath(manifest, path)
+                                or project
+                            )
 
-        result.extend(projects)
+                    if project:
+                        projects = [project]
 
-    def _getpath(x):
+                if not projects:
-      return x.relpath
+                    raise NoSuchProjectError(arg)
-    result.sort(key=_getpath)
-    return result
 
-  def FindProjects(self, args, inverse=False, all_manifests=False):
+                for project in projects:
-    """Find projects from command line arguments.
+                    if not missing_ok and not project.Exists:
+                        raise NoSuchProjectError(
+                            "%s (%s)"
+                            % (arg, project.RelPath(local=not all_manifests))
+                        )
+                    if not project.MatchesGroups(groups):
+                        raise InvalidProjectGroupsError(arg)
 
-    Args:
+                result.extend(projects)
-      args: a list of (case-insensitive) strings, projects to search for.
-      inverse: a boolean, if True, then projects not matching any |args| are
-               returned.
-      all_manifests: a boolean, if True then all manifests and submanifests are
-                     used.  If False, then only the local (sub)manifest is used.
-    """
-    result = []
-    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
-    for project in self.GetProjects('', all_manifests=all_manifests):
-      paths = [project.name, project.RelPath(local=not all_manifests)]
-      for pattern in patterns:
-        match = any(pattern.search(x) for x in paths)
-        if not inverse and match:
-          result.append(project)
-          break
-        if inverse and match:
-          break
-      else:
-        if inverse:
-          result.append(project)
-    result.sort(key=lambda project: (project.manifest.path_prefix,
-                                     project.relpath))
-    return result
 
-  def ManifestList(self, opt):
+        def _getpath(x):
-    """Yields all of the manifests to traverse.
+            return x.relpath
 
-    Args:
+        result.sort(key=_getpath)
-      opt: The command options.
+        return result
-    """
+
-    top = self.outer_manifest
+    def FindProjects(self, args, inverse=False, all_manifests=False):
-    if not opt.outer_manifest or opt.this_manifest_only:
+        """Find projects from command line arguments.
-      top = self.manifest
+
-    yield top
+        Args:
-    if not opt.this_manifest_only:
+            args: a list of (case-insensitive) strings, projects to search for.
-      for child in top.all_children:
+            inverse: a boolean, if True, then projects not matching any |args|
-        yield child
+                are returned.
+            all_manifests: a boolean, if True then all manifests and
+                submanifests are used. If False, then only the local
+                (sub)manifest is used.
+        """
+        result = []
+        patterns = [re.compile(r"%s" % a, re.IGNORECASE) for a in args]
+        for project in self.GetProjects("", all_manifests=all_manifests):
+            paths = [project.name, project.RelPath(local=not all_manifests)]
+            for pattern in patterns:
+                match = any(pattern.search(x) for x in paths)
+                if not inverse and match:
+                    result.append(project)
+                    break
+                if inverse and match:
+                    break
+            else:
+                if inverse:
+                    result.append(project)
+        result.sort(
+            key=lambda project: (project.manifest.path_prefix, project.relpath)
+        )
+        return result
+
+    def GetSmartSyncOverridePath(self, manifest=None) -> str:
+        """Return the path where smart sync writes its override manifest."""
+        if manifest is None:
+            manifest = self.manifest
+        return os.path.join(
+            manifest.manifestProject.worktree, "smart_sync_override.xml"
+        )
+
+    def TryOverrideManifestWithSmartSync(self, manifest=None) -> None:
+        """Override manifest with smart_sync_override.xml if it exists."""
+        if manifest is None:
+            manifest = self.manifest
+        smart_sync_manifest_path = self.GetSmartSyncOverridePath(manifest)
+        if os.path.isfile(smart_sync_manifest_path):
+            manifest.Override(smart_sync_manifest_path)
+
+    def ManifestList(self, opt):
+        """Yields all of the manifests to traverse.
+
+        Args:
+            opt: The command options.
+        """
+        top = self.outer_manifest
+        if not opt.outer_manifest or opt.this_manifest_only:
+            top = self.manifest
+        yield top
+        if not opt.this_manifest_only:
+            yield from top.all_children
 
 
 class InteractiveCommand(Command):
-  """Command which requires user interaction on the tty and
+    """Command which requires user interaction on the tty and must not run
-     must not run within a pager, even if the user asks to.
+    within a pager, even if the user asks to.
-  """
+    """
 
-  def WantPager(self, _opt):
+    def WantPager(self, _opt):
-    return False
+        return False
 
 
 class PagedCommand(Command):
-  """Command which defaults to output in a pager, as its
+    """Command which defaults to output in a pager, as its display tends to be
-     display tends to be larger than one screen full.
+    larger than one screen full.
-  """
+    """
 
-  def WantPager(self, _opt):
+    def WantPager(self, _opt):
-    return True
+        return True
 
 
-class MirrorSafeCommand(object):
+class MirrorSafeCommand:
-  """Command permits itself to run within a mirror,
+    """Command permits itself to run within a mirror, and does not require a
-     and does not require a working directory.
+    working directory.
-  """
+    """
-
-
-class GitcAvailableCommand(object):
-  """Command that requires GITC to be available, but does
-     not require the local client to be a GITC client.
-  """
-
-
-class GitcClientCommand(object):
-  """Command that requires the local client to be a GITC
-     client.
-  """

completion.bash

@@ -1,4 +1,4 @@
-# Copyright 2021 The Android Open Source Project
+# Copyright (C) 2021 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

completion.zsh

@@ -0,0 +1,449 @@
+#compdef _repo repo
+
+# Copyright (C) 2026 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_repo() {
+  local context state line
+  typeset -A opt_args
+
+  local -a subcommands
+  subcommands=(
+    'abandon:Abandon a development branch'
+    'branches:View current topic branches'
+    'checkout:Checkout a branch'
+    'cherry-pick:Cherry-pick a change'
+    'diff:Show changes between commit and working tree'
+    'diffmanifests:Show differences between two manifests'
+    'download:Download and find a change'
+    'forall:Run a shell command in each project'
+    'gc:Garbage collect'
+    'grep:Print lines matching a pattern'
+    'help:Display help about a command'
+    'info:Get info about the manifest'
+    'init:Initialize a repo client'
+    'list:List projects and their names'
+    'manifest:Manifest management'
+    'overview:Display overview of topic branches'
+    'prune:Prune topic branches'
+    'rebase:Rebase local branches'
+    'selfupdate:Update repo'
+    'smartsync:Update working tree to latest known good revision'
+    'stage:Stage file(s) for commit'
+    'start:Start a new branch for development'
+    'status:Show the working tree status'
+    'sync:Update working tree'
+    'upload:Upload changes to the review server'
+    'version:Display version'
+    'wipe:Wipe uncommitted changes'
+  )
+
+  local -a global_opts
+  global_opts=(
+    '(-h --help)'{-h,--help}'[Show help message]'
+    '--help-all[Show help message for all commands]'
+    '(-p --paginate)'{-p,--paginate}'[Pipe all output into less]'
+    '--no-pager[Do not pipe output into a pager]'
+    '--color=[Control color output]:color:(always never auto)'
+    '--trace[Trace git command execution]'
+    '--trace-to-stderr[Trace git command execution to stderr]'
+    '--trace-python[Trace python execution]'
+    '--time[Time execute of command]'
+    '--version[Show version]'
+    '--show-toplevel[Show top level of workspace]'
+    '--event-log=[File to log events to]:file:_files'
+    '--git-trace2-event-log=[File to log git trace2 events to]:file:_files'
+    '--submanifest-path=[Path to submanifest]:path:_files'
+  )
+
+  _arguments -C \
+    ${global_opts} \
+    '1: :->cmds' \
+    '*:: :->subcmds'
+
+  case ${state} in
+  cmds)
+    _describe -t commands 'repo command' subcommands
+    ;;
+  subcmds)
+    local cmd=${words[1]}
+    curcontext="${curcontext%:*}-${cmd}:"
+
+    local -a common_opts
+    common_opts=(
+      '(-h --help)'{-h,--help}'[Show help message]'
+      '(-v --verbose)'{-v,--verbose}'[Show verbose messages]'
+      '(-q --quiet)'{-q,--quiet}'[Show only errors]'
+      '(-j --jobs)'{-j,--jobs=}'[Number of jobs to run in parallel]:jobs:'
+      '--outer-manifest[Use outer manifest]'
+      '--no-outer-manifest[Do not use outer manifest]'
+      '--this-manifest-only[Only use this manifest]'
+      '--no-this-manifest-only[Do not only use this manifest]'
+      '--all-manifests[Use all manifests]'
+    )
+
+    case ${cmd} in
+    abandon)
+      _arguments \
+        ${common_opts} \
+        '--all[Abandon all branches]' \
+        '1: :->branch' \
+        '*: :->project'
+      ;;
+    branches)
+      _arguments \
+        ${common_opts} \
+        '*: :->project'
+      ;;
+    checkout)
+      _arguments \
+        ${common_opts} \
+        '1: :->branch' \
+        '*: :->project'
+      ;;
+    cherry-pick)
+      _arguments \
+        ${common_opts} \
+        '1: :_message "sha1"'
+      ;;
+    diff)
+      _arguments \
+        ${common_opts} \
+        '(-u --absolute)'{-u,--absolute}'[Show absolute paths]' \
+        '*: :->project'
+      ;;
+    diffmanifests)
+      _arguments \
+        '--raw[Show raw diff]' \
+        '--no-color[Do not show color]' \
+        '--pretty-format=[Pretty format]:format:' \
+        '1: :_files -g "*.xml"' \
+        '2: :_files -g "*.xml"'
+      ;;
+    download)
+      _arguments \
+        ${common_opts} \
+        '(-b --branch)'{-b,--branch=}'[One or more branches to check]:branch:' \
+        '(-c --cherry-pick)'{-c,--cherry-pick}'[Cherry-pick the change]' \
+        '(-x --record-origin)'{-x,--record-origin}'[Record origin of cherry-pick]' \
+        '(-r --revert)'{-r,--revert}'[Revert the change]' \
+        '(-f --ff-only)'{-f,--ff-only}'[Force fast-forward]' \
+        '*: :_message "change[/patchset]"'
+      ;;
+    forall)
+      _arguments \
+        ${common_opts} \
+        '(-r --regex)'{-r,--regex}'[Execute command only on projects matching regex]' \
+        '(-i --inverse-regex)'{-i,--inverse-regex}'[Execute command only on projects not matching regex]' \
+        '(-g --groups)'{-g,--groups=}'[Execute command only on projects matching groups]:groups:' \
+        '(-c --command)'{-c,--command}'[Command to execute]' \
+        '(-e --abort-on-errors)'{-e,--abort-on-errors}'[Abort on errors]' \
+        '--ignore-missing[Ignore missing projects]' \
+        '--interactive[Run interactively]' \
+        '-p[Show project headers]' \
+        '*: :->project'
+      ;;
+    gc)
+      _arguments \
+        ${common_opts} \
+        '(-n --dry-run)'{-n,--dry-run}'[Dry run]' \
+        '(-y --yes)'{-y,--yes}'[Answer yes to all prompts]' \
+        '--repack[Repack objects]'
+      ;;
+    grep)
+      _arguments \
+        ${common_opts} \
+        '--cached[Search cached files]' \
+        '(-r --revision)'{-r,--revision=}'[Search in revision]:revision:' \
+        '-e[Pattern]' \
+        '(-i --ignore-case)'{-i,--ignore-case}'[Ignore case]' \
+        '(-a --text)'{-a,--text}'[Treat all files as text]' \
+        '-I[Do not match binary files]' \
+        '(-w --word-regexp)'{-w,--word-regexp}'[Match word boundaries]' \
+        '(-v --invert-match)'{-v,--invert-match}'[Invert match]' \
+        '(-G --basic-regexp)'{-G,--basic-regexp}'[Basic regexp]' \
+        '(-E --extended-regexp)'{-E,--extended-regexp}'[Extended regexp]' \
+        '(-F --fixed-strings)'{-F,--fixed-strings}'[Fixed strings]' \
+        '--all-match[All match]' \
+        '--and[And]' \
+        '--or[Or]' \
+        '--not[Not]' \
+        '-([Open paren]' \
+        '-)[Close paren]' \
+        '-n[Line number]' \
+        '-C[Context]:lines:' \
+        '-B[Before context]:lines:' \
+        '-A[After context]:lines:' \
+        '-l[Name only]' \
+        '--name-only[Name only]' \
+        '--files-with-matches[Files with matches]' \
+        '-L[Files without match]' \
+        '--files-without-match[Files without match]' \
+        '1: :->pattern' \
+        '*: :->project'
+      ;;
+    help)
+      _arguments \
+        '(-a --all)'{-a,--all}'[Show all commands]' \
+        '--help-all[Show help message for all commands]' \
+        '1: :->help_cmds'
+      ;;
+    info)
+      _arguments \
+        ${common_opts} \
+        '(-d --diff)'{-d,--diff}'[Show diff]' \
+        '(-o --overview)'{-o,--overview}'[Show overview]' \
+        '(-c --current-branch)'{-c,--current-branch}'[Show current branch]' \
+        '--no-current-branch[Do not show current branch]' \
+        '(-l --local-only)'{-l,--local-only}'[Local only]' \
+        '*: :->project'
+      ;;
+    init)
+      _arguments \
+        '(-u --manifest-url)'{-u,--manifest-url=}'[Manifest URL]:url:' \
+        '(-b --manifest-branch)'{-b,--manifest-branch=}'[Manifest branch]:branch:' \
+        '--manifest-upstream-branch=[Manifest upstream branch]:branch:' \
+        '(-m --manifest-name)'{-m,--manifest-name=}'[Manifest name]:file:_files -g "*.xml"' \
+        '(-g --groups)'{-g,--groups=}'[Restrict manifest projects to groups]:groups:' \
+        '(-p --platform)'{-p,--platform=}'[Restrict manifest projects to platform]:platform:' \
+        '--submodules[Sync submodules]' \
+        '--standalone-manifest[Standalone manifest]' \
+        '--manifest-depth=[Manifest depth]:depth:' \
+        '(-c --current-branch)'{-c,--current-branch}'[Sync current branch only]' \
+        '--no-current-branch[Do not sync current branch only]' \
+        '--tags[Sync tags]' \
+        '--no-tags[Do not sync tags]' \
+        '--mirror[Mirror]' \
+        '--archive[Archive]' \
+        '--worktree[Worktree]' \
+        '--reference=[Reference repository]:repository:_files -/' \
+        '--dissociate[Dissociate from reference]' \
+        '--depth=[Depth]:depth:' \
+        '--partial-clone[Partial clone]' \
+        '--no-partial-clone[Do not partial clone]' \
+        '--partial-clone-exclude=[Exclude from partial clone]:projects:' \
+        '--clone-filter=[Clone filter]:filter:' \
+        '--use-superproject[Use superproject]' \
+        '--no-use-superproject[Do not use superproject]' \
+        '--clone-bundle[Use clone bundle]' \
+        '--no-clone-bundle[Do not use clone bundle]' \
+        '--git-lfs[Use git lfs]' \
+        '--no-git-lfs[Do not use git lfs]' \
+        '--repo-url=[Repo URL]:url:' \
+        '--repo-rev=[Repo revision]:revision:' \
+        '--no-repo-verify[Do not verify repo]' \
+        '--config-name[Use config name]'
+      ;;
+    list)
+      _arguments \
+        ${common_opts} \
+        '(-r --regex)'{-r,--regex}'[Filter by regex]' \
+        '(-g --groups)'{-g,--groups=}'[Filter by groups]:groups:' \
+        '(-a --all)'{-a,--all}'[Show all projects]' \
+        '(-n --name-only)'{-n,--name-only}'[Show only names]' \
+        '(-p --path-only)'{-p,--path-only}'[Show only paths]' \
+        '(-f --fullpath)'{-f,--fullpath}'[Show full paths]' \
+        '--relative-to=[Show paths relative to]:path:_files -/' \
+        '*: :->project'
+      ;;
+    manifest)
+      _arguments \
+        '(-r --revision-as-HEAD)'{-r,--revision-as-HEAD}'[Save revisions as current HEAD]' \
+        '(-m --manifest-name)'{-m,--manifest-name=}'[Manifest name]:file:_files -g "*.xml"' \
+        '--suppress-upstream-revision[Suppress upstream revision]' \
+        '--suppress-dest-branch[Suppress dest branch]' \
+        '--format=[Output format]:format:(xml json)' \
+        '--pretty[Pretty print]' \
+        '--no-local-manifests[Ignore local manifests]' \
+        '(-o --output-file)'{-o,--output-file=}'[Output file]:file:_files'
+      ;;
+    overview)
+      _arguments \
+        ${common_opts} \
+        '(-c --current-branch)'{-c,--current-branch}'[Show current branch]' \
+        '--no-current-branch[Do not show current branch]' \
+        '*: :->project'
+      ;;
+    prune)
+      _arguments \
+        ${common_opts} \
+        '*: :->project'
+      ;;
+    rebase)
+      _arguments \
+        ${common_opts} \
+        '--fail-fast[Fail fast]' \
+        '(-f --force-rebase)'{-f,--force-rebase}'[Force rebase]' \
+        '--no-ff[No fast forward]' \
+        '--autosquash[Autosquash]' \
+        '--whitespace=[Whitespace option]:option:' \
+        '--auto-stash[Auto stash]' \
+        '(-m --onto-manifest)'{-m,--onto-manifest}'[Rebase onto manifest]' \
+        '(-i --interactive)'{-i,--interactive}'[Interactive rebase]' \
+        '*: :->project'
+      ;;
+    selfupdate)
+      _arguments \
+        '--no-repo-verify[Do not verify repo]'
+      ;;
+    smartsync | sync)
+      _arguments \
+        ${common_opts} \
+        '--jobs-network=[Number of network jobs]:jobs:' \
+        '--jobs-checkout=[Number of checkout jobs]:jobs:' \
+        '(-f --force-broken)'{-f,--force-broken}'[Continue sync even if a project fails]' \
+        '--fail-fast[Fail fast]' \
+        '--force-sync[Overwrite existing git directories]' \
+        '--force-checkout[Force checkout]' \
+        '--force-remove-dirty[Force remove dirty]' \
+        '--rebase[Rebase local branches]' \
+        '(-l --local-only)'{-l,--local-only}'[Only use local files]' \
+        '--no-manifest-update[Do not update manifest]' \
+        '--nmu[Do not update manifest]' \
+        '--interleaved[Interleave output]' \
+        '--no-interleaved[Do not interleave output]' \
+        '(-n --network-only)'{-n,--network-only}'[Only fetch]' \
+        '(-d --detach)'{-d,--detach}'[Detach HEAD]' \
+        '(-c --current-branch)'{-c,--current-branch}'[Fetch only current branch]' \
+        '--no-current-branch[Do not fetch only current branch]' \
+        '(-m --manifest-name)'{-m,--manifest-name=}'[Manifest name]:file:_files -g "*.xml"' \
+        '--clone-bundle[Use clone bundle]' \
+        '--no-clone-bundle[Do not use clone bundle]' \
+        '(-u --manifest-server-username)'{-u,--manifest-server-username=}'[Username for manifest server]:username:' \
+        '(-p --manifest-server-password)'{-p,--manifest-server-password=}'[Password for manifest server]:password:' \
+        '--fetch-submodules[Fetch submodules]' \
+        '--use-superproject[Use superproject]' \
+        '--no-use-superproject[Do not use superproject]' \
+        '--tags[Sync tags]' \
+        '--no-tags[Do not sync tags]' \
+        '--optimized-fetch[Optimized fetch]' \
+        '--retry-fetches=[Retry fetches]:count:' \
+        '--prune[Prune]' \
+        '--no-prune[Do not prune]' \
+        '--auto-gc[Run auto gc]' \
+        '--no-auto-gc[Do not run auto gc]' \
+        '(-s --smart-sync)'{-s,--smart-sync}'[Smart sync]' \
+        '(-t --smart-tag)'{-t,--smart-tag=}'[Smart tag]:tag:' \
+        '--no-repo-verify[Do not verify repo]' \
+        '--no-verify[Do not verify]' \
+        '--verify[Verify]' \
+        '--ignore-hooks[Ignore hooks]' \
+        '*: :->project'
+      ;;
+    stage)
+      _arguments \
+        ${common_opts} \
+        '(-i --interactive)'{-i,--interactive}'[Interactive staging]' \
+        '*: :->project'
+      ;;
+    start)
+      _arguments \
+        ${common_opts} \
+        '--all[Start branch in all projects]' \
+        '(-r --rev --revision)'{-r,--rev=,--revision=}'[Revision]:revision:' \
+        '--head[Head]' \
+        '--HEAD[Head]' \
+        '1: :->newbranch' \
+        '*: :->project'
+      ;;
+    status)
+      _arguments \
+        ${common_opts} \
+        '(-o --orphans)'{-o,--orphans}'[Show orphans]' \
+        '*: :->project'
+      ;;
+    upload)
+      _arguments \
+        ${common_opts} \
+        '(-t --topic-branch)'{-t,--topic-branch}'[Topic branch]' \
+        '--topic=[Topic]:topic:' \
+        '--hashtag=[Hashtag]:hashtag:' \
+        '--ht=[Hashtag]:hashtag:' \
+        '--hashtag-branch[Hashtag branch]' \
+        '--htb[Hashtag branch]' \
+        '(-l --label)'{-l,--label=}'[Label]:label:' \
+        '--pd=[Patchset description]:description:' \
+        '--patchset-description=[Patchset description]:description:' \
+        '--re=[Reviewers]:reviewers:' \
+        '--reviewers=[Reviewers]:reviewers:' \
+        '--cc=[CC]:cc:' \
+        '--br=[Branch]:branch:' \
+        '--branch=[Branch]:branch:' \
+        '(-c --current-branch)'{-c,--current-branch}'[Upload current branch]' \
+        '--no-current-branch[Do not upload current branch]' \
+        '--ne[No emails]' \
+        '--no-emails[No emails]' \
+        '(-p --private)'{-p,--private}'[Private]' \
+        '(-w --wip)'{-w,--wip}'[Work in progress]' \
+        '(-r --ready)'{-r,--ready}'[Ready]' \
+        '(-o --push-option)'{-o,--push-option=}'[Push option]:option:' \
+        '(-D --destination --dest)'{-D,--destination=,--dest=}'[Destination]:destination:' \
+        '(-n --dry-run)'{-n,--dry-run}'[Dry run]' \
+        '(-y --yes)'{-y,--yes}'[Answer yes to all prompts]' \
+        '--ignore-untracked-files[Ignore untracked files]' \
+        '--no-ignore-untracked-files[Do not ignore untracked files]' \
+        '--no-cert-checks[Do not check certificates]' \
+        '--no-verify[Do not verify]' \
+        '--verify[Verify]' \
+        '--ignore-hooks[Ignore hooks]' \
+        '*: :->project'
+      ;;
+    version)
+      _arguments ${common_opts}
+      ;;
+    wipe)
+      _arguments \
+        ${common_opts} \
+        '(-f --force)'{-f,--force}'[Force wipe]' \
+        '--force-uncommitted[Force uncommitted]' \
+        '--force-shared[Force shared]' \
+        '*: :->project'
+      ;;
+    esac
+
+    # Handle states for positional arguments.
+    case ${state} in
+    branch)
+      [[ ${PREFIX} != -* ]] && _repo_branches
+      ;;
+    project)
+      [[ ${PREFIX} != -* ]] && _repo_projects
+      ;;
+    pattern)
+      _message 'pattern'
+      ;;
+    newbranch)
+      _message 'new branch name'
+      ;;
+    help_cmds)
+      _describe -t commands 'repo command' subcommands
+      ;;
+    esac
+    ;;
+  esac
+}
+
+_repo_branches() {
+  local -a branches
+  branches=(${(f)"$(_call_program branches repo branches 2>/dev/null | sed -E 's/^.*[[:space:]]([^[:space:]]+)[[:space:]]+\|.*$/\1/' | awk '{print $1}')"})
+  _describe -t branches 'branch' branches
+}
+
+_repo_projects() {
+  local -a projects
+  projects=(${(f)"$(_call_program projects repo list -n 2>/dev/null)"})
+  _describe -t projects 'project' projects
+}
+
+_repo "$@"

constraints.txt

@@ -0,0 +1,2 @@
+# NB: Keep in sync with run_tests.vpython3.
+black<26

docs/internal-fs-layout.md

@@ -42,8 +42,12 @@ For example, if you want to change the manifest branch, you can simply run
     change the git URL/branch that this tracks, re-run `repo init` with the new
     settings.
 
-*   `.repo_fetchtimes.json`: Used by `repo sync` to record stats when syncing
+*   `.repo_fetchtimes.json`: Used by `repo sync` to record fetch times when
-    the various projects.
+    syncing the various projects.
+
+*   `.repo_localsyncstate.json`: Used by `repo sync` to detect and warn on
+    on partial tree syncs.  Partial syncs are allowed by `repo` itself, but are
+    unsupported by many projects where `repo` is used.
 
 ### Manifests
 
@@ -137,7 +141,7 @@ Instead, you should use standard Git workflows like [git worktree] or
     (e.g. a local mirror & a public review server) while avoiding duplicating
     the content.  However, this can run into problems if different remotes use
     the same path on their respective servers.  Best to avoid that.
-*   `subprojects/`: Like `projects/`, but for git submodules.
+*   `modules/`: Like `projects/`, but for git submodules.
 *   `subproject-objects/`: Like `project-objects/`, but for git submodules.
 *   `worktrees/`: Bare checkouts of every project synced by the manifest.  The
     filesystem layout matches the `<project name=...` setting in the manifest
@@ -222,27 +226,30 @@ The `[remote]` settings are automatically populated/updated from the manifest.
 
 The `[branch]` settings are updated by `repo start` and `git branch`.
 
-| Setting                       | Subcommands   | Use/Meaning |
+| Setting                               | Subcommands   | Use/Meaning |
-|-------------------------------|---------------|-------------|
+|---------------------------------------|---------------|-------------|
-| review.\<url\>.autocopy       | upload        | Automatically add to `--cc=<value>` |
+| review.\<url\>.autocopy               | upload        | Automatically add to `--cc=<value>` |
-| review.\<url\>.autoreviewer   | upload        | Automatically add to `--reviewers=<value>` |
+| review.\<url\>.autoreviewer           | upload        | Automatically add to `--reviewers=<value>` |
-| review.\<url\>.autoupload     | upload        | Automatically answer "yes" or "no" to all prompts |
+| review.\<url\>.autoupload             | upload        | Automatically answer "yes" or "no" to all prompts |
-| review.\<url\>.uploadhashtags | upload        | Automatically add to `--hashtag=<value>` |
+| review.\<url\>.uploadhashtags         | upload        | Automatically add to `--hashtag=<value>` |
-| review.\<url\>.uploadlabels   | upload        | Automatically add to `--label=<value>` |
+| review.\<url\>.uploadlabels           | upload        | Automatically add to `--label=<value>` |
-| review.\<url\>.uploadnotify   | upload        | [Notify setting][upload-notify] to use |
+| review.\<url\>.uploadnotify           | upload        | [Notify setting][upload-notify] to use |
-| review.\<url\>.uploadtopic    | upload        | Default [topic] to use |
+| review.\<url\>.uploadtopic            | upload        | Default [topic] to use |
-| review.\<url\>.username       | upload        | Override username with `ssh://` review URIs |
+| review.\<url\>.uploadwarningthreshold | upload        | Warn when attempting to upload more than this many CLs |
-| remote.\<remote\>.fetch       | sync          | Set of refs to fetch |
+| review.\<url\>.username               | upload        | Override username with `ssh://` review URIs |
-| remote.\<remote\>.projectname | \<network\>   | The name of the project as it exists in Gerrit review |
+| remote.\<remote\>.fetch               | sync          | Set of refs to fetch |
-| remote.\<remote\>.pushurl     | upload        | The base URI for pushing CLs |
+| remote.\<remote\>.projectname         | \<network\>   | The name of the project as it exists in Gerrit review |
-| remote.\<remote\>.review      | upload        | The URI of the Gerrit review server |
+| remote.\<remote\>.pushurl             | upload        | The base URI for pushing CLs |
-| remote.\<remote\>.url         | sync & upload | The URI of the git project to fetch |
+| remote.\<remote\>.review              | upload        | The URI of the Gerrit review server |
-| branch.\<branch\>.merge       | sync & upload | The branch to merge & upload & track |
+| remote.\<remote\>.url                 | sync & upload | The URI of the git project to fetch |
-| branch.\<branch\>.remote      | sync & upload | The remote to track |
+| branch.\<branch\>.merge               | sync & upload | The branch to merge & upload & track |
+| branch.\<branch\>.remote              | sync & upload | The remote to track |
 
 ## ~/ dotconfig layout
 
-Repo will create & maintain a few files in the user's home directory.
+Repo will create & maintain a few files under the `.repoconfig/` directory.
+This is placed in the user's home directory by default but can be changed by
+setting `REPO_CONFIG_DIR`.
 
 *   `.repoconfig/`: Repo's per-user directory for all random config files/state.
 *   `.repoconfig/config`: Per-user settings using [git-config] file format.

docs/manifest-format.md

@@ -51,6 +51,7 @@ following DTD:
   <!ATTLIST default dest-branch CDATA #IMPLIED>
   <!ATTLIST default upstream    CDATA #IMPLIED>
   <!ATTLIST default sync-j      CDATA #IMPLIED>
+  <!ATTLIST default sync-j-max  CDATA #IMPLIED>
   <!ATTLIST default sync-c      CDATA #IMPLIED>
   <!ATTLIST default sync-s      CDATA #IMPLIED>
   <!ATTLIST default sync-tags   CDATA #IMPLIED>
@@ -59,7 +60,7 @@ following DTD:
   <!ATTLIST manifest-server url CDATA #REQUIRED>
 
   <!ELEMENT submanifest EMPTY>
-  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest name           ID    #REQUIRED>
   <!ATTLIST submanifest remote         IDREF #IMPLIED>
   <!ATTLIST submanifest project        CDATA #IMPLIED>
   <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
@@ -72,18 +73,19 @@ following DTD:
                      project*,
                      copyfile*,
                      linkfile*)>
-  <!ATTLIST project name        CDATA #REQUIRED>
+  <!ATTLIST project name          CDATA #REQUIRED>
-  <!ATTLIST project path        CDATA #IMPLIED>
+  <!ATTLIST project path          CDATA #IMPLIED>
-  <!ATTLIST project remote      IDREF #IMPLIED>
+  <!ATTLIST project remote        IDREF #IMPLIED>
-  <!ATTLIST project revision    CDATA #IMPLIED>
+  <!ATTLIST project revision      CDATA #IMPLIED>
-  <!ATTLIST project dest-branch CDATA #IMPLIED>
+  <!ATTLIST project dest-branch   CDATA #IMPLIED>
-  <!ATTLIST project groups      CDATA #IMPLIED>
+  <!ATTLIST project groups        CDATA #IMPLIED>
-  <!ATTLIST project sync-c      CDATA #IMPLIED>
+  <!ATTLIST project sync-c        CDATA #IMPLIED>
-  <!ATTLIST project sync-s      CDATA #IMPLIED>
+  <!ATTLIST project sync-s        CDATA #IMPLIED>
-  <!ATTLIST project sync-tags   CDATA #IMPLIED>
+  <!ATTLIST project sync-tags     CDATA #IMPLIED>
-  <!ATTLIST project upstream CDATA #IMPLIED>
+  <!ATTLIST project upstream      CDATA #IMPLIED>
-  <!ATTLIST project clone-depth CDATA #IMPLIED>
+  <!ATTLIST project clone-depth   CDATA #IMPLIED>
-  <!ATTLIST project force-path CDATA #IMPLIED>
+  <!ATTLIST project force-path    CDATA #IMPLIED>
+  <!ATTLIST project sync-strategy CDATA #IMPLIED>
 
   <!ELEMENT annotation EMPTY>
   <!ATTLIST annotation name  CDATA #REQUIRED>
@@ -95,23 +97,30 @@ following DTD:
   <!ATTLIST copyfile dest CDATA #REQUIRED>
 
   <!ELEMENT linkfile EMPTY>
-  <!ATTLIST linkfile src CDATA #REQUIRED>
+  <!ATTLIST linkfile src  CDATA #REQUIRED>
   <!ATTLIST linkfile dest CDATA #REQUIRED>
 
-  <!ELEMENT extend-project EMPTY>
+  <!ELEMENT extend-project (annotation*,
-  <!ATTLIST extend-project name CDATA #REQUIRED>
+                            copyfile*,
-  <!ATTLIST extend-project path CDATA #IMPLIED>
+                            linkfile*)>
-  <!ATTLIST extend-project dest-path CDATA #IMPLIED>
+  <!ATTLIST extend-project name        CDATA #REQUIRED>
-  <!ATTLIST extend-project groups CDATA #IMPLIED>
+  <!ATTLIST extend-project path        CDATA #IMPLIED>
-  <!ATTLIST extend-project revision CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-path   CDATA #IMPLIED>
-  <!ATTLIST extend-project remote CDATA #IMPLIED>
+  <!ATTLIST extend-project groups      CDATA #IMPLIED>
+  <!ATTLIST extend-project revision    CDATA #IMPLIED>
+  <!ATTLIST extend-project remote      CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
+  <!ATTLIST extend-project upstream    CDATA #IMPLIED>
+  <!ATTLIST extend-project base-rev    CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
-  <!ATTLIST remove-project name  CDATA #REQUIRED>
+  <!ATTLIST remove-project name     CDATA #IMPLIED>
-  <!ATTLIST remove-project optional  CDATA #IMPLIED>
+  <!ATTLIST remove-project path     CDATA #IMPLIED>
+  <!ATTLIST remove-project optional CDATA #IMPLIED>
+  <!ATTLIST remove-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT repo-hooks EMPTY>
-  <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
+  <!ATTLIST repo-hooks in-project   CDATA #REQUIRED>
   <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
 
   <!ELEMENT superproject EMPTY>
@@ -120,11 +129,12 @@ following DTD:
   <!ATTLIST superproject revision CDATA #IMPLIED>
 
   <!ELEMENT contactinfo EMPTY>
-  <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
+  <!ATTLIST contactinfo bugurl CDATA #REQUIRED>
 
   <!ELEMENT include EMPTY>
-  <!ATTLIST include name   CDATA #REQUIRED>
+  <!ATTLIST include name     CDATA #REQUIRED>
-  <!ATTLIST include groups CDATA #IMPLIED>
+  <!ATTLIST include groups   CDATA #IMPLIED>
+  <!ATTLIST include revision CDATA #IMPLIED>
 ]>
 ```
 
@@ -205,7 +215,9 @@ can be found.  Used when syncing a revision locked manifest in
 -c mode to avoid having to sync the entire ref space. Project elements
 not setting their own `upstream` will inherit this value.
 
-Attribute `sync-j`: Number of parallel jobs to use when synching.
+Attribute `sync-j`: Number of parallel jobs to use when syncing.
+
+Attribute `sync-j-max`: Maximum number of parallel jobs to use when syncing.
 
 Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
@@ -225,26 +237,7 @@ At most one manifest-server may be specified. The url attribute
 is used to specify the URL of a manifest server, which is an
 XML RPC service.
 
-The manifest server should implement the following RPC methods:
+See the [smart sync documentation](./smart-sync.md) for more details.
-
-    GetApprovedManifest(branch, target)
-
-Return a manifest in which each project is pegged to a known good revision
-for the current branch and target. This is used by repo sync when the
---smart-sync option is given.
-
-The target to use is defined by environment variables TARGET_PRODUCT
-and TARGET_BUILD_VARIANT. These variables are used to create a string
-of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
-If one of those variables or both are not present, the program will call
-GetApprovedManifest without the target parameter and the manifest server
-should choose a reasonable default target.
-
-    GetManifest(tag)
-
-Return a manifest in which each project is pegged to the revision at
-the specified tag. This is used by repo sync when the --smart-tag option
-is given.
 
 
 ### Element submanifest
@@ -298,7 +291,7 @@ should be placed.  If not supplied, `revision` is used.
 
 `path` may not be an absolute path or use "." or ".." path components.
 
-Attribute `groups`: List of additional groups to which all projects
+Attribute `groups`: Set of additional groups to which all projects
 in the included submanifest belong. This appends and recurses, meaning
 all projects in submanifests carry all parent submanifest groups.
 Same syntax as the corresponding element of `project`.
@@ -366,7 +359,7 @@ When using `repo upload`, changes will be submitted for code
 review on this branch. If unspecified both here and in the
 default element, `revision` is used instead.
 
-Attribute `groups`: List of groups to which this project belongs,
+Attribute `groups`: Set of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
 "all", and each project automatically belongs to a group of
 its name:`name` and path:`path`.  E.g. for
@@ -397,6 +390,22 @@ rather than the `name` attribute.  This attribute only applies to the
 local mirrors syncing, it will be ignored when syncing the projects in a
 client working directory.
 
+Attribute `sync-strategy`: Set the sync strategy used when fetching this
+project.  Currently the only supported value is `stateless`.  When set to
+`stateless`, repo will run a reflog expiration and aggressive garbage collection
+at the end of the sync process.  This is useful for projects that contain
+large binary files and use `clone-depth="1"`, where garbage can accumulate
+as binaries are added, deleted, or modified across successive syncs.
+
+During a stateless sync, repo checks the following before cleaning up:
+1. The project does not share an object directory with other projects.
+2. The working tree is clean (no uncommitted changes, no untracked files).
+3. There are no unpushed local commits.
+4. There is no Git stash.
+
+If any of these conditions are not met, repo falls back to a standard
+sync without garbage collection.
+
 ### Element extend-project
 
 Modify the attributes of the named project.
@@ -406,6 +415,11 @@ attributes of an existing project without completely replacing the
 existing project definition.  This makes the local manifest more robust
 against changes to the original manifest.
 
+The `extend-project` element can also contain `annotation`, `copyfile`, and
+`linkfile` child elements. These are added to the project's definition. A
+`copyfile` or `linkfile` with a `dest` that already exists in the project
+will overwrite the original.
+
 Attribute `path`: If specified, limit the change to projects checked out
 at the specified path, rather than all projects with the given name.
 
@@ -414,7 +428,7 @@ of the repo client where the Git working directory for this project
 should be placed.  This is used to move a project in the checkout by
 overriding the existing `path` setting.
 
-Attribute `groups`: List of additional groups to which this project
+Attribute `groups`: Set of additional groups to which this project
 belongs.  Same syntax as the corresponding element of `project`.
 
 Attribute `revision`: If specified, overrides the revision of the original
@@ -423,22 +437,37 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `dest-branch`: If specified, overrides the dest-branch of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `upstream`: If specified, overrides the upstream of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `base-rev`: If specified, adds a check against the revision
+to be extended. Manifest parse will fail and give a list of mismatch extends
+if the revisions being extended have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
-project or remote element. Each element describes a name-value pair.
+project element, an extend-project element, or a remote element. Each
-For projects, this name-value pair will be exported into each project's
+element describes a name-value pair. For projects, this name-value pair
-environment during a 'forall' command, prefixed with `REPO__`.  In addition,
+will be exported into each project's environment during a 'forall'
-there is an optional attribute "keep" which accepts the case insensitive values
+command, prefixed with `REPO__`.  In addition, there is an optional
-"true" (default) or "false".  This attribute determines whether or not the
+attribute "keep" which accepts the case insensitive values "true"
+(default) or "false". This attribute determines whether or not the
 annotation will be kept when exported with the manifest subcommand.
 
 ### Element copyfile
 
 Zero or more copyfile elements may be specified as children of a
-project element. Each element describes a src-dest pair of files;
+project element, or an extend-project element. Each element describes a
-the "src" file will be copied to the "dest" place during `repo sync`
+src-dest pair of files; the "src" file will be copied to the "dest"
-command.
+place during `repo sync` command.
 
 "src" is project relative, "dest" is relative to the top of the tree.
 Copying from paths outside of the project or to paths outside of the repo
@@ -449,10 +478,14 @@ Intermediate paths must not be symlinks either.
 
 Parent directories of "dest" will be automatically created if missing.
 
+The files are copied in the order they are specified in the manifests.
+If multiple elements specify the same source and destination, they will
+only be applied as one, based on the first occurence. Files are copied
+before any links specified via linkfile elements are created.
+
 ### Element linkfile
 
-It's just like copyfile and runs at the same time as copyfile but
+It's just like copyfile, but instead of copying it creates a symlink.
-instead of copying it creates a symlink.
 
 The symlink is created at "dest" (relative to the top of the tree) and
 points to the path specified by "src" which is a path in the project.
@@ -462,9 +495,14 @@ Parent directories of "dest" will be automatically created if missing.
 The symlink target may be a file or directory, but it may not point outside
 of the repo client.
 
+The links are created in the order they are specified in the manifests.
+If multiple elements specify the same source and destination, they will
+only be applied as one, based on the first occurence. Links are created
+after any files specified via copyfile elements are copied.
+
 ### Element remove-project
 
-Deletes the named project from the internal manifest table, possibly
+Deletes a project from the internal manifest table, possibly
 allowing a subsequent project element in the same manifest file to
 replace the project with a different source.
 
@@ -472,9 +510,28 @@ This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
+The project `name` or project `path` can be used to specify the remove target
+meaning one of them is required. If only name is specified, all
+projects with that name are removed.
+
+If both name and path are specified, only projects with the same name and
+path are removed, meaning projects with the same name but in other
+locations are kept.
+
+If only path is specified, a matching project is removed regardless of its
+name. Logic otherwise behaves like both are specified.
+
 Attribute `optional`: Set to true to ignore remove-project elements with no
 matching `project` element.
 
+Attribute `base-rev`: If specified, adds a check against the revision
+to be removed. Manifest parse will fail and give a list of mismatch removes
+if the revisions being removed have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element repo-hooks
 
 NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.
@@ -540,11 +597,17 @@ the manifest repository's root.
 "name" may not be an absolute path or use "." or ".." path components.
 These restrictions are not enforced for [Local Manifests].
 
-Attribute `groups`: List of additional groups to which all projects
+Attribute `groups`: Set of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
 all projects in included manifests carry all parent include groups.
+This also applies to all extend-project elements in the included manifests.
 Same syntax as the corresponding element of `project`.
 
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`)
+default to which all projects in the included manifest belong. This recurses,
+meaning it will apply to all projects in all manifests included as a result of
+this element.
+
 ## Local Manifests {#local-manifests}
 
 Additional remotes and projects may be added through local manifest

docs/python-support.md

@@ -1,47 +1,92 @@
 # Supported Python Versions
 
-With Python 2.7 officially going EOL on [01 Jan 2020](https://pythonclock.org/),
+This documents the current supported Python versions, and tries to provide
-we need a support plan for the repo project itself.
+guidance for when we decide to drop support for older versions.
-Inevitably, there will be a long tail of users who still want to use Python 2 on
-their old LTS/corp systems and have little power to change the system.
 
 ## Summary
 
-* Python 3.6 (released Dec 2016) is required by default starting with repo-2.x.
+*   Python 3.6 (released Dec 2016) is required starting with repo-2.0.
-* Older versions of Python (e.g. v2.7) may use the legacy feature-frozen branch
+*   Older versions of Python (e.g. v2.7) may use old releases via the repo-1.x
-  based on repo-1.x.
+    branch, but no support is provided.
 
-## Overview
+## repo hooks
-
-We provide a branch for Python 2 users that is feature-frozen.
-Bugfixes may be added on a best-effort basis or from the community, but largely
-no new features will be added, nor is support guaranteed.
-
-Users can select this during `repo init` time via the [repo launcher].
-Otherwise the default branches (e.g. stable & main) will be used which will
-require Python 3.
-
-This means the [repo launcher] needs to support both Python 2 & Python 3, but
-since it doesn't import any other repo code, this shouldn't be too problematic.
-
-The main branch will require Python 3.6 at a minimum.
-If the system has an older version of Python 3, then users will have to select
-the legacy Python 2 branch instead.
-
-### repo hooks
 
 Projects that use [repo hooks] run on independent schedules.
-They might migrate to Python 3 earlier or later than us.
+Since it's not possible to detect what version of Python the hooks were written
-To support them, we'll probe the shebang of the hook script and if we find an
+or tested against, we always import & exec them with the active Python version.
-interpreter in there that indicates a different version than repo is currently
-running under, we'll attempt to reexec ourselves under that.
 
-For example, a hook with a header like `#!/usr/bin/python2` will have repo
+If the user's Python is too new for the [repo hooks], then it is up to the hooks
-execute `/usr/bin/python2` to execute the hook code specifically if repo is
+maintainer to update.
-currently running Python 3.
 
-For more details, consult the [repo hooks] documentation.
+## Repo launcher
+
+The [repo launcher] is an independent script that can support older versions of
+Python without holding back the rest of the codebase.
+If it detects the current version of Python is too old, it will try to reexec
+via a newer version of Python via standard `pythonX.Y` interpreter names.
+
+However, this is provided as a nicety when it is not onerous, and there is no
+official support for older versions of Python than the rest of the codebase.
+
+If your default python interpreters are too old to run the launcher even though
+you have newer versions installed, your choices are:
+
+*   Modify the [repo launcher]'s shebang to suite your environment.
+*   Download an older version of the [repo launcher] and don't upgrade it.
+    Be aware that we do not guarantee old repo launchers will work with current
+    versions of repo.  Bug reports using old launchers will not be accepted.
+
+## When to drop support
+
+So far, Python 3.6 has provided most of the interesting features that we want
+(e.g. typing & f-strings), and there haven't been features in newer versions
+that are critical to us.
+
+That said, let's assume we need functionality that only exists in Python 3.7.
+How do we decide when it's acceptable to drop Python 3.6?
+
+1.  Review the [Project References](./release-process.md#project-references) to
+    see what major distros are using the previous version of Python, and when
+    they go EOL.  Generally we care about Ubuntu LTS & current/previous Debian
+    stable versions.
+    *   If they're all EOL already, then go for it, drop support.
+    *   If they aren't EOL, start a thread on [repo-discuss] to see how the user
+        base feels about the proposal.
+1.  Update the "soft" versions in the codebase.  This will start warning users
+    that the older version is deprecated.
+    *   Update [repo](/repo) if the launcher needs updating.
+        This only helps with people who download newer launchers.
+    *   Update [main.py](/main.py) for the main codebase.
+        This warns for everyone regardless of [repo launcher] version.
+    *   Update [requirements.json](/requirements.json).
+        This allows [repo launcher] to display warnings/errors without having
+        to execute the new codebase.  This helps in case of syntax or module
+        changes where older versions won't even be able to import the new code.
+1.  After some grace period (ideally at least 2 quarters after the first release
+    with the updated soft requirements), update the "hard" versions, and then
+    start using the new functionality.
+
+## Python 2.7 & 3.0-3.5
+
+> **There is no support for these versions.**
+> **Do not file bugs if you are using old Python versions.**
+> **Any such reports will be marked invalid and ignored.**
+> **Upgrade your distro and/or runtime instead.**
+
+Fetch an old version of the [repo launcher]:
+
+```sh
+$ curl https://storage.googleapis.com/git-repo-downloads/repo-2.32 > ~/.bin/repo-2.32
+$ chmod a+rx ~/.bin/repo-2.32
+```
+
+Then initialize an old version of repo:
+
+```sh
+$ repo-2.32 init --repo-rev=repo-1 ...
+```
 
 
+[repo-discuss]: https://groups.google.com/forum/#!forum/repo-discuss
 [repo hooks]: ./repo-hooks.md
 [repo launcher]: ../repo

docs/release-process.md

@@ -96,6 +96,9 @@ If that tag is valid, then repo will warn and use that commit instead.
 
 If that tag cannot be verified, it gives up and forces the user to resolve.
 
+If env variable `REPO_SKIP_SELF_UPDATE` is defined, this will
+bypass the self update algorithm.
+
 ### Force an update
 
 The `repo selfupdate` command can be used to force an immediate update.
@@ -143,23 +146,14 @@ internal processes for accessing the restricted keys.
 ***
 
 ```sh
-# Set the gpg key directory.
-$ export GNUPGHOME=~/.gnupg/repo/
-
-# Verify the listed key is “Repo Maintainer”.
-$ gpg -K
-
-# Pick whatever branch or commit you want to tag.
-$ r=main
-
 # Pick the new version.
-$ t=1.12.10
+$ t=v2.30
 
-# Create the signed tag.
+# Create a new signed tag with the current HEAD.
-$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+$ ./release/sign-tag.py $t
 
 # Verify the signed tag.
-$ git show v$t
+$ git show $t
 ```
 
 ### Push the new release
@@ -168,11 +162,11 @@ Once you're ready to make the release available to everyone, push it to the
 `stable` branch.
 
 Make sure you never push the tag itself to the stable branch!
-Only push the commit -- notice the use of `$t` and `$r` below.
+Only push the commit -- note the use of `^0` below.
 
 ```sh
-$ git push https://gerrit-review.googlesource.com/git-repo v$t
+$ git push https://gerrit-review.googlesource.com/git-repo $t
-$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+$ git push https://gerrit-review.googlesource.com/git-repo $t^0:stable
 ```
 
 If something goes horribly wrong, you can force push the previous version to the
@@ -195,7 +189,9 @@ You can create a short changelog using the command:
 ```sh
 # If you haven't pushed to the stable branch yet, you can use origin/stable.
 # If you have pushed, change origin/stable to the previous release tag.
-$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+# This assumes "main" is the current tagged release.  If it's newer, change it
+# to the current release tag too.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..main
 ```
 
 ## Project References
@@ -209,7 +205,7 @@ still support them.
 Things in italics are things we used to care about but probably don't anymore.
 
 |   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
-|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|-----|--------|-----|
+|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|:---:|:------:|:---:|
 | Apr 2008 |              |              |                 | 5.0          |
 | Jun 2008 |              |              |                 | 5.1          |
 | Oct 2008 | *Oct 2013*   |              | 2.6.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
@@ -248,7 +244,7 @@ Things in italics are things we used to care about but probably don't anymore.
 | Feb 2014 | *Dec 2014*   | **1.9.0**    |                 |              | *14.04 Trusty* |
 | Mar 2014 | *Mar 2019*   |              | *3.4.0*         |              | *14.04 Trusty* - 15.10 Wily / *Jessie* |
 | Mar 2014 |              |              |                 | 6.6          | *14.04 Trusty* - 14.10 Utopic |
-| Apr 2014 | *Apr 2022*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
+| Apr 2014 | *Apr 2024*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
 | May 2014 | *Dec 2014*   | 2.0.0        |
 | Aug 2014 | *Dec 2014*   | *2.1.0*      |                 |              | 14.10 Utopic - 15.04 Vivid / *Jessie* |
 | Oct 2014 |              |              |                 | 6.7          | 15.04 Vivid |
@@ -269,7 +265,7 @@ Things in italics are things we used to care about but probably don't anymore.
 | Jan 2016 | *Jul 2017*   | *2.7.0*      |                 |              | *16.04 Xenial* |
 | Feb 2016 |              |              |                 | 7.2          | *16.04 Xenial* |
 | Mar 2016 | *Jul 2017*   | 2.8.0        |
-| Apr 2016 | *Apr 2024*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
+| Apr 2016 | *Apr 2026*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
 | Jun 2016 | *Jul 2017*   | 2.9.0        |                 |              | 16.10 Yakkety |
 | Jul 2016 |              |              |                 | 7.3          | 16.10 Yakkety |
 | Sep 2016 | *Sep 2017*   | 2.10.0       |
@@ -291,7 +287,7 @@ Things in italics are things we used to care about but probably don't anymore.
 | Apr 2018 |              |              |                 | 7.7          | 18.10 Cosmic |
 | Apr 2018 | **Apr 2028** |              |                 |              | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 | 7.6 |
 | Jun 2018 | *Mar 2021*   | 2.18.0       |
-| Jun 2018 | **Jun 2023** |              | 3.7.0           |              | 19.04 Disco - **20.04 Focal** / **Buster** |
+| Jun 2018 | **Jun 2023** |              | 3.7.0           |              | 19.04 Disco - **Buster** |
 | Aug 2018 |              |              |                 | 7.8          |
 | Sep 2018 | *Mar 2021*   | 2.19.0       |                 |              | 18.10 Cosmic |
 | Oct 2018 |              |              |                 | 7.9          | 19.04 Disco / **Buster** |
@@ -319,14 +315,33 @@ Things in italics are things we used to care about but probably don't anymore.
 | Oct 2020 |              |              |                 |              | 20.10 Groovy         | 2.27.0   | 2.7.18 3.8.6 | 8.3 |
 | Oct 2020 | **Oct 2025** |              | 3.9.0           |              | 21.04 Hirsute / **Bullseye** |
 | Dec 2020 | *Mar 2021*   | 2.30.0       |                 |              | 21.04 Hirsute / **Bullseye** |
-| Mar 2021 |              | 2.31.0       |
+| Mar 2021 |              | 2.31.0       |                 | 8.5          |
-| Mar 2021 |              |              |                 | 8.5          |
 | Apr 2021 |              |              |                 | 8.6          |
 | Apr 2021 | *Jan 2022*   |              |                 |              | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 | 8.4 |
 | Jun 2021 |              | 2.32.0       |
-| Aug 2021 |              | 2.33.0       |
+| Aug 2021 |              | 2.33.0       |                 | 8.7          |
-| Aug 2021 |              |              |                 | 8.7          |
 | Aug 2021 | **Aug 2026** |              |                 |              | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
+| Sep 2021 |              |              |                 | 8.8          |
+| Oct 2021 |              | 2.34.0       | 3.10.0          |              | **22.04 Jammy** |
+| Jan 2022 |              | 2.35.0       |
+| Feb 2022 |              |              |                 | 8.9          | **22.04 Jammy** |
+| Apr 2022 |              | 2.36.0       |                 | 9.0          |
+| Apr 2022 | **Apr 2032** |              |                 |              | **22.04 Jammy**      | 2.34.1 | 2.7.18 3.10.6 | 8.9 |
+| Jun 2022 |              | 2.37.0       |
+| Oct 2022 |              | 2.38.0       |                 | 9.1          |
+| Oct 2022 |              |              | 3.11.0          |              | **Bookworm** |
+| Dec 2022 |              | 2.39.0       |                 |              | **Bookworm** |
+| Feb 2023 |              |              |                 | 9.2          | **Bookworm** |
+| Mar 2023 |              | 2.40.0       |                 | 9.3          |
+| Jun 2023 |              | 2.41.0       |
+| Jun 2023 | **Jun 2028** |              |                 |              | **Debian 12 Bookworm** | 2.39.2 | 3.11.2 | 9.2 |
+| Aug 2023 |              | 2.42.0       |                 | 9.4          |
+| Oct 2023 |              |              | 3.12.0          | 9.5          |
+| Nov 2022 |              | 2.43.0       |
+| Dec 2023 |              |              |                 | 9.6          |
+| Feb 2024 |              | 2.44.0       |
+| Mar 2024 |              |              |                 | 9.7          |
+| Oct 2024 |              |              | 3.13.0          |
 | **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
 
 
@@ -335,7 +350,7 @@ Things in italics are things we used to care about but probably don't anymore.
 [rel-g]: https://en.wikipedia.org/wiki/Git#Releases
 [rel-o]: https://www.openssh.com/releasenotes.html
 [rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
-[rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
+[rel-u]: https://wiki.ubuntu.com/Releases
 [example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
 [repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
 [go/repo-release]: https://goto.google.com/repo-release

docs/repo-hooks.md

@@ -133,3 +133,44 @@ def main(project_list, worktree_list=None, **kwargs):
       kwargs: Leave this here for forward-compatibility.
     """
 ```
+
+### post-sync
+
+This hook runs when `repo sync` completes without errors.
+
+Note: This includes cases where no actual checkout may occur. The hook will still run.
+For example:
+- `repo sync -n` performs network fetches only and skips the checkout phase.
+- `repo sync <project>` only updates the specified project(s).
+- Partial failures may still result in a successful exit.
+
+This hook is useful for post-processing tasks such as setting up git hooks,
+bootstrapping configuration files, or running project initialization logic.
+
+The hook is defined using the existing `<repo-hooks>` manifest block and is
+optional. If the hook script fails or is missing, `repo sync` will still
+complete successfully, and the error will be printed as a warning.
+
+Example:
+
+```xml
+<project name="myorg/dev-tools" path="tools" revision="main" />
+<repo-hooks in-project="myorg/dev-tools" enabled-list="post-sync">
+  <hook name="post-sync" />
+</repo-hooks>
+```
+
+The `post-sync.py` file should be defined like:
+
+```py
+def main(repo_topdir=None, sync_duration_seconds=None, **kwargs):
+    """Main function invoked directly by repo.
+
+    We must use the name "main" as that is what repo requires.
+
+    Args:
+      repo_topdir: The absolute path to the top-level directory of the repo workspace.
+      sync_duration_seconds: The duration of the sync operation in seconds.
+      kwargs: Leave this here for forward-compatibility.
+    """
+```

docs/smart-sync.md

@@ -0,0 +1,129 @@
+# repo Smart Syncing
+
+Repo normally fetches & syncs manifests from the same URL specified during
+`repo init`, and that often fetches the latest revisions of all projects in
+the manifest.  This flow works well for tracking and developing with the
+latest code, but often it's desirable to sync to other points.  For example,
+to get a local build matching a specific release or build to reproduce bugs
+reported by other people.
+
+Repo's sync subcommand has support for fetching manifests from a server over
+an XML-RPC connection.  The local configuration and network API are defined by
+repo, but individual projects have to host their own server for the client to
+communicate with.
+
+This process is called "smart syncing" -- instead of blindly fetching the latest
+revision of all projects and getting an unknown state to develop against, the
+client passes a request to the server and is given a matching manifest that
+typically specifies specific commits for every project to fetch a known source
+state.
+
+[TOC]
+
+## Manifest Configuration
+
+The manifest specifies the server to communicate with via the
+the [`<manifest-server>` element](manifest-format.md#Element-manifest_server)
+element.  This is how the client knows what service to talk to.
+
+```xml
+  <manifest-server url="https://example.com/your/manifest/server/url" />
+```
+
+If the URL starts with `persistent-`, then the
+[`git-remote-persistent-https` helper](https://github.com/git/git/blob/HEAD/contrib/persistent-https/README)
+is used to communicate with the server.
+
+## Credentials
+
+Credentials may be specified directly in typical `username:password`
+[URI syntax](https://en.wikipedia.org/wiki/URI#Syntax) in the
+`<manifest-server>` element directly in the manifest.
+
+If they are not specified, `repo sync` has `--manifest-server-username=USERNAME`
+and `--manifest-server-password=PASSWORD` options.
+
+If those are not used, then repo will look up the host in your
+[`~/.netrc`](https://docs.python.org/3/library/netrc.html) database.
+
+When making the connection, cookies matching the host are automatically loaded
+from the cookiejar specified in
+[Git's `http.cookiefile` setting](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpcookieFile).
+
+## Manifest Server
+
+Unfortunately, there are no public reference implementations.  Google has an
+internal one for Android, but it is written using Google's internal systems,
+so wouldn't be that helpful as a reference.
+
+That said, the XML-RPC API is pretty simple, so any standard XML-RPC server
+example would do.  Google's internal server uses Python's
+[xmlrpc.server.SimpleXMLRPCDispatcher](https://docs.python.org/3/library/xmlrpc.server.html).
+
+## Network API
+
+The manifest server should implement the following RPC methods.
+
+### GetApprovedManifest
+
+> `GetApprovedManifest(branch: str, target: Optional[str]) -> str`
+
+The meaning of `branch` and `target` is not strictly defined.  The server may
+interpret them however it wants.  The recommended interpretation is that the
+`branch` matches the manifest branch, and `target` is an identifier for your
+project that matches something users would build.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+### GetManifest
+
+> `GetManifest(tag: str) -> str`
+
+The meaning of `tag` is not strictly defined.  Projects are encouraged to use
+a system where the tag matches a unique source state.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+## Client Options
+
+Once repo has successfully downloaded the manifest from the server, it saves a
+copy into `.repo/manifests/smart_sync_override.xml` so users can examine it.
+The next time `repo sync` is run, this file is automatically replaced or removed
+based on the current set of options.
+
+### --smart-sync
+
+Repo will call `GetApprovedManifest(branch[, target])`.
+
+The `branch` is determined by the current manifest branch as specified by
+`--manifest-branch=BRANCH` when running `repo init`.
+
+The `target` is defined by environment variables in the order below.  If none
+of them match, then `target` is omitted.  These variables were decided as they
+match the settings Android build environments automatically setup.
+
+1.  `${SYNC_TARGET}`: If defined, the value is used directly.
+2.  `${TARGET_PRODUCT}-${TARGET_RELEASE}-${TARGET_BUILD_VARIANT}`: If these
+    variables are all defined, then they are merged with `-` and used.
+3.  `${TARGET_PRODUCT}-${TARGET_BUILD_VARIANT}`: If these variables are all
+    defined, then they are merged with `-` and used.
+
+### --smart-tag=TAG
+
+Repo will call `GetManifest(TAG)`.

docs/windows.md

@@ -50,8 +50,11 @@ Git worktrees (see the previous section for more info).
 Repo will use symlinks heavily internally.
 On *NIX platforms, this isn't an issue, but Windows makes it a bit difficult.
 
-There are some documents out there for how to do this, but usually the easiest
+The easiest method to allow users to create symlinks is by enabling
-answer is to run your shell as an Administrator and invoke repo/git in that.
+[Windows Developer Mode](https://learn.microsoft.com/en-us/windows/advanced-settings/developer-mode).
+
+The next easiest answer is to run your shell as an Administrator and invoke
+repo/git in that.
 
 This isn't a great solution, but Windows doesn't make this easy, so here we are.
 
@@ -142,7 +145,7 @@ You will have to specify this flag every time you upload.
 
 [mintty]: https://github.com/mintty/mintty/issues/56
 
-### repohooks always fail with an close_fds error.
+### repohooks always fail with an close_fds error
 
 When using the [reference repohooks project][repohooks] included in AOSP,
 you might see errors like this when running `repo upload`:

editor.py

@@ -14,102 +14,106 @@
 
 import os
 import re
-import sys
 import subprocess
+import sys
 import tempfile
 
 from error import EditorError
 import platform_utils
 
 
-class Editor(object):
+class Editor:
-  """Manages the user's preferred text editor."""
+    """Manages the user's preferred text editor."""
 
-  _editor = None
+    _editor = None
-  globalConfig = None
+    globalConfig = None
 
-  @classmethod
+    @classmethod
-  def _GetEditor(cls):
+    def _GetEditor(cls):
-    if cls._editor is None:
+        if cls._editor is None:
-      cls._editor = cls._SelectEditor()
+            cls._editor = cls._SelectEditor()
-    return cls._editor
+        return cls._editor
 
-  @classmethod
+    @classmethod
-  def _SelectEditor(cls):
+    def _SelectEditor(cls):
-    e = os.getenv('GIT_EDITOR')
+        e = os.getenv("GIT_EDITOR")
-    if e:
+        if e:
-      return e
+            return e
 
-    if cls.globalConfig:
+        if cls.globalConfig:
-      e = cls.globalConfig.GetString('core.editor')
+            e = cls.globalConfig.GetString("core.editor")
-      if e:
+            if e:
-        return e
+                return e
 
-    e = os.getenv('VISUAL')
+        e = os.getenv("VISUAL")
-    if e:
+        if e:
-      return e
+            return e
 
-    e = os.getenv('EDITOR')
+        e = os.getenv("EDITOR")
-    if e:
+        if e:
-      return e
+            return e
 
-    if os.getenv('TERM') == 'dumb':
+        if os.getenv("TERM") == "dumb":
-      print(
+            print(
-          """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
+                """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
 Tried to fall back to vi but terminal is dumb.  Please configure at
-least one of these before using this command.""", file=sys.stderr)
+least one of these before using this command.""",  # noqa: E501
-      sys.exit(1)
+                file=sys.stderr,
+            )
+            sys.exit(1)
 
-    return 'vi'
+        return "vi"
 
-  @classmethod
+    @classmethod
-  def EditString(cls, data):
+    def EditString(cls, data):
-    """Opens an editor to edit the given content.
+        """Opens an editor to edit the given content.
 
-    Args:
+        Args:
-      data: The text to edit.
+            data: The text to edit.
 
-    Returns:
+        Returns:
-      New value of edited text.
+            New value of edited text.
 
-    Raises:
+        Raises:
-      EditorError: The editor failed to run.
+            EditorError: The editor failed to run.
-    """
+        """
-    editor = cls._GetEditor()
+        editor = cls._GetEditor()
-    if editor == ':':
+        if editor == ":":
-      return data
+            return data
 
-    fd, path = tempfile.mkstemp()
+        fd, path = tempfile.mkstemp()
-    try:
+        try:
-      os.write(fd, data.encode('utf-8'))
+            os.write(fd, data.encode("utf-8"))
-      os.close(fd)
+            os.close(fd)
-      fd = None
+            fd = None
 
-      if platform_utils.isWindows():
+            if platform_utils.isWindows():
-        # Split on spaces, respecting quoted strings
+                # Split on spaces, respecting quoted strings
-        import shlex
+                import shlex
-        args = shlex.split(editor)
-        shell = False
-      elif re.compile("^.*[$ \t'].*$").match(editor):
-        args = [editor + ' "$@"', 'sh']
-        shell = True
-      else:
-        args = [editor]
-        shell = False
-      args.append(path)
 
-      try:
+                args = shlex.split(editor)
-        rc = subprocess.Popen(args, shell=shell).wait()
+                shell = False
-      except OSError as e:
+            elif re.compile("^.*[$ \t'].*$").match(editor):
-        raise EditorError('editor failed, %s: %s %s'
+                args = [editor + ' "$@"', "sh"]
-                          % (str(e), editor, path))
+                shell = True
-      if rc != 0:
+            else:
-        raise EditorError('editor failed with exit status %d: %s %s'
+                args = [editor]
-                          % (rc, editor, path))
+                shell = False
+            args.append(path)
 
-      with open(path, mode='rb') as fd2:
+            try:
-        return fd2.read().decode('utf-8')
+                rc = subprocess.Popen(args, shell=shell).wait()
-    finally:
+            except OSError as e:
-      if fd:
+                raise EditorError(f"editor failed, {str(e)}: {editor} {path}")
-        os.close(fd)
+            if rc != 0:
-      platform_utils.remove(path)
+                raise EditorError(
+                    "editor failed with exit status %d: %s %s"
+                    % (rc, editor, path)
+                )
+
+            with open(path, mode="rb") as fd2:
+                return fd2.read().decode("utf-8")
+        finally:
+            if fd:
+                os.close(fd)
+            platform_utils.remove(path)

error.py

@@ -12,124 +12,178 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
 
-class ManifestParseError(Exception):
+
-  """Failed to parse the manifest file.
+class BaseRepoError(Exception):
-  """
+    """All repo specific exceptions derive from BaseRepoError."""
+
+
+class RepoError(BaseRepoError):
+    """Exceptions thrown inside repo that can be handled."""
+
+    def __init__(self, *args, project: str = None) -> None:
+        super().__init__(*args)
+        self.project = project
+
+
+class RepoExitError(BaseRepoError):
+    """Exception thrown that result in termination of repo program.
+    - Should only be handled in main.py
+    """
+
+    def __init__(
+        self,
+        *args,
+        exit_code: int = 1,
+        aggregate_errors: List[Exception] = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.exit_code = exit_code
+        self.aggregate_errors = aggregate_errors
+
+
+class RepoUnhandledExceptionError(RepoExitError):
+    """Exception that maintains error as reason for program exit."""
+
+    def __init__(
+        self,
+        error: BaseException,
+        **kwargs,
+    ) -> None:
+        super().__init__(error, **kwargs)
+        self.error = error
+
+
+class SilentRepoExitError(RepoExitError):
+    """RepoExitError that should no include CLI logging of issue/issues."""
+
+
+class ManifestParseError(RepoExitError):
+    """Failed to parse the manifest file."""
 
 
 class ManifestInvalidRevisionError(ManifestParseError):
-  """The revision value in a project is incorrect.
+    """The revision value in a project is incorrect."""
-  """
 
 
 class ManifestInvalidPathError(ManifestParseError):
-  """A path used in <copyfile> or <linkfile> is incorrect.
+    """A path used in <copyfile> or <linkfile> is incorrect."""
-  """
 
 
-class NoManifestException(Exception):
+class NoManifestException(RepoExitError):
-  """The required manifest does not exist.
+    """The required manifest does not exist."""
-  """
 
-  def __init__(self, path, reason):
+    def __init__(self, path, reason, **kwargs):
-    super().__init__(path, reason)
+        super().__init__(path, reason, **kwargs)
-    self.path = path
+        self.path = path
-    self.reason = reason
+        self.reason = reason
 
-  def __str__(self):
+    def __str__(self):
-    return self.reason
+        return self.reason
 
 
-class EditorError(Exception):
+class EditorError(RepoError):
-  """Unspecified error from the user's text editor.
+    """Unspecified error from the user's text editor."""
-  """
 
-  def __init__(self, reason):
+    def __init__(self, reason, **kwargs):
-    super().__init__(reason)
+        super().__init__(reason, **kwargs)
-    self.reason = reason
+        self.reason = reason
 
-  def __str__(self):
+    def __str__(self):
-    return self.reason
+        return self.reason
 
 
-class GitError(Exception):
+class GitError(RepoError):
-  """Unspecified internal error from git.
+    """Unspecified git related error."""
-  """
 
-  def __init__(self, command):
+    def __init__(self, message, command_args=None, **kwargs):
-    super().__init__(command)
+        super().__init__(message, **kwargs)
-    self.command = command
+        self.message = message
+        self.command_args = command_args
 
-  def __str__(self):
+    def __str__(self):
-    return self.command
+        return self.message
 
 
-class UploadError(Exception):
+class GitAuthError(RepoExitError):
-  """A bundle upload to Gerrit did not succeed.
+    """Cannot talk to remote due to auth issue."""
-  """
-
-  def __init__(self, reason):
-    super().__init__(reason)
-    self.reason = reason
-
-  def __str__(self):
-    return self.reason
 
 
-class DownloadError(Exception):
+class UploadError(RepoError):
-  """Cannot download a repository.
+    """A bundle upload to Gerrit did not succeed."""
-  """
 
-  def __init__(self, reason):
+    def __init__(self, reason, **kwargs):
-    super().__init__(reason)
+        super().__init__(reason, **kwargs)
-    self.reason = reason
+        self.reason = reason
 
-  def __str__(self):
+    def __str__(self):
-    return self.reason
+        return self.reason
 
 
-class NoSuchProjectError(Exception):
+class DownloadError(RepoExitError):
-  """A specified project does not exist in the work tree.
+    """Cannot download a repository."""
-  """
 
-  def __init__(self, name=None):
+    def __init__(self, reason, **kwargs):
-    super().__init__(name)
+        super().__init__(reason, **kwargs)
-    self.name = name
+        self.reason = reason
 
-  def __str__(self):
+    def __str__(self):
-    if self.name is None:
+        return self.reason
-      return 'in current directory'
-    return self.name
 
 
-class InvalidProjectGroupsError(Exception):
+class InvalidArgumentsError(RepoExitError):
-  """A specified project is not suitable for the specified groups
+    """Invalid command Arguments."""
-  """
-
-  def __init__(self, name=None):
-    super().__init__(name)
-    self.name = name
-
-  def __str__(self):
-    if self.name is None:
-      return 'in current directory'
-    return self.name
 
 
-class RepoChangedException(Exception):
+class SyncError(RepoExitError):
-  """Thrown if 'repo sync' results in repo updating its internal
+    """Cannot sync repo."""
-     repo or manifest repositories.  In this special case we must
-     use exec to re-execute repo with the new code and manifest.
-  """
-
-  def __init__(self, extra_args=None):
-    super().__init__(extra_args)
-    self.extra_args = extra_args or []
 
 
-class HookError(Exception):
+class UpdateManifestError(RepoExitError):
-  """Thrown if a 'repo-hook' could not be run.
+    """Cannot update manifest."""
 
-  The common case is that the file wasn't present when we tried to run it.
+
-  """
+class NoSuchProjectError(RepoExitError):
+    """A specified project does not exist in the work tree."""
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(**kwargs)
+        self.name = name
+
+    def __str__(self):
+        if self.name is None:
+            return "in current directory"
+        return self.name
+
+
+class InvalidProjectGroupsError(RepoExitError):
+    """A specified project is not suitable for the specified groups"""
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(**kwargs)
+        self.name = name
+
+    def __str__(self):
+        if self.name is None:
+            return "in current directory"
+        return self.name
+
+
+class RepoChangedException(BaseRepoError):
+    """Thrown if 'repo sync' results in repo updating its internal
+    repo or manifest repositories.  In this special case we must
+    use exec to re-execute repo with the new code and manifest.
+    """
+
+    def __init__(self, extra_args=None):
+        super().__init__(extra_args)
+        self.extra_args = extra_args or []
+
+
+class HookError(RepoError):
+    """Thrown if a 'repo-hook' could not be run.
+
+    The common case is that the file wasn't present when we tried to run it.
+    """

event_log.py

@@ -15,161 +15,178 @@
 import json
 import multiprocessing
 
-TASK_COMMAND = 'command'
+
-TASK_SYNC_NETWORK = 'sync-network'
+TASK_COMMAND = "command"
-TASK_SYNC_LOCAL = 'sync-local'
+TASK_SYNC_NETWORK = "sync-network"
+TASK_SYNC_LOCAL = "sync-local"
 
 
-class EventLog(object):
+class EventLog:
-  """Event log that records events that occurred during a repo invocation.
+    """Event log that records events that occurred during a repo invocation.
 
-  Events are written to the log as a consecutive JSON entries, one per line.
+    Events are written to the log as a consecutive JSON entries, one per line.
-  Each entry contains the following keys:
+    Each entry contains the following keys:
-  - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
+    - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
-        The ID is only unique for the invocation of the repo command.
+          The ID is only unique for the invocation of the repo command.
-  - name: Name of the object being operated upon.
+    - name: Name of the object being operated upon.
-  - task_name: The task that was performed.
+    - task_name: The task that was performed.
-  - start: Timestamp of when the operation started.
+    - start: Timestamp of when the operation started.
-  - finish: Timestamp of when the operation finished.
+    - finish: Timestamp of when the operation finished.
-  - success: Boolean indicating if the operation was successful.
+    - success: Boolean indicating if the operation was successful.
-  - try_count: A counter indicating the try count of this task.
+    - try_count: A counter indicating the try count of this task.
 
-  Optionally:
+    Optionally:
-  - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
+    - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
-            events.
+        events.
 
-  Valid task_names include:
+    Valid task_names include:
-  - command: The invocation of a subcommand.
+    - command: The invocation of a subcommand.
-  - sync-network: The network component of a sync command.
+    - sync-network: The network component of a sync command.
-  - sync-local: The local component of a sync command.
+    - sync-local: The local component of a sync command.
 
-  Specific tasks may include additional informational properties.
+    Specific tasks may include additional informational properties.
-  """
-
-  def __init__(self):
-    """Initializes the event log."""
-    self._log = []
-    self._parent = None
-
-  def Add(self, name, task_name, start, finish=None, success=None,
-          try_count=1, kind='RepoOp'):
-    """Add an event to the log.
-
-    Args:
-      name: Name of the object being operated upon.
-      task_name: A sub-task that was performed for name.
-      start: Timestamp of when the operation started.
-      finish: Timestamp of when the operation finished.
-      success: Boolean indicating if the operation was successful.
-      try_count: A counter indicating the try count of this task.
-      kind: The kind of the object for the unique identifier.
-
-    Returns:
-      A dictionary of the event added to the log.
     """
-    event = {
-        'id': (kind, _NextEventId()),
-        'name': name,
-        'task_name': task_name,
-        'start_time': start,
-        'try': try_count,
-    }
 
-    if self._parent:
+    def __init__(self):
-      event['parent'] = self._parent['id']
+        """Initializes the event log."""
+        self._log = []
+        self._parent = None
 
-    if success is not None or finish is not None:
+    def Add(
-        self.FinishEvent(event, finish, success)
+        self,
+        name,
+        task_name,
+        start,
+        finish=None,
+        success=None,
+        try_count=1,
+        kind="RepoOp",
+    ):
+        """Add an event to the log.
 
-    self._log.append(event)
+        Args:
-    return event
+            name: Name of the object being operated upon.
+            task_name: A sub-task that was performed for name.
+            start: Timestamp of when the operation started.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
+            try_count: A counter indicating the try count of this task.
+            kind: The kind of the object for the unique identifier.
 
-  def AddSync(self, project, task_name, start, finish, success):
+        Returns:
-    """Add a event to the log for a sync command.
+            A dictionary of the event added to the log.
+        """
+        event = {
+            "id": (kind, _NextEventId()),
+            "name": name,
+            "task_name": task_name,
+            "start_time": start,
+            "try": try_count,
+        }
 
-    Args:
+        if self._parent:
-      project: Project being synced.
+            event["parent"] = self._parent["id"]
-      task_name: A sub-task that was performed for name.
-                 One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
-      start: Timestamp of when the operation started.
-      finish: Timestamp of when the operation finished.
-      success: Boolean indicating if the operation was successful.
 
-    Returns:
+        if success is not None or finish is not None:
-      A dictionary of the event added to the log.
+            self.FinishEvent(event, finish, success)
-    """
-    event = self.Add(project.relpath, task_name, start, finish, success)
-    if event is not None:
-      event['project'] = project.name
-      if project.revisionExpr:
-        event['revision'] = project.revisionExpr
-      if project.remote.url:
-        event['project_url'] = project.remote.url
-      if project.remote.fetchUrl:
-        event['remote_url'] = project.remote.fetchUrl
-      try:
-        event['git_hash'] = project.GetCommitRevisionId()
-      except Exception:
-        pass
-    return event
 
-  def GetStatusString(self, success):
+        self._log.append(event)
-    """Converst a boolean success to a status string.
+        return event
 
-    Args:
+    def AddSync(self, project, task_name, start, finish, success):
-      success: Boolean indicating if the operation was successful.
+        """Add a event to the log for a sync command.
 
-    Returns:
+        Args:
-      status string.
+            project: Project being synced.
-    """
+            task_name: A sub-task that was performed for name.
-    return 'pass' if success else 'fail'
+                One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
+            start: Timestamp of when the operation started.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
 
-  def FinishEvent(self, event, finish, success):
+        Returns:
-    """Finishes an incomplete event.
+            A dictionary of the event added to the log.
+        """
+        event = self.Add(project.relpath, task_name, start, finish, success)
+        if event is not None:
+            event["project"] = project.name
+            if project.revisionExpr:
+                event["revision"] = project.revisionExpr
+            if project.remote.url:
+                event["project_url"] = project.remote.url
+            if project.remote.fetchUrl:
+                event["remote_url"] = project.remote.fetchUrl
+            try:
+                event["git_hash"] = project.GetCommitRevisionId()
+            except Exception:
+                pass
+        return event
 
-    Args:
+    def GetStatusString(self, success):
-      event: An event that has been added to the log.
+        """Converst a boolean success to a status string.
-      finish: Timestamp of when the operation finished.
-      success: Boolean indicating if the operation was successful.
 
-    Returns:
+        Args:
-      A dictionary of the event added to the log.
+            success: Boolean indicating if the operation was successful.
-    """
-    event['status'] = self.GetStatusString(success)
-    event['finish_time'] = finish
-    return event
 
-  def SetParent(self, event):
+        Returns:
-    """Set a parent event for all new entities.
+            status string.
+        """
+        return "pass" if success else "fail"
 
-    Args:
+    def FinishEvent(self, event, finish, success):
-      event: The event to use as a parent.
+        """Finishes an incomplete event.
-    """
-    self._parent = event
 
-  def Write(self, filename):
+        Args:
-    """Writes the log out to a file.
+            event: An event that has been added to the log.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
 
-    Args:
+        Returns:
-      filename: The file to write the log to.
+            A dictionary of the event added to the log.
-    """
+        """
-    with open(filename, 'w+') as f:
+        event["status"] = self.GetStatusString(success)
-      for e in self._log:
+        event["finish_time"] = finish
-        json.dump(e, f, sort_keys=True)
+        return event
-        f.write('\n')
+
+    def SetParent(self, event):
+        """Set a parent event for all new entities.
+
+        Args:
+            event: The event to use as a parent.
+        """
+        self._parent = event
+
+    def Write(self, filename):
+        """Writes the log out to a file.
+
+        Args:
+            filename: The file to write the log to.
+        """
+        with open(filename, "w+") as f:
+            for e in self._log:
+                json.dump(e, f, sort_keys=True)
+                f.write("\n")
 
 
-# An integer id that is unique across this invocation of the program.
+# An integer id that is unique across this invocation of the program, to be set
-_EVENT_ID = multiprocessing.Value('i', 1)
+# by the first Add event. We can't set it here since it results in leaked
+# resources (see: https://issues.gerritcodereview.com/353656374).
+_EVENT_ID = None
 
 
 def _NextEventId():
-  """Helper function for grabbing the next unique id.
+    """Helper function for grabbing the next unique id.
 
-  Returns:
+    Returns:
-    A unique, to this invocation of the program, integer id.
+        A unique, to this invocation of the program, integer id.
-  """
+    """
-  with _EVENT_ID.get_lock():
+    global _EVENT_ID
-    val = _EVENT_ID.value
+    if _EVENT_ID is None:
-    _EVENT_ID.value += 1
+        # There is a small chance of race condition - two parallel processes
-  return val
+        # setting up _EVENT_ID. However, we expect TASK_COMMAND to happen before
+        # mp kicks in.
+        _EVENT_ID = multiprocessing.Value("i", 1)
+    with _EVENT_ID.get_lock():
+        val = _EVENT_ID.value
+        _EVENT_ID.value += 1
+    return val

fetch.py

@@ -19,27 +19,40 @@ import sys
 from urllib.parse import urlparse
 from urllib.request import urlopen
 
+from error import RepoExitError
+
+
+class FetchFileError(RepoExitError):
+    """Exit error when fetch_file fails."""
+
 
 def fetch_file(url, verbose=False):
-  """Fetch a file from the specified source using the appropriate protocol.
+    """Fetch a file from the specified source using the appropriate protocol.
 
-  Returns:
+    Returns:
-    The contents of the file as bytes.
+        The contents of the file as bytes.
-  """
+    """
-  scheme = urlparse(url).scheme
+    scheme = urlparse(url).scheme
-  if scheme == 'gs':
+    if scheme == "gs":
-    cmd = ['gsutil', 'cat', url]
+        cmd = ["gcloud", "storage", "cat", url]
-    try:
+        errors = []
-      result = subprocess.run(
+        try:
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            result = subprocess.run(
-          check=True)
+                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
-      if result.stderr and verbose:
+            )
-        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+            if result.stderr and verbose:
-              file=sys.stderr)
+                print(
-      return result.stdout
+                    'warning: non-fatal error running "gcloud storage": %s'
-    except subprocess.CalledProcessError as e:
+                    % result.stderr,
-      print('fatal: error running "gsutil": %s' % e.stderr,
+                    file=sys.stderr,
-            file=sys.stderr)
+                )
-    sys.exit(1)
+            return result.stdout
-  with urlopen(url) as f:
+        except subprocess.CalledProcessError as e:
-    return f.read()
+            errors.append(e)
+            print(
+                'fatal: error running "gcloud storage": %s' % e.stderr,
+                file=sys.stderr,
+            )
+        raise FetchFileError(aggregate_errors=errors)
+    with urlopen(url) as f:
+        return f.read()

git_command.py

@@ -13,301 +13,637 @@
 # limitations under the License.
 
 import functools
+import json
 import os
-import sys
+import re
 import subprocess
+import sys
+from typing import Any, Optional
 
 from error import GitError
-from git_refs import HEAD
+from error import RepoExitError
+from git_trace2_event_log_base import BaseEventLog
 import platform_utils
-from repo_trace import REPO_TRACE, IsTrace, Trace
+from repo_logging import RepoLogger
+from repo_trace import IsTrace
+from repo_trace import REPO_TRACE
+from repo_trace import Trace
 from wrapper import Wrapper
 
-GIT = 'git'
+
-# NB: These do not need to be kept in sync with the repo launcher script.
+GIT = "git"
-# These may be much newer as it allows the repo launcher to roll between
+GIT_DIR = "GIT_DIR"
-# different repo releases while source versions might require a newer git.
-#
-# The soft version is when we start warning users that the version is old and
-# we'll be dropping support for it.  We'll refuse to work with versions older
-# than the hard version.
-#
-# git-1.7 is in (EOL) Ubuntu Precise.  git-1.9 is in Ubuntu Trusty.
-MIN_GIT_VERSION_SOFT = (1, 9, 1)
-MIN_GIT_VERSION_HARD = (1, 7, 2)
-GIT_DIR = 'GIT_DIR'
 
 LAST_GITDIR = None
 LAST_CWD = None
+DEFAULT_GIT_FAIL_MESSAGE = "git command failure"
+ERROR_EVENT_LOGGING_PREFIX = "RepoGitCommandError"
+# Common line length limit
+GIT_ERROR_STDOUT_LINES = 1
+GIT_ERROR_STDERR_LINES = 10
+INVALID_GIT_EXIT_CODE = 126
+
+logger = RepoLogger(__file__)
 
 
-class _GitCall(object):
+class _GitCall:
-  @functools.lru_cache(maxsize=None)
+    @functools.lru_cache(maxsize=None)  # noqa: B019
-  def version_tuple(self):
+    def version_tuple(self):
-    ret = Wrapper().ParseGitVersion()
+        ret = Wrapper().ParseGitVersion()
-    if ret is None:
+        if ret is None:
-      print('fatal: unable to detect git version', file=sys.stderr)
+            msg = "fatal: unable to detect git version"
-      sys.exit(1)
+            logger.error(msg)
-    return ret
+            raise GitRequireError(msg)
+        return ret
 
-  def __getattr__(self, name):
+    def __getattr__(self, name):
-    name = name.replace('_', '-')
+        name = name.replace("_", "-")
 
-    def fun(*cmdv):
+        def fun(*cmdv):
-      command = [name]
+            command = [name]
-      command.extend(cmdv)
+            command.extend(cmdv)
-      return GitCommand(None, command).Wait() == 0
+            return GitCommand(None, command, add_event_log=False).Wait() == 0
-    return fun
+
+        return fun
 
 
 git = _GitCall()
 
 
 def RepoSourceVersion():
-  """Return the version of the repo.git tree."""
+    """Return the version of the repo.git tree."""
-  ver = getattr(RepoSourceVersion, 'version', None)
+    ver = getattr(RepoSourceVersion, "version", None)
 
-  # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+    # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
-  # to initialize version info we provide.
+    # to initialize version info we provide.
-  if ver is None:
+    if ver is None:
-    env = GitCommand._GetBasicEnv()
+        env = GitCommand._GetBasicEnv()
 
-    proj = os.path.dirname(os.path.abspath(__file__))
+        proj = os.path.dirname(os.path.abspath(__file__))
-    env[GIT_DIR] = os.path.join(proj, '.git')
+        env[GIT_DIR] = os.path.join(proj, ".git")
-    result = subprocess.run([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
+        result = subprocess.run(
-                            stderr=subprocess.DEVNULL, encoding='utf-8',
+            [GIT, "describe", "HEAD"],
-                            env=env, check=False)
+            stdout=subprocess.PIPE,
-    if result.returncode == 0:
+            stderr=subprocess.DEVNULL,
-      ver = result.stdout.strip()
+            encoding="utf-8",
-      if ver.startswith('v'):
+            env=env,
-        ver = ver[1:]
+            check=False,
-    else:
+        )
-      ver = 'unknown'
+        if result.returncode == 0:
-    setattr(RepoSourceVersion, 'version', ver)
+            ver = result.stdout.strip()
+            if ver.startswith("v"):
+                ver = ver[1:]
+        else:
+            ver = "unknown"
+        RepoSourceVersion.version = ver
 
-  return ver
+    return ver
 
 
-class UserAgent(object):
+@functools.lru_cache(maxsize=None)
-  """Mange User-Agent settings when talking to external services
+def GetEventTargetPath():
+    """Get the 'trace2.eventtarget' path from git configuration.
 
-  We follow the style as documented here:
+    Returns:
-  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+        path: git config's 'trace2.eventtarget' path if it exists, or None
-  """
+    """
+    path = None
+    cmd = ["config", "--get", "trace2.eventtarget"]
+    # TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
+    # system git config variables.
+    p = GitCommand(
+        None,
+        cmd,
+        capture_stdout=True,
+        capture_stderr=True,
+        bare=True,
+        add_event_log=False,
+    )
+    retval = p.Wait()
+    if retval == 0:
+        # Strip trailing carriage-return in path.
+        path = p.stdout.rstrip("\n")
+        if path == "":
+            return None
+    elif retval != 1:
+        # `git config --get` is documented to produce an exit status of `1`
+        # if the requested variable is not present in the configuration.
+        # Report any other return value as an error.
+        logger.error(
+            "repo: error: 'git config --get' call failed with return code: "
+            "%r, stderr: %r",
+            retval,
+            p.stderr,
+        )
+    return path
 
-  _os = None
-  _repo_ua = None
-  _git_ua = None
 
-  @property
+class UserAgent:
-  def os(self):
+    """Mange User-Agent settings when talking to external services
-    """The operating system name."""
-    if self._os is None:
-      os_name = sys.platform
-      if os_name.lower().startswith('linux'):
-        os_name = 'Linux'
-      elif os_name == 'win32':
-        os_name = 'Win32'
-      elif os_name == 'cygwin':
-        os_name = 'Cygwin'
-      elif os_name == 'darwin':
-        os_name = 'Darwin'
-      self._os = os_name
 
-    return self._os
+    We follow the style as documented here:
+    https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+    """
 
-  @property
+    _os = None
-  def repo(self):
+    _repo_ua = None
-    """The UA when connecting directly from repo."""
+    _git_ua = None
-    if self._repo_ua is None:
-      py_version = sys.version_info
-      self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
-          RepoSourceVersion(),
-          self.os,
-          git.version_tuple().full,
-          py_version.major, py_version.minor, py_version.micro)
 
-    return self._repo_ua
+    @property
+    def os(self):
+        """The operating system name."""
+        if self._os is None:
+            os_name = sys.platform
+            if os_name.lower().startswith("linux"):
+                os_name = "Linux"
+            elif os_name == "win32":
+                os_name = "Win32"
+            elif os_name == "cygwin":
+                os_name = "Cygwin"
+            elif os_name == "darwin":
+                os_name = "Darwin"
+            self._os = os_name
 
-  @property
+        return self._os
-  def git(self):
-    """The UA when running git."""
-    if self._git_ua is None:
-      self._git_ua = 'git/%s (%s) git-repo/%s' % (
-          git.version_tuple().full,
-          self.os,
-          RepoSourceVersion())
 
-    return self._git_ua
+    @property
+    def repo(self):
+        """The UA when connecting directly from repo."""
+        if self._repo_ua is None:
+            py_version = sys.version_info
+            self._repo_ua = "git-repo/%s (%s) git/%s Python/%d.%d.%d" % (
+                RepoSourceVersion(),
+                self.os,
+                git.version_tuple().full,
+                py_version.major,
+                py_version.minor,
+                py_version.micro,
+            )
+
+        return self._repo_ua
+
+    @property
+    def git(self):
+        """The UA when running git."""
+        if self._git_ua is None:
+            self._git_ua = (
+                f"git/{git.version_tuple().full} ({self.os}) "
+                f"git-repo/{RepoSourceVersion()}"
+            )
+        return self._git_ua
 
 
 user_agent = UserAgent()
 
 
-def git_require(min_version, fail=False, msg=''):
+def git_require(min_version, fail=False, msg=""):
-  git_version = git.version_tuple()
+    git_version = git.version_tuple()
-  if min_version <= git_version:
+    if min_version <= git_version:
-    return True
+        return True
-  if fail:
+    if fail:
-    need = '.'.join(map(str, min_version))
+        need = ".".join(map(str, min_version))
-    if msg:
+        if msg:
-      msg = ' for ' + msg
+            msg = " for " + msg
-    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
+        error_msg = f"fatal: git {need} or later required{msg}"
-    sys.exit(1)
+        logger.error(error_msg)
-  return False
+        raise GitRequireError(error_msg)
+    return False
 
 
-class GitCommand(object):
+def _build_env(
-  def __init__(self,
+    _kwargs_only=(),
-               project,
+    bare: Optional[bool] = False,
-               cmdv,
+    disable_editor: Optional[bool] = False,
-               bare=False,
+    ssh_proxy: Optional[Any] = None,
-               input=None,
+    gitdir: Optional[str] = None,
-               capture_stdout=False,
+    objdir: Optional[str] = None,
-               capture_stderr=False,
+):
-               merge_output=False,
+    """Constucts an env dict for command execution."""
-               disable_editor=False,
+
-               ssh_proxy=None,
+    assert _kwargs_only == (), "_build_env only accepts keyword arguments."
-               cwd=None,
+
-               gitdir=None,
+    env = GitCommand._GetBasicEnv()
-               objdir=None):
-    env = self._GetBasicEnv()
 
     if disable_editor:
-      env['GIT_EDITOR'] = ':'
+        env["GIT_EDITOR"] = ":"
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_proxy.sock()
+        env["REPO_SSH_SOCK"] = ssh_proxy.sock()
-      env['GIT_SSH'] = ssh_proxy.proxy
+        env["GIT_SSH"] = ssh_proxy.proxy
-      env['GIT_SSH_VARIANT'] = 'ssh'
+        env["GIT_SSH_VARIANT"] = "ssh"
-    if 'http_proxy' in env and 'darwin' == sys.platform:
+    if "http_proxy" in env and "darwin" == sys.platform:
-      s = "'http.proxy=%s'" % (env['http_proxy'],)
+        s = f"'http.proxy={env['http_proxy']}'"
-      p = env.get('GIT_CONFIG_PARAMETERS')
+        p = env.get("GIT_CONFIG_PARAMETERS")
-      if p is not None:
+        if p is not None:
-        s = p + ' ' + s
+            s = p + " " + s
-      env['GIT_CONFIG_PARAMETERS'] = s
+        env["GIT_CONFIG_PARAMETERS"] = s
-    if 'GIT_ALLOW_PROTOCOL' not in env:
+    if "GIT_ALLOW_PROTOCOL" not in env:
-      env['GIT_ALLOW_PROTOCOL'] = (
+        env["GIT_ALLOW_PROTOCOL"] = (
-          'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+            "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc"
-    env['GIT_HTTP_USER_AGENT'] = user_agent.git
+        )
-
+    env["GIT_HTTP_USER_AGENT"] = user_agent.git
-    if project:
-      if not cwd:
-        cwd = project.worktree
-      if not gitdir:
-        gitdir = project.gitdir
-    # Git on Windows wants its paths only using / for reliability.
-    if platform_utils.isWindows():
-      if objdir:
-        objdir = objdir.replace('\\', '/')
-      if gitdir:
-        gitdir = gitdir.replace('\\', '/')
 
     if objdir:
-      # Set to the place we want to save the objects.
+        # Set to the place we want to save the objects.
-      env['GIT_OBJECT_DIRECTORY'] = objdir
+        env["GIT_OBJECT_DIRECTORY"] = objdir
-      if gitdir:
-        # Allow git to search the original place in case of local or unique refs
-        # that git will attempt to resolve even if we aren't fetching them.
-        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
 
-    command = [GIT]
+        alt_objects = os.path.join(gitdir, "objects") if gitdir else None
-    if bare:
+        if alt_objects and os.path.realpath(alt_objects) != os.path.realpath(
-      if gitdir:
+            objdir
+        ):
+            # Allow git to search the original place in case of local or unique
+            # refs that git will attempt to resolve even if we aren't fetching
+            # them.
+            env["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = alt_objects
+    if bare and gitdir is not None:
         env[GIT_DIR] = gitdir
-      cwd = None
-    command.append(cmdv[0])
-    # Need to use the --progress flag for fetch/clone so output will be
-    # displayed as by default git only does progress output if stderr is a TTY.
-    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
-      if '--progress' not in cmdv and '--quiet' not in cmdv:
-        command.append('--progress')
-    command.extend(cmdv[1:])
 
-    stdin = subprocess.PIPE if input else None
-    stdout = subprocess.PIPE if capture_stdout else None
-    stderr = (subprocess.STDOUT if merge_output else
-              (subprocess.PIPE if capture_stderr else None))
-
-    if IsTrace():
-      global LAST_CWD
-      global LAST_GITDIR
-
-      dbg = ''
-
-      if cwd and LAST_CWD != cwd:
-        if LAST_GITDIR or LAST_CWD:
-          dbg += '\n'
-        dbg += ': cd %s\n' % cwd
-        LAST_CWD = cwd
-
-      if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
-        if LAST_GITDIR or LAST_CWD:
-          dbg += '\n'
-        dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
-        LAST_GITDIR = env[GIT_DIR]
-
-      if 'GIT_OBJECT_DIRECTORY' in env:
-        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
-      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
-        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % env['GIT_ALTERNATE_OBJECT_DIRECTORIES']
-
-      dbg += ': '
-      dbg += ' '.join(command)
-      if stdin == subprocess.PIPE:
-        dbg += ' 0<|'
-      if stdout == subprocess.PIPE:
-        dbg += ' 1>|'
-      if stderr == subprocess.PIPE:
-        dbg += ' 2>|'
-      elif stderr == subprocess.STDOUT:
-        dbg += ' 2>&1'
-      Trace('%s', dbg)
-
-    try:
-      p = subprocess.Popen(command,
-                           cwd=cwd,
-                           env=env,
-                           encoding='utf-8',
-                           errors='backslashreplace',
-                           stdin=stdin,
-                           stdout=stdout,
-                           stderr=stderr)
-    except Exception as e:
-      raise GitError('%s: %s' % (command[1], e))
-
-    if ssh_proxy:
-      ssh_proxy.add_client(p)
-
-    self.process = p
-    if input:
-      if isinstance(input, str):
-        input = input.encode('utf-8')
-      p.stdin.write(input)
-      p.stdin.close()
-
-    try:
-      self.stdout, self.stderr = p.communicate()
-    finally:
-      if ssh_proxy:
-        ssh_proxy.remove_client(p)
-    self.rc = p.wait()
-
-  @staticmethod
-  def _GetBasicEnv():
-    """Return a basic env for running git under.
-
-    This is guaranteed to be side-effect free.
-    """
-    env = os.environ.copy()
-    for key in (REPO_TRACE,
-                GIT_DIR,
-                'GIT_ALTERNATE_OBJECT_DIRECTORIES',
-                'GIT_OBJECT_DIRECTORY',
-                'GIT_WORK_TREE',
-                'GIT_GRAFT_FILE',
-                'GIT_INDEX_FILE'):
-      env.pop(key, None)
     return env
 
-  def Wait(self):
+
-    return self.rc
+class GitCommand:
+    """Wrapper around a single git invocation."""
+
+    def __init__(
+        self,
+        project,
+        cmdv,
+        bare=False,
+        input=None,
+        capture_stdout=False,
+        capture_stderr=False,
+        merge_output=False,
+        disable_editor=False,
+        ssh_proxy=None,
+        cwd=None,
+        gitdir=None,
+        objdir=None,
+        verify_command=False,
+        add_event_log=True,
+        log_as_error=True,
+    ):
+        if project:
+            if not cwd:
+                cwd = project.worktree
+            if not gitdir:
+                gitdir = project.gitdir
+
+        self.project = project
+        self.cmdv = cmdv
+        self.verify_command = verify_command
+        self.stdout, self.stderr = None, None
+
+        # Git on Windows wants its paths only using / for reliability.
+        if platform_utils.isWindows():
+            if objdir:
+                objdir = objdir.replace("\\", "/")
+            if gitdir:
+                gitdir = gitdir.replace("\\", "/")
+
+        env = _build_env(
+            disable_editor=disable_editor,
+            ssh_proxy=ssh_proxy,
+            objdir=objdir,
+            gitdir=gitdir,
+            bare=bare,
+        )
+
+        command = [GIT]
+        if bare:
+            cwd = None
+        command_name = cmdv[0]
+        command.append(command_name)
+
+        if command_name in ("fetch", "clone"):
+            env["GIT_TERMINAL_PROMPT"] = "0"
+            # Need to use the --progress flag for fetch/clone so output will be
+            # displayed as by default git only does progress output if stderr is
+            # a TTY.
+            if sys.stderr.isatty():
+                if "--progress" not in cmdv and "--quiet" not in cmdv:
+                    command.append("--progress")
+        command.extend(cmdv[1:])
+
+        event_log = (
+            BaseEventLog(env=env, add_init_count=True)
+            if add_event_log
+            else None
+        )
+
+        try:
+            self._RunCommand(
+                command,
+                env,
+                capture_stdout=capture_stdout,
+                capture_stderr=capture_stderr,
+                merge_output=merge_output,
+                ssh_proxy=ssh_proxy,
+                cwd=cwd,
+                input=input,
+            )
+            self.VerifyCommand()
+        except GitCommandError as e:
+            if event_log is not None:
+                error_info = json.dumps(
+                    {
+                        "ErrorType": type(e).__name__,
+                        "Project": e.project,
+                        "CommandName": command_name,
+                        "Message": str(e),
+                        "ReturnCode": (
+                            str(e.git_rc) if e.git_rc is not None else None
+                        ),
+                        "IsError": log_as_error,
+                    }
+                )
+                event_log.ErrorEvent(
+                    f"{ERROR_EVENT_LOGGING_PREFIX}:{error_info}"
+                )
+                event_log.Write(GetEventTargetPath())
+            if isinstance(e, GitPopenCommandError):
+                raise
+
+    def _RunCommand(
+        self,
+        command,
+        env,
+        capture_stdout=False,
+        capture_stderr=False,
+        merge_output=False,
+        ssh_proxy=None,
+        cwd=None,
+        input=None,
+    ):
+        # Set subprocess.PIPE for streams that need to be captured.
+        stdin = subprocess.PIPE if input else None
+        stdout = subprocess.PIPE if capture_stdout else None
+        stderr = (
+            subprocess.STDOUT
+            if merge_output
+            else (subprocess.PIPE if capture_stderr else None)
+        )
+
+        # tee_stderr acts like a tee command for stderr, in that, it captures
+        # stderr from the subprocess and streams it back to sys.stderr, while
+        # keeping a copy in-memory.
+        # This allows us to store stderr logs from the subprocess into
+        # GitCommandError.
+        # Certain git operations, such as `git push`, writes diagnostic logs,
+        # such as, progress bar for pushing, into stderr. To ensure we don't
+        # break git's UX, we need to write to sys.stderr as we read from the
+        # subprocess. Setting encoding or errors makes subprocess return
+        # io.TextIOWrapper, which is line buffered. To avoid line-buffering
+        # while tee-ing stderr, we unset these kwargs. See GitCommand._Tee
+        # for tee-ing between the streams.
+        # We tee stderr iff the caller doesn't want to capture any stream to
+        # not disrupt the existing flow.
+        # See go/tee-repo-stderr for more context.
+        tee_stderr = False
+        kwargs = {"encoding": "utf-8", "errors": "backslashreplace"}
+        if not (stdin or stdout or stderr):
+            tee_stderr = True
+            # stderr will be written back to sys.stderr even though it is
+            # piped here.
+            stderr = subprocess.PIPE
+            kwargs = {}
+
+        dbg = ""
+        if IsTrace():
+            global LAST_CWD
+            global LAST_GITDIR
+
+            if cwd and LAST_CWD != cwd:
+                if LAST_GITDIR or LAST_CWD:
+                    dbg += "\n"
+                dbg += ": cd %s\n" % cwd
+                LAST_CWD = cwd
+
+            if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
+                if LAST_GITDIR or LAST_CWD:
+                    dbg += "\n"
+                dbg += ": export GIT_DIR=%s\n" % env[GIT_DIR]
+                LAST_GITDIR = env[GIT_DIR]
+
+            if "GIT_OBJECT_DIRECTORY" in env:
+                dbg += (
+                    ": export GIT_OBJECT_DIRECTORY=%s\n"
+                    % env["GIT_OBJECT_DIRECTORY"]
+                )
+            if "GIT_ALTERNATE_OBJECT_DIRECTORIES" in env:
+                dbg += ": export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n" % (
+                    env["GIT_ALTERNATE_OBJECT_DIRECTORIES"]
+                )
+
+            dbg += ": "
+            dbg += " ".join(command)
+            if stdin == subprocess.PIPE:
+                dbg += " 0<|"
+            if stdout == subprocess.PIPE:
+                dbg += " 1>|"
+            if stderr == subprocess.PIPE:
+                dbg += " 2>|"
+            elif stderr == subprocess.STDOUT:
+                dbg += " 2>&1"
+
+        with Trace(
+            "git command %s %s with debug: %s", LAST_GITDIR, command, dbg
+        ):
+            try:
+                p = subprocess.Popen(
+                    command,
+                    cwd=cwd,
+                    env=env,
+                    stdin=stdin,
+                    stdout=stdout,
+                    stderr=stderr,
+                    **kwargs,
+                )
+            except Exception as e:
+                raise GitPopenCommandError(
+                    message=f"{command[1]}: {e}",
+                    project=self.project.name if self.project else None,
+                    command_args=self.cmdv,
+                )
+
+            if ssh_proxy:
+                ssh_proxy.add_client(p)
+
+            self.process = p
+
+            try:
+                if tee_stderr:
+                    # tee_stderr streams stderr to sys.stderr while capturing
+                    # a copy within self.stderr. tee_stderr is only enabled
+                    # when the caller wants to pipe no stream.
+                    self.stderr = self._Tee(p.stderr, sys.stderr)
+                else:
+                    self.stdout, self.stderr = p.communicate(input=input)
+            finally:
+                if ssh_proxy:
+                    ssh_proxy.remove_client(p)
+            self.rc = p.wait()
+
+    @staticmethod
+    def _Tee(in_stream, out_stream):
+        """Writes text from in_stream to out_stream while recording in buffer.
+
+        Args:
+            in_stream: I/O stream to be read from.
+            out_stream: I/O stream to write to.
+
+        Returns:
+            A str containing everything read from the in_stream.
+        """
+        buffer = ""
+        read_size = 1024 if sys.version_info < (3, 7) else -1
+        chunk = in_stream.read1(read_size)
+        while chunk:
+            # Convert to str.
+            if not hasattr(chunk, "encode"):
+                chunk = chunk.decode("utf-8", "backslashreplace")
+
+            buffer += chunk
+            out_stream.write(chunk)
+            out_stream.flush()
+
+            chunk = in_stream.read1(read_size)
+
+        return buffer
+
+    @staticmethod
+    def _GetBasicEnv():
+        """Return a basic env for running git under.
+
+        This is guaranteed to be side-effect free.
+        """
+        env = os.environ.copy()
+        for key in (
+            REPO_TRACE,
+            GIT_DIR,
+            "GIT_ALTERNATE_OBJECT_DIRECTORIES",
+            "GIT_OBJECT_DIRECTORY",
+            "GIT_WORK_TREE",
+            "GIT_GRAFT_FILE",
+            "GIT_INDEX_FILE",
+        ):
+            env.pop(key, None)
+        return env
+
+    def VerifyCommand(self):
+        if self.rc == 0:
+            return None
+        stdout = (
+            "\n".join(self.stdout.split("\n")[:GIT_ERROR_STDOUT_LINES])
+            if self.stdout
+            else None
+        )
+        stderr = (
+            "\n".join(self.stderr.split("\n")[:GIT_ERROR_STDERR_LINES])
+            if self.stderr
+            else None
+        )
+        project = self.project.name if self.project else None
+        raise GitCommandError(
+            project=project,
+            command_args=self.cmdv,
+            git_rc=self.rc,
+            git_stdout=stdout,
+            git_stderr=stderr,
+        )
+
+    def Wait(self):
+        if self.verify_command:
+            self.VerifyCommand()
+        return self.rc
+
+
+class GitRequireError(RepoExitError):
+    """Error raised when git version is unavailable or invalid."""
+
+    def __init__(self, message, exit_code: int = INVALID_GIT_EXIT_CODE):
+        super().__init__(message, exit_code=exit_code)
+
+
+class GitCommandError(GitError):
+    """
+    Error raised from a failed git command.
+    Note that GitError can refer to any Git related error (e.g. branch not
+    specified for project.py 'UploadForReview'), while GitCommandError is
+    raised exclusively from non-zero exit codes returned from git commands.
+    """
+
+    # Tuples with error formats and suggestions for those errors.
+    _ERROR_TO_SUGGESTION = [
+        (
+            re.compile("couldn't find remote ref .*"),
+            "Check if the provided ref exists in the remote.",
+        ),
+        (
+            re.compile("unable to access '.*': .*"),
+            (
+                "Please make sure you have the correct access rights and the "
+                "repository exists."
+            ),
+        ),
+        (
+            re.compile("'.*' does not appear to be a git repository"),
+            "Are you running this repo command outside of a repo workspace?",
+        ),
+        (
+            re.compile("not a git repository"),
+            "Are you running this repo command outside of a repo workspace?",
+        ),
+    ]
+
+    def __init__(
+        self,
+        message: str = DEFAULT_GIT_FAIL_MESSAGE,
+        git_rc: int = None,
+        git_stdout: str = None,
+        git_stderr: str = None,
+        **kwargs,
+    ):
+        super().__init__(
+            message,
+            **kwargs,
+        )
+        self.git_rc = git_rc
+        self.git_stdout = git_stdout
+        self.git_stderr = git_stderr
+
+    @property
+    @functools.lru_cache(maxsize=None)  # noqa: B019
+    def suggestion(self):
+        """Returns helpful next steps for the given stderr."""
+        if not self.git_stderr:
+            return self.git_stderr
+
+        for err, suggestion in self._ERROR_TO_SUGGESTION:
+            if err.search(self.git_stderr):
+                return suggestion
+
+        return None
+
+    def __str__(self):
+        args = "[]" if not self.command_args else " ".join(self.command_args)
+        error_type = type(self).__name__
+        string = f"{error_type}: '{args}' on {self.project} failed"
+
+        if self.message != DEFAULT_GIT_FAIL_MESSAGE:
+            string += f": {self.message}"
+
+        if self.git_stdout:
+            string += f"\nstdout: {self.git_stdout}"
+
+        if self.git_stderr:
+            string += f"\nstderr: {self.git_stderr}"
+
+        if self.suggestion:
+            string += f"\nsuggestion: {self.suggestion}"
+
+        return string
+
+
+class GitPopenCommandError(GitError):
+    """
+    Error raised when subprocess.Popen fails for a GitCommand
+    """

git_refs.py

@@ -13,153 +13,184 @@
 # limitations under the License.
 
 import os
-from repo_trace import Trace
+
+from git_command import GitCommand
 import platform_utils
-
+from repo_trace import Trace
-HEAD = 'HEAD'
-R_CHANGES = 'refs/changes/'
-R_HEADS = 'refs/heads/'
-R_TAGS = 'refs/tags/'
-R_PUB = 'refs/published/'
-R_WORKTREE = 'refs/worktree/'
-R_WORKTREE_M = R_WORKTREE + 'm/'
-R_M = 'refs/remotes/m/'
 
 
-class GitRefs(object):
+HEAD = "HEAD"
-  def __init__(self, gitdir):
+R_CHANGES = "refs/changes/"
-    self._gitdir = gitdir
+R_HEADS = "refs/heads/"
-    self._phyref = None
+R_TAGS = "refs/tags/"
-    self._symref = None
+R_PUB = "refs/published/"
-    self._mtime = {}
+R_WORKTREE = "refs/worktree/"
+R_WORKTREE_M = R_WORKTREE + "m/"
+R_M = "refs/remotes/m/"
 
-  @property
-  def all(self):
-    self._EnsureLoaded()
-    return self._phyref
 
-  def get(self, name):
+class GitRefs:
-    try:
+    def __init__(self, gitdir):
-      return self.all[name]
+        self._gitdir = gitdir
-    except KeyError:
+        self._phyref = None
-      return ''
+        self._symref = None
+        self._mtime = {}
 
-  def deleted(self, name):
+    @property
-    if self._phyref is not None:
+    def all(self):
-      if name in self._phyref:
+        self._EnsureLoaded()
-        del self._phyref[name]
+        return self._phyref
 
-      if name in self._symref:
+    def get(self, name):
-        del self._symref[name]
+        try:
+            return self.all[name]
+        except KeyError:
+            return ""
 
-      if name in self._mtime:
+    def deleted(self, name):
-        del self._mtime[name]
+        if self._phyref is not None:
+            if name in self._phyref:
+                del self._phyref[name]
 
-  def symref(self, name):
+            if name in self._symref:
-    try:
+                del self._symref[name]
-      self._EnsureLoaded()
-      return self._symref[name]
-    except KeyError:
-      return ''
 
-  def _EnsureLoaded(self):
+            if name in self._mtime:
-    if self._phyref is None or self._NeedUpdate():
+                del self._mtime[name]
-      self._LoadAll()
 
-  def _NeedUpdate(self):
+    def symref(self, name):
-    Trace(': scan refs %s', self._gitdir)
+        try:
+            self._EnsureLoaded()
+            return self._symref[name]
+        except KeyError:
+            return ""
 
-    for name, mtime in self._mtime.items():
+    def _EnsureLoaded(self):
-      try:
+        if self._phyref is None or self._NeedUpdate():
-        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+            self._LoadAll()
-          return True
-      except OSError:
-        return True
-    return False
 
-  def _LoadAll(self):
+    def _NeedUpdate(self):
-    Trace(': load refs %s', self._gitdir)
+        with Trace(": scan refs %s", self._gitdir):
+            for name, mtime in self._mtime.items():
+                try:
+                    if mtime != os.path.getmtime(
+                        os.path.join(self._gitdir, name)
+                    ):
+                        return True
+                except OSError:
+                    return True
+            return False
 
-    self._phyref = {}
+    def _LoadAll(self):
-    self._symref = {}
+        with Trace(": load refs %s", self._gitdir):
-    self._mtime = {}
+            self._phyref = {}
+            self._symref = {}
+            self._mtime = {}
 
-    self._ReadPackedRefs()
+            self._ReadRefs()
-    self._ReadLoose('refs/')
+            self._ReadSymbolicRef(HEAD)
-    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
 
-    scan = self._symref
+            scan = self._symref
-    attempts = 0
+            attempts = 0
-    while scan and attempts < 5:
+            while scan and attempts < 5:
-      scan_next = {}
+                scan_next = {}
-      for name, dest in scan.items():
+                for name, dest in scan.items():
-        if dest in self._phyref:
+                    if dest in self._phyref:
-          self._phyref[name] = self._phyref[dest]
+                        self._phyref[name] = self._phyref[dest]
-        else:
+                    else:
-          scan_next[name] = dest
+                        scan_next[name] = dest
-      scan = scan_next
+                scan = scan_next
-      attempts += 1
+                attempts += 1
 
-  def _ReadPackedRefs(self):
+            self._TrackMtime(HEAD)
-    path = os.path.join(self._gitdir, 'packed-refs')
+            self._TrackMtime("config")
-    try:
+            self._TrackMtime("packed-refs")
-      fd = open(path, 'r')
+            self._TrackTreeMtimes("refs")
-      mtime = os.path.getmtime(path)
+            self._TrackTreeMtimes("reftable")
-    except IOError:
-      return
-    except OSError:
-      return
-    try:
-      for line in fd:
-        line = str(line)
-        if line[0] == '#':
-          continue
-        if line[0] == '^':
-          continue
 
-        line = line[:-1]
+    @staticmethod
-        p = line.split(' ')
+    def _IsNullRef(ref_id: str) -> bool:
-        ref_id = p[0]
+        """Check if a ref_id is a null object ID."""
-        name = p[1]
+        return ref_id and all(ch == "0" for ch in ref_id)
 
-        self._phyref[name] = ref_id
+    def _ReadRefs(self) -> None:
-    finally:
+        """Read all references using git for-each-ref."""
-      fd.close()
+        p = GitCommand(
-    self._mtime['packed-refs'] = mtime
+            None,
+            ["for-each-ref", "--format=%(objectname)%00%(refname)%00%(symref)"],
+            capture_stdout=True,
+            capture_stderr=True,
+            bare=True,
+            gitdir=self._gitdir,
+        )
+        if p.Wait() != 0:
+            return
 
-  def _ReadLoose(self, prefix):
+        for line in p.stdout.splitlines():
-    base = os.path.join(self._gitdir, prefix)
+            ref_id, name, symref = line.split("\0")
-    for name in platform_utils.listdir(base):
+            if symref:
-      p = os.path.join(base, name)
+                self._symref[name] = symref
-      # We don't implement the full ref validation algorithm, just the simple
+            elif ref_id and not self._IsNullRef(ref_id):
-      # rules that would show up in local filesystems.
+                self._phyref[name] = ref_id
-      # https://git-scm.com/docs/git-check-ref-format
-      if name.startswith('.') or name.endswith('.lock'):
-        pass
-      elif platform_utils.isdir(p):
-        self._mtime[prefix] = os.path.getmtime(base)
-        self._ReadLoose(prefix + name + '/')
-      else:
-        self._ReadLoose1(p, prefix + name)
 
-  def _ReadLoose1(self, path, name):
+    def _ReadSymbolicRef(self, name: str) -> None:
-    try:
+        """Read a symbolic reference."""
-      with open(path) as fd:
+        p = GitCommand(
-        mtime = os.path.getmtime(path)
+            None,
-        ref_id = fd.readline()
+            ["symbolic-ref", "-q", name],
-    except (OSError, UnicodeError):
+            capture_stdout=True,
-      return
+            capture_stderr=True,
+            bare=True,
+            gitdir=self._gitdir,
+        )
+        if p.Wait() == 0:
+            ref = p.stdout.strip()
+            if ref:
+                self._symref[name] = ref
+                return
 
-    try:
+        p = GitCommand(
-      ref_id = ref_id.decode()
+            None,
-    except AttributeError:
+            ["rev-parse", "--verify", "-q", name],
-      pass
+            capture_stdout=True,
-    if not ref_id:
+            capture_stderr=True,
-      return
+            bare=True,
-    ref_id = ref_id[:-1]
+            gitdir=self._gitdir,
+        )
+        if p.Wait() == 0:
+            ref_id = p.stdout.strip()
+            if ref_id:
+                self._phyref[name] = ref_id
 
-    if ref_id.startswith('ref: '):
+    def _TrackMtime(self, name: str) -> None:
-      self._symref[name] = ref_id[5:]
+        """Track the modification time of a specific gitdir path."""
-    else:
+        path = os.path.join(self._gitdir, name)
-      self._phyref[name] = ref_id
+        try:
-    self._mtime[name] = mtime
+            self._mtime[name] = os.path.getmtime(path)
+        except OSError:
+            return
+
+    def _TrackTreeMtimes(self, root: str) -> None:
+        """Recursively track modification times for a directory tree."""
+        root_path = os.path.join(self._gitdir, root)
+        try:
+            if not platform_utils.isdir(root_path):
+                return
+        except OSError:
+            return
+
+        to_scan = [root]
+        while to_scan:
+            name = to_scan.pop()
+            self._TrackMtime(name)
+            path = os.path.join(self._gitdir, name)
+            if not platform_utils.isdir(path):
+                continue
+
+            for child in platform_utils.listdir(path):
+                child_name = os.path.join(name, child)
+                child_path = os.path.join(self._gitdir, child_name)
+                if platform_utils.isdir(child_path):
+                    to_scan.append(child_name)
+                else:
+                    self._TrackMtime(child_name)

git_ssh

@@ -1,5 +1,4 @@
 #!/bin/sh
-#
 # Copyright (C) 2009 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

git_superproject.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Provide functionality to get all projects and their commit ids from Superproject.
+"""Provide functionality to get projects and their commit ids from Superproject.
 
 For more information on superproject, check out:
 https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
@@ -22,432 +22,626 @@ Examples:
   UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
-import hashlib
 import functools
+import glob
+import hashlib
 import os
 import sys
+import tempfile
 import time
 from typing import NamedTuple
+import urllib.parse
 
-from git_command import git_require, GitCommand
+from git_command import git_require
+from git_command import GitCommand
+from git_config import IsId
 from git_config import RepoConfig
-from git_refs import R_HEADS
+from git_refs import GitRefs
+import platform_utils
 
-_SUPERPROJECT_GIT_NAME = 'superproject.git'
+
-_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
+_SUPERPROJECT_GIT_NAME = "superproject.git"
+_SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml"
 
 
 class SyncResult(NamedTuple):
-  """Return the status of sync and whether caller should exit."""
+    """Return the status of sync and whether caller should exit."""
 
-  # Whether the superproject sync was successful.
+    # Whether the superproject sync was successful.
-  success: bool
+    success: bool
-  # Whether the caller should exit.
+    # Whether the caller should exit.
-  fatal: bool
+    fatal: bool
 
 
 class CommitIdsResult(NamedTuple):
-  """Return the commit ids and whether caller should exit."""
+    """Return the commit ids and whether caller should exit."""
 
-  # A dictionary with the projects/commit ids on success, otherwise None.
+    # A dictionary with the projects/commit ids on success, otherwise None.
-  commit_ids: dict
+    commit_ids: dict
-  # Whether the caller should exit.
+    # Whether the caller should exit.
-  fatal: bool
+    fatal: bool
 
 
 class UpdateProjectsResult(NamedTuple):
-  """Return the overriding manifest file and whether caller should exit."""
+    """Return the overriding manifest file and whether caller should exit."""
 
-  # Path name of the overriding manifest file if successful, otherwise None.
+    # Path name of the overriding manifest file if successful, otherwise None.
-  manifest_path: str
+    manifest_path: str
-  # Whether the caller should exit.
+    # Whether the caller should exit.
-  fatal: bool
+    fatal: bool
 
 
-class Superproject(object):
+class Superproject:
-  """Get commit ids from superproject.
+    """Get commit ids from superproject.
 
-  Initializes a local copy of a superproject for the manifest. This allows
+    Initializes a bare local copy of a superproject for the manifest. This
-  lookup of commit ids for all projects. It contains _project_commit_ids which
+    allows lookup of commit ids for all projects. It contains
-  is a dictionary with project/commit id entries.
+    _project_commit_ids which is a dictionary with project/commit id entries.
-  """
-  def __init__(self, manifest, name, remote, revision,
-               superproject_dir='exp-superproject'):
-    """Initializes superproject.
-
-    Args:
-      manifest: A Manifest object that is to be written to a file.
-      name: The unique name of the superproject
-      remote: The RemoteSpec for the remote.
-      revision: The name of the git branch to track.
-      superproject_dir: Relative path under |manifest.subdir| to checkout
-          superproject.
     """
-    self._project_commit_ids = None
-    self._manifest = manifest
-    self.name = name
-    self.remote = remote
-    self.revision = self._branch = revision
-    self._repodir = manifest.repodir
-    self._superproject_dir = superproject_dir
-    self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
-                                                          superproject_dir)
-    self._manifest_path = os.path.join(self._superproject_path,
-                                       _SUPERPROJECT_MANIFEST_NAME)
-    git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
-    self._remote_url = remote.url
-    self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
-    self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
-    # The following are command arguemnts, rather than superproject attributes,
+    def __init__(
-    # and were included here originally.  They should eventually become
+        self,
-    # arguments that are passed down from the public methods, instead of being
+        manifest,
-    # treated as attributes.
+        name,
-    self._git_event_log = None
+        remote,
-    self._quiet = False
+        revision,
-    self._print_messages = False
+        superproject_dir="exp-superproject",
+    ):
+        """Initializes superproject.
 
-  def SetQuiet(self, value):
+        Args:
-    """Set the _quiet attribute."""
+            manifest: A Manifest object that is to be written to a file.
-    self._quiet = value
+            name: The unique name of the superproject
+            remote: The RemoteSpec for the remote.
+            revision: The name of the git branch to track.
+            superproject_dir: Relative path under |manifest.subdir| to checkout
+                superproject.
+        """
+        self._project_commit_ids = None
+        self._manifest = manifest
+        self.name = name
+        self.remote = remote
+        self.revision = revision
+        self._repodir = manifest.repodir
+        self._superproject_dir = superproject_dir
+        self._superproject_path = manifest.SubmanifestInfoDir(
+            manifest.path_prefix, superproject_dir
+        )
+        self._manifest_path = os.path.join(
+            self._superproject_path, _SUPERPROJECT_MANIFEST_NAME
+        )
+        git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-"
+        self._remote_url = remote.url
+        self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
+        self._work_git = os.path.join(
+            self._superproject_path, self._work_git_name
+        )
 
-  def SetPrintMessages(self, value):
+        # The following are command arguemnts, rather than superproject
-    """Set the _print_messages attribute."""
+        # attributes, and were included here originally.  They should eventually
-    self._print_messages = value
+        # become arguments that are passed down from the public methods, instead
+        # of being treated as attributes.
+        self._git_event_log = None
+        self._quiet = False
+        self._print_messages = False
 
-  @property
+    def SetQuiet(self, value):
-  def project_commit_ids(self):
+        """Set the _quiet attribute."""
-    """Returns a dictionary of projects and their commit ids."""
+        self._quiet = value
-    return self._project_commit_ids
 
-  @property
+    def SetPrintMessages(self, value):
-  def manifest_path(self):
+        """Set the _print_messages attribute."""
-    """Returns the manifest path if the path exists or None."""
+        self._print_messages = value
-    return self._manifest_path if os.path.exists(self._manifest_path) else None
 
-  def _LogMessage(self, message):
+    def SetRevisionId(self, revision_id: str) -> None:
-    """Logs message to stderr and _git_event_log."""
+        """Set the revisionId of the superproject to sync to."""
-    if self._print_messages:
+        self.revision = revision_id
-      print(message, file=sys.stderr)
-    self._git_event_log.ErrorEvent(message, f'{message}')
 
-  def _LogMessagePrefix(self):
+    @property
-    """Returns the prefix string to be logged in each log message"""
+    def commit_id(self):
-    return f'repo superproject branch: {self._branch} url: {self._remote_url}'
+        """Returns the commit ID of the superproject checkout."""
+        cmd = ["rev-parse", self.revision]
+        p = GitCommand(
+            None,  # project
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval != 0:
+            self._LogWarning(
+                "git rev-parse call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return None
+        return p.stdout
 
-  def _LogError(self, message):
+    @property
-    """Logs error message to stderr and _git_event_log."""
+    def project_commit_ids(self):
-    self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
+        """Returns a dictionary of projects and their commit ids."""
+        return self._project_commit_ids
 
-  def _LogWarning(self, message):
+    @property
-    """Logs warning message to stderr and _git_event_log."""
+    def manifest_path(self):
-    self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
+        """Returns the manifest path if the path exists or None."""
+        return (
+            self._manifest_path if os.path.exists(self._manifest_path) else None
+        )
 
-  def _Init(self):
+    @property
-    """Sets up a local Git repository to get a copy of a superproject.
+    def repo_id(self):
+        """Returns the repo ID for the superproject.
 
-    Returns:
+        For example, if the superproject points to:
-      True if initialization is successful, or False.
+            https://android-review.googlesource.com/platform/superproject/
-    """
+        Then the repo_id would be:
-    if not os.path.exists(self._superproject_path):
+            android/platform/superproject
-      os.mkdir(self._superproject_path)
+        """
-    if not self._quiet and not os.path.exists(self._work_git):
+        review_url = self.remote.review
-      print('%s: Performing initial setup for superproject; this might take '
+        if review_url:
-            'several minutes.' % self._work_git)
+            parsed_url = urllib.parse.urlparse(review_url)
-    cmd = ['init', '--bare', self._work_git_name]
+            netloc = parsed_url.netloc
-    p = GitCommand(None,
+            if netloc:
-                   cmd,
+                parts = netloc.split("-review", 1)
-                   cwd=self._superproject_path,
+                host = parts[0]
-                   capture_stdout=True,
+                rev = GitRefs(self._work_git).get("HEAD")
-                   capture_stderr=True)
+                return f"{host}/{self.name}@{rev}"
-    retval = p.Wait()
+        return None
-    if retval:
-      self._LogWarning(f'git init call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
-      return False
-    return True
 
-  def _Fetch(self):
+    def _LogMessage(self, fmt, *inputs):
-    """Fetches a local copy of a superproject for the manifest based on |_remote_url|.
+        """Logs message to stderr and _git_event_log."""
+        message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}"
+        if self._print_messages:
+            print(message, file=sys.stderr)
+        if self._git_event_log:
+            self._git_event_log.ErrorEvent(message, fmt)
 
-    Returns:
+    def _LogMessagePrefix(self):
-      True if fetch is successful, or False.
+        """Returns the prefix string to be logged in each log message"""
-    """
+        return (
-    if not os.path.exists(self._work_git):
+            f"repo superproject revision: {self.revision} "
-      self._LogWarning(f'git fetch missing directory: {self._work_git}')
+            f"url: {self._remote_url}"
-      return False
+        )
-    if not git_require((2, 28, 0)):
-      self._LogWarning('superproject requires a git version 2.28 or later')
-      return False
-    cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
-           '--filter', 'blob:none']
-    if self._branch:
-      cmd += [self._branch + ':' + self._branch]
-    p = GitCommand(None,
-                   cmd,
-                   cwd=self._work_git,
-                   capture_stdout=True,
-                   capture_stderr=True)
-    retval = p.Wait()
-    if retval:
-      self._LogWarning(f'git fetch call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
-      return False
-    return True
 
-  def _LsTree(self):
+    def _LogError(self, fmt, *inputs):
-    """Gets the commit ids for all projects.
+        """Logs error message to stderr and _git_event_log."""
+        self._LogMessage(f"error: {fmt}", *inputs)
 
-    Works only in git repositories.
+    def _LogWarning(self, fmt, *inputs):
+        """Logs warning message to stderr and _git_event_log."""
+        self._LogMessage(f"warning: {fmt}", *inputs)
 
-    Returns:
+    def _Init(self):
-      data: data returned from 'git ls-tree ...' instead of None.
+        """Sets up a local Git repository to get a copy of a superproject.
-    """
-    if not os.path.exists(self._work_git):
-      self._LogWarning(f'git ls-tree missing directory: {self._work_git}')
-      return None
-    data = None
-    branch = 'HEAD' if not self._branch else self._branch
-    cmd = ['ls-tree', '-z', '-r', branch]
 
-    p = GitCommand(None,
+        Returns:
-                   cmd,
+            True if initialization is successful, or False.
-                   cwd=self._work_git,
+        """
-                   capture_stdout=True,
+        if not os.path.exists(self._superproject_path):
-                   capture_stderr=True)
+            os.mkdir(self._superproject_path)
-    retval = p.Wait()
-    if retval == 0:
-      data = p.stdout
-    else:
-      self._LogWarning(f'git ls-tree call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
-    return data
 
-  def Sync(self, git_event_log):
+        if os.path.exists(self._work_git):
-    """Gets a local copy of a superproject for the manifest.
+            return True
 
-    Args:
+        if not self._quiet:
-      git_event_log: an EventLog, for git tracing.
+            print(
+                "%s: Performing initial setup for superproject; this might "
+                "take several minutes." % self._work_git
+            )
 
-    Returns:
+        tmp_gitdir_prefix = ".tmp-superproject-initgitdir-"
-      SyncResult
+        tmp_gitdir = tempfile.mkdtemp(
-    """
+            prefix=tmp_gitdir_prefix,
-    self._git_event_log = git_event_log
+            dir=self._superproject_path,
-    if not self._manifest.superproject:
+        )
-      self._LogWarning(f'superproject tag is not defined in manifest: '
+        tmp_git_name = os.path.basename(tmp_gitdir)
-                       f'{self._manifest.manifestFile}')
-      return SyncResult(False, False)
 
-    _PrintBetaNotice()
+        try:
+            cmd = ["init", "--bare", tmp_git_name]
+            p = GitCommand(
+                None,
+                cmd,
+                cwd=self._superproject_path,
+                capture_stdout=True,
+                capture_stderr=True,
+            )
+            retval = p.Wait()
+            if retval:
+                self._LogWarning(
+                    "git init call failed, command: git {}, "
+                    "return code: {}, stderr: {}",
+                    cmd,
+                    retval,
+                    p.stderr,
+                )
+                return False
 
-    should_exit = True
+            platform_utils.rename(tmp_gitdir, self._work_git)
-    if not self._remote_url:
+            tmp_gitdir = None
-      self._LogWarning(f'superproject URL is not defined in manifest: '
+            return True
-                       f'{self._manifest.manifestFile}')
+        finally:
-      return SyncResult(False, should_exit)
+            # Clean up the temporary directory created during the process,
+            # as well as any stale ones left over from previous attempts.
+            if tmp_gitdir and os.path.exists(tmp_gitdir):
+                platform_utils.rmtree(tmp_gitdir)
 
-    if not self._Init():
+            age_threshold = 60 * 60 * 24  # 1 day in seconds
-      return SyncResult(False, should_exit)
+            now = time.time()
-    if not self._Fetch():
+            for tmp_dir in glob.glob(
-      return SyncResult(False, should_exit)
+                os.path.join(self._superproject_path, f"{tmp_gitdir_prefix}*")
-    if not self._quiet:
+            ):
-      print('%s: Initial setup for superproject completed.' % self._work_git)
+                try:
-    return SyncResult(True, False)
+                    mtime = os.path.getmtime(tmp_dir)
+                    if now - mtime > age_threshold:
+                        platform_utils.rmtree(tmp_dir)
+                except OSError:
+                    pass
 
-  def _GetAllProjectsCommitIds(self):
+    def _Fetch(self):
-    """Get commit ids for all projects from superproject and save them in _project_commit_ids.
+        """Fetches a superproject for the manifest based on |_remote_url|.
 
-    Returns:
+        This runs git fetch which stores a local copy the superproject.
-      CommitIdsResult
-    """
-    sync_result = self.Sync(self._git_event_log)
-    if not sync_result.success:
-      return CommitIdsResult(None, sync_result.fatal)
 
-    data = self._LsTree()
+        Returns:
-    if not data:
+            True if fetch is successful, or False.
-      self._LogWarning(f'git ls-tree failed to return data for manifest: '
+        """
-                       f'{self._manifest.manifestFile}')
+        if not os.path.exists(self._work_git):
-      return CommitIdsResult(None, True)
+            self._LogWarning("git fetch missing directory: {}", self._work_git)
+            return False
+        if not git_require((2, 28, 0)):
+            self._LogWarning(
+                "superproject requires a git version 2.28 or later"
+            )
+            return False
+        cmd = [
+            "fetch",
+            self._remote_url,
+            "--depth",
+            "1",
+            "--force",
+            "--no-tags",
+            "--filter",
+            "blob:none",
+        ]
 
-    # Parse lines like the following to select lines starting with '160000' and
+        # Check if there is a local ref that we can pass to --negotiation-tip.
-    # build a dictionary with project path (last element) and its commit id (3rd element).
+        # If this is the first fetch, it does not exist yet.
-    #
+        # We use --negotiation-tip to speed up the fetch. Superproject branches
-    # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
+        # do not share commits. So this lets git know it only needs to send
-    # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00
+        # commits reachable from the specified local refs.
-    commit_ids = {}
+        rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}")
-    for line in data.split('\x00'):
+        if rev_commit:
-      ls_data = line.split(None, 3)
+            cmd.extend(["--negotiation-tip", rev_commit])
-      if not ls_data:
-        break
-      if ls_data[0] == '160000':
-        commit_ids[ls_data[3]] = ls_data[2]
 
-    self._project_commit_ids = commit_ids
+        if self.revision:
-    return CommitIdsResult(commit_ids, False)
+            # If revision is a commit hash, fetch it directly to avoid
+            # creating a local branch of the same name.
+            refspec = (
+                self.revision
+                if IsId(self.revision)
+                else f"{self.revision}:{self.revision}"
+            )
+            cmd.append(refspec)
+        p = GitCommand(
+            None,
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval:
+            self._LogWarning(
+                "git fetch call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return False
+        return True
 
-  def _WriteManifestFile(self):
+    def _LsTree(self):
-    """Writes manifest to a file.
+        """Gets the commit ids for all projects.
 
-    Returns:
+        Works only in git repositories.
-      manifest_path: Path name of the file into which manifest is written instead of None.
-    """
-    if not os.path.exists(self._superproject_path):
-      self._LogWarning(f'missing superproject directory: {self._superproject_path}')
-      return None
-    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(),
-                                        omit_local=True).toxml()
-    manifest_path = self._manifest_path
-    try:
-      with open(manifest_path, 'w', encoding='utf-8') as fp:
-        fp.write(manifest_str)
-    except IOError as e:
-      self._LogError(f'cannot write manifest to : {manifest_path} {e}')
-      return None
-    return manifest_path
 
-  def _SkipUpdatingProjectRevisionId(self, project):
+        Returns:
-    """Checks if a project's revision id needs to be updated or not.
+            data: data returned from 'git ls-tree ...'. None on error.
+        """
+        if not os.path.exists(self._work_git):
+            self._LogWarning(
+                "git ls-tree missing directory: {}", self._work_git
+            )
+            return None
+        data = None
+        branch = "HEAD" if not self.revision else self.revision
+        cmd = ["ls-tree", "-z", "-r", branch]
 
-    Revision id for projects from local manifest will not be updated.
+        p = GitCommand(
+            None,
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval == 0:
+            data = p.stdout
+        else:
+            self._LogWarning(
+                "git ls-tree call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return None
+        return data
 
-    Args:
+    def Sync(self, git_event_log):
-      project: project whose revision id is being updated.
+        """Gets a local copy of a superproject for the manifest.
 
-    Returns:
+        Args:
-      True if a project's revision id should not be updated, or False,
+            git_event_log: an EventLog, for git tracing.
-    """
-    path = project.relpath
-    if not path:
-      return True
-    # Skip the project with revisionId.
-    if project.revisionId:
-      return True
-    # Skip the project if it comes from the local manifest.
-    return project.manifest.IsFromLocalManifest(project)
 
-  def UpdateProjectsRevisionId(self, projects, git_event_log):
+        Returns:
-    """Update revisionId of every project in projects with the commit id.
+            SyncResult
+        """
+        self._git_event_log = git_event_log
+        if not self._manifest.superproject:
+            self._LogWarning(
+                "superproject tag is not defined in manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return SyncResult(False, False)
 
-    Args:
+        should_exit = True
-      projects: a list of projects whose revisionId needs to be updated.
+        if not self._remote_url:
-      git_event_log: an EventLog, for git tracing.
+            self._LogWarning(
+                "superproject URL is not defined in manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return SyncResult(False, should_exit)
 
-    Returns:
+        if not self._Init():
-      UpdateProjectsResult
+            return SyncResult(False, should_exit)
-    """
+        if IsId(self.revision) and self.commit_id:
-    self._git_event_log = git_event_log
+            return SyncResult(True, False)
-    commit_ids_result = self._GetAllProjectsCommitIds()
+        if not self._Fetch():
-    commit_ids = commit_ids_result.commit_ids
+            return SyncResult(False, should_exit)
-    if not commit_ids:
+        if not self._quiet:
-      return UpdateProjectsResult(None, commit_ids_result.fatal)
+            print(
+                "%s: Initial setup for superproject completed." % self._work_git
+            )
+        return SyncResult(True, False)
 
-    projects_missing_commit_ids = []
+    def _GetAllProjectsCommitIds(self):
-    for project in projects:
+        """Get commit ids for all projects from superproject and save them.
-      if self._SkipUpdatingProjectRevisionId(project):
-        continue
-      path = project.relpath
-      commit_id = commit_ids.get(path)
-      if not commit_id:
-        projects_missing_commit_ids.append(path)
 
-    # If superproject doesn't have a commit id for a project, then report an
+        Commit ids are saved in _project_commit_ids.
-    # error event and continue as if do not use superproject is specified.
-    if projects_missing_commit_ids:
-      self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} '
-                       f'to report missing commit_ids for: {projects_missing_commit_ids}')
-      return UpdateProjectsResult(None, False)
 
-    for project in projects:
+        Returns:
-      if not self._SkipUpdatingProjectRevisionId(project):
+            CommitIdsResult
-        project.SetRevisionId(commit_ids.get(project.relpath))
+        """
+        sync_result = self.Sync(self._git_event_log)
+        if not sync_result.success:
+            return CommitIdsResult(None, sync_result.fatal)
 
-    manifest_path = self._WriteManifestFile()
+        data = self._LsTree()
-    return UpdateProjectsResult(manifest_path, False)
+        if not data:
+            self._LogWarning(
+                "git ls-tree failed to return data for manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return CommitIdsResult(None, True)
 
+        # Parse lines like the following to select lines starting with '160000'
+        # and build a dictionary with project path (last element) and its commit
+        # id (3rd element).
+        #
+        # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
+        # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00  # noqa: E501
+        commit_ids = {}
+        for line in data.split("\x00"):
+            ls_data = line.split(None, 3)
+            if not ls_data:
+                break
+            if ls_data[0] == "160000":
+                commit_ids[ls_data[3]] = ls_data[2]
 
-@functools.lru_cache(maxsize=10)
+        self._project_commit_ids = commit_ids
-def _PrintBetaNotice():
+        return CommitIdsResult(commit_ids, False)
-  """Print the notice of beta status."""
+
-  print('NOTICE: --use-superproject is in beta; report any issues to the '
+    def _WriteManifestFile(self):
-        'address described in `repo version`', file=sys.stderr)
+        """Writes manifest to a file.
+
+        Returns:
+            manifest_path: Path name of the file into which manifest is written
+                instead of None.
+        """
+        if not os.path.exists(self._superproject_path):
+            self._LogWarning(
+                "missing superproject directory: {}", self._superproject_path
+            )
+            return None
+        manifest_str = self._manifest.ToXml(
+            filter_groups=self._manifest.GetManifestGroupsStr(),
+            omit_local=True,
+        ).toxml()
+        manifest_path = self._manifest_path
+        try:
+            with open(manifest_path, "w", encoding="utf-8") as fp:
+                fp.write(manifest_str)
+        except OSError as e:
+            self._LogError("cannot write manifest to : {} {}", manifest_path, e)
+            return None
+        return manifest_path
+
+    def _SkipUpdatingProjectRevisionId(self, project):
+        """Checks if a project's revision id needs to be updated or not.
+
+        Revision id for projects from local manifest will not be updated.
+
+        Args:
+            project: project whose revision id is being updated.
+
+        Returns:
+            True if a project's revision id should not be updated, or False,
+        """
+        path = project.relpath
+        if not path:
+            return True
+        # Skip the project with revisionId.
+        if project.revisionId:
+            return True
+        # Skip the project if it comes from the local manifest.
+        return project.manifest.IsFromLocalManifest(project)
+
+    def UpdateProjectsRevisionId(self, projects, git_event_log):
+        """Update revisionId of every project in projects with the commit id.
+
+        Args:
+            projects: a list of projects whose revisionId needs to be updated.
+            git_event_log: an EventLog, for git tracing.
+
+        Returns:
+            UpdateProjectsResult
+        """
+        self._git_event_log = git_event_log
+        commit_ids_result = self._GetAllProjectsCommitIds()
+        commit_ids = commit_ids_result.commit_ids
+        if not commit_ids:
+            return UpdateProjectsResult(None, commit_ids_result.fatal)
+
+        projects_missing_commit_ids = []
+        for project in projects:
+            if self._SkipUpdatingProjectRevisionId(project):
+                continue
+            path = project.relpath
+            commit_id = commit_ids.get(path)
+            if not commit_id:
+                projects_missing_commit_ids.append(path)
+
+        # If superproject doesn't have a commit id for a project, then report an
+        # error event and continue as if do not use superproject is specified.
+        if projects_missing_commit_ids:
+            self._LogWarning(
+                "please file a bug using {} to report missing "
+                "commit_ids for: {}",
+                self._manifest.contactinfo.bugurl,
+                projects_missing_commit_ids,
+            )
+            return UpdateProjectsResult(None, False)
+
+        for project in projects:
+            if not self._SkipUpdatingProjectRevisionId(project):
+                project.SetRevisionId(commit_ids.get(project.relpath))
+
+        manifest_path = self._WriteManifestFile()
+        return UpdateProjectsResult(manifest_path, False)
 
 
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
-  """Returns the user choice of whether to use superproject."""
+    """Returns the user choice of whether to use superproject."""
-  user_cfg = RepoConfig.ForUser()
+    user_cfg = RepoConfig.ForUser()
-  time_now = int(time.time())
+    time_now = int(time.time())
 
-  user_value = user_cfg.GetBoolean('repo.superprojectChoice')
+    user_value = user_cfg.GetBoolean("repo.superprojectChoice")
-  if user_value is not None:
+    if user_value is not None:
-    user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
+        user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire")
-    if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now:
+        if (
-      # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+            user_expiration is None
-      # default value.
+            or user_expiration <= 0
-      if user_value:
+            or user_expiration >= time_now
-        print(('You are currently enrolled in Git submodules experiment '
+        ):
-               '(go/android-submodules-quickstart).  Use --no-use-superproject '
+            # TODO(b/190688390) - Remove prompt when we are comfortable with the
-               'to override.\n'), file=sys.stderr)
+            # new default value.
-      else:
+            if user_value:
-        print(('You are not currently enrolled in Git submodules experiment '
+                print(
-               '(go/android-submodules-quickstart).  Use --use-superproject '
+                    (
-               'to override.\n'), file=sys.stderr)
+                        "You are currently enrolled in Git submodules "
-      return user_value
+                        "experiment (go/android-submodules-quickstart).  Use "
+                        "--no-use-superproject to override.\n"
+                    ),
+                    file=sys.stderr,
+                )
+            else:
+                print(
+                    (
+                        "You are not currently enrolled in Git submodules "
+                        "experiment (go/android-submodules-quickstart).  Use "
+                        "--use-superproject to override.\n"
+                    ),
+                    file=sys.stderr,
+                )
+            return user_value
 
-  # We don't have an unexpired choice, ask for one.
+    # We don't have an unexpired choice, ask for one.
-  system_cfg = RepoConfig.ForSystem()
+    system_cfg = RepoConfig.ForSystem()
-  system_value = system_cfg.GetBoolean('repo.superprojectChoice')
+    system_value = system_cfg.GetBoolean("repo.superprojectChoice")
-  if system_value:
+    if system_value:
-    # The system configuration is proposing that we should enable the
+        # The system configuration is proposing that we should enable the
-    # use of superproject. Treat the user as enrolled for two weeks.
+        # use of superproject. Treat the user as enrolled for two weeks.
-    #
+        #
-    # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+        # TODO(b/190688390) - Remove prompt when we are comfortable with the new
-    # default value.
+        # default value.
-    userchoice = True
+        userchoice = True
-    time_choiceexpire = time_now + (86400 * 14)
+        time_choiceexpire = time_now + (86400 * 14)
-    user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
+        user_cfg.SetString(
-    user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
+            "repo.superprojectChoiceExpire", str(time_choiceexpire)
-    print('You are automatically enrolled in Git submodules experiment '
+        )
-          '(go/android-submodules-quickstart) for another two weeks.\n',
+        user_cfg.SetBoolean("repo.superprojectChoice", userchoice)
-          file=sys.stderr)
+        print(
-    return True
+            "You are automatically enrolled in Git submodules experiment "
+            "(go/android-submodules-quickstart) for another two weeks.\n",
+            file=sys.stderr,
+        )
+        return True
 
-  # For all other cases, we would not use superproject by default.
+    # For all other cases, we would not use superproject by default.
-  return False
+    return False
 
 
 def PrintMessages(use_superproject, manifest):
-  """Returns a boolean if error/warning messages are to be printed.
+    """Returns a boolean if error/warning messages are to be printed.
 
-  Args:
+    Args:
-    use_superproject: option value from optparse.
+        use_superproject: option value from optparse.
-    manifest: manifest to use.
+        manifest: manifest to use.
-  """
+    """
-  return use_superproject is not None or bool(manifest.superproject)
+    return use_superproject is not None or bool(manifest.superproject)
 
 
 def UseSuperproject(use_superproject, manifest):
-  """Returns a boolean if use-superproject option is enabled.
+    """Returns a boolean if use-superproject option is enabled.
 
-  Args:
+    Args:
-    use_superproject: option value from optparse.
+        use_superproject: option value from optparse.
-    manifest: manifest to use.
+        manifest: manifest to use.
 
-  Returns:
+    Returns:
-    Whether the superproject should be used.
+        Whether the superproject should be used.
-  """
+    """
 
-  if not manifest.superproject:
+    if not manifest.superproject:
-    # This (sub) manifest does not have a superproject definition.
+        # This (sub) manifest does not have a superproject definition.
-    return False
+        return False
-  elif use_superproject is not None:
+    elif use_superproject is not None:
-    return use_superproject
+        return use_superproject
-  else:
-    client_value = manifest.manifestProject.use_superproject
-    if client_value is not None:
-      return client_value
-    elif manifest.superproject:
-      return _UseSuperprojectFromConfiguration()
     else:
-      return False
+        client_value = manifest.manifestProject.use_superproject
+        if client_value is not None:
+            return client_value
+        elif manifest.superproject:
+            return _UseSuperprojectFromConfiguration()
+        else:
+            return False

git_trace2_event_log.py

@@ -12,320 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Provide event logging in the git trace2 EVENT format.
+"""Event logging in the git trace2 EVENT format."""
 
-The git trace2 EVENT format is defined at:
+from git_command import GetEventTargetPath
-https://www.kernel.org/pub/software/scm/git/docs/technical/api-trace2.html#_event_format
+from git_command import RepoSourceVersion
-https://git-scm.com/docs/api-trace2#_the_event_format_target
+from git_trace2_event_log_base import BaseEventLog
-
-  Usage:
-
-  git_trace_log = EventLog()
-  git_trace_log.StartEvent()
-  ...
-  git_trace_log.ExitEvent()
-  git_trace_log.Write()
-"""
 
 
-import datetime
+class EventLog(BaseEventLog):
-import errno
+    """Event log that records events that occurred during a repo invocation.
-import json
-import os
-import socket
-import sys
-import tempfile
-import threading
 
-from git_command import GitCommand, RepoSourceVersion
+    Events are written to the log as a consecutive JSON entries, one per line.
+    Entries follow the git trace2 EVENT format.
 
+    Each entry contains the following common keys:
+    - event: The event name
+    - sid: session-id - Unique string to allow process instance to be
+          identified.
+    - thread: The thread name.
+    - time: is the UTC time of the event.
 
-class EventLog(object):
+    Valid 'event' names and event specific fields are documented here:
-  """Event log that records events that occurred during a repo invocation.
+    https://git-scm.com/docs/api-trace2#_event_format
-
-  Events are written to the log as a consecutive JSON entries, one per line.
-  Entries follow the git trace2 EVENT format.
-
-  Each entry contains the following common keys:
-  - event: The event name
-  - sid: session-id - Unique string to allow process instance to be identified.
-  - thread: The thread name.
-  - time: is the UTC time of the event.
-
-  Valid 'event' names and event specific fields are documented here:
-  https://git-scm.com/docs/api-trace2#_event_format
-  """
-
-  def __init__(self, env=None):
-    """Initializes the event log."""
-    self._log = []
-    # Try to get session-id (sid) from environment (setup in repo launcher).
-    KEY = 'GIT_TRACE2_PARENT_SID'
-    if env is None:
-      env = os.environ
-
-    now = datetime.datetime.utcnow()
-
-    # Save both our sid component and the complete sid.
-    # We use our sid component (self._sid) as the unique filename prefix and
-    # the full sid (self._full_sid) in the log itself.
-    self._sid = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid())
-    parent_sid = env.get(KEY)
-    # Append our sid component to the parent sid (if it exists).
-    if parent_sid is not None:
-      self._full_sid = parent_sid + '/' + self._sid
-    else:
-      self._full_sid = self._sid
-
-    # Set/update the environment variable.
-    # Environment handling across systems is messy.
-    try:
-      env[KEY] = self._full_sid
-    except UnicodeEncodeError:
-      env[KEY] = self._full_sid.encode()
-
-    # Add a version event to front of the log.
-    self._AddVersionEvent()
-
-  @property
-  def full_sid(self):
-    return self._full_sid
-
-  def _AddVersionEvent(self):
-    """Adds a 'version' event at the beginning of current log."""
-    version_event = self._CreateEventDict('version')
-    version_event['evt'] = "2"
-    version_event['exe'] = RepoSourceVersion()
-    self._log.insert(0, version_event)
-
-  def _CreateEventDict(self, event_name):
-    """Returns a dictionary with the common keys/values for git trace2 events.
-
-    Args:
-      event_name: The event name.
-
-    Returns:
-      Dictionary with the common event fields populated.
-    """
-    return {
-        'event': event_name,
-        'sid': self._full_sid,
-        'thread': threading.currentThread().getName(),
-        'time': datetime.datetime.utcnow().isoformat() + 'Z',
-    }
-
-  def StartEvent(self):
-    """Append a 'start' event to the current log."""
-    start_event = self._CreateEventDict('start')
-    start_event['argv'] = sys.argv
-    self._log.append(start_event)
-
-  def ExitEvent(self, result):
-    """Append an 'exit' event to the current log.
-
-    Args:
-      result: Exit code of the event
-    """
-    exit_event = self._CreateEventDict('exit')
-
-    # Consider 'None' success (consistent with event_log result handling).
-    if result is None:
-      result = 0
-    exit_event['code'] = result
-    self._log.append(exit_event)
-
-  def CommandEvent(self, name, subcommands):
-    """Append a 'command' event to the current log.
-
-    Args:
-      name: Name of the primary command (ex: repo, git)
-      subcommands: List of the sub-commands (ex: version, init, sync)
-    """
-    command_event = self._CreateEventDict('command')
-    command_event['name'] = name
-    command_event['subcommands'] = subcommands
-    self._log.append(command_event)
-
-  def LogConfigEvents(self, config, event_dict_name):
-    """Append a |event_dict_name| event for each config key in |config|.
-
-    Args:
-      config: Configuration dictionary.
-      event_dict_name: Name of the event dictionary for items to be logged under.
-    """
-    for param, value in config.items():
-      event = self._CreateEventDict(event_dict_name)
-      event['param'] = param
-      event['value'] = value
-      self._log.append(event)
-
-  def DefParamRepoEvents(self, config):
-    """Append a 'def_param' event for each repo.* config key to the current log.
-
-    Args:
-      config: Repo configuration dictionary
-    """
-    # Only output the repo.* config parameters.
-    repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
-    self.LogConfigEvents(repo_config, 'def_param')
-
-  def GetDataEventName(self, value):
-    """Returns 'data-json' if the value is an array else returns 'data'."""
-    return 'data-json' if value[0] == '[' and value[-1] == ']' else 'data'
-
-  def LogDataConfigEvents(self, config, prefix):
-    """Append a 'data' event for each config key/value in |config| to the current log.
-
-    For each keyX and valueX of the config, "key" field of the event is '|prefix|/keyX'
-    and the "value" of the "key" field is valueX.
-
-    Args:
-      config: Configuration dictionary.
-      prefix: Prefix for each key that is logged.
-    """
-    for key, value in config.items():
-      event = self._CreateEventDict(self.GetDataEventName(value))
-      event['key'] = f'{prefix}/{key}'
-      event['value'] = value
-      self._log.append(event)
-
-  def ErrorEvent(self, msg, fmt):
-    """Append a 'error' event to the current log."""
-    error_event = self._CreateEventDict('error')
-    error_event['msg'] = msg
-    error_event['fmt'] = fmt
-    self._log.append(error_event)
-
-  def _GetEventTargetPath(self):
-    """Get the 'trace2.eventtarget' path from git configuration.
-
-    Returns:
-      path: git config's 'trace2.eventtarget' path if it exists, or None
-    """
-    path = None
-    cmd = ['config', '--get', 'trace2.eventtarget']
-    # TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
-    # system git config variables.
-    p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
-                   bare=True)
-    retval = p.Wait()
-    if retval == 0:
-      # Strip trailing carriage-return in path.
-      path = p.stdout.rstrip('\n')
-    elif retval != 1:
-      # `git config --get` is documented to produce an exit status of `1` if
-      # the requested variable is not present in the configuration. Report any
-      # other return value as an error.
-      print("repo: error: 'git config --get' call failed with return code: %r, stderr: %r" % (
-          retval, p.stderr), file=sys.stderr)
-    return path
-
-  def _WriteLog(self, write_fn):
-    """Writes the log out using a provided writer function.
-
-    Generate compact JSON output for each item in the log, and write it using
-    write_fn.
-
-    Args:
-      write_fn: A function that accepts byts and writes them to a destination.
     """
 
-    for e in self._log:
+    def __init__(self, **kwargs):
-      # Dump in compact encoding mode.
+        super().__init__(repo_source_version=RepoSourceVersion(), **kwargs)
-      # See 'Compact encoding' in Python docs:
-      # https://docs.python.org/3/library/json.html#module-json
-      write_fn(json.dumps(e, indent=None, separators=(',', ':')).encode('utf-8') + b'\n')
 
-  def Write(self, path=None):
+    def Write(self, path=None, **kwargs):
-    """Writes the log out to a file or socket.
+        if path is None:
+            path = self._GetEventTargetPath()
+        return super().Write(path=path, **kwargs)
 
-    Log is only written if 'path' or 'git config --get trace2.eventtarget'
+    def _GetEventTargetPath(self):
-    provide a valid path (or socket) to write logs to.
+        return GetEventTargetPath()
-
-    Logging filename format follows the git trace2 style of being a unique
-    (exclusive writable) file.
-
-    Args:
-      path: Path to where logs should be written. The path may have a prefix of
-          the form "af_unix:[{stream|dgram}:]", in which case the path is
-          treated as a Unix domain socket. See
-          https://git-scm.com/docs/api-trace2#_enabling_a_target for details.
-
-    Returns:
-      log_path: Path to the log file or socket if log is written, otherwise None
-    """
-    log_path = None
-    # If no logging path is specified, get the path from 'trace2.eventtarget'.
-    if path is None:
-      path = self._GetEventTargetPath()
-
-    # If no logging path is specified, exit.
-    if path is None:
-      return None
-
-    path_is_socket = False
-    socket_type = None
-    if isinstance(path, str):
-      parts = path.split(':', 1)
-      if parts[0] == 'af_unix' and len(parts) == 2:
-        path_is_socket = True
-        path = parts[1]
-        parts = path.split(':', 1)
-        if parts[0] == 'stream' and len(parts) == 2:
-          socket_type = socket.SOCK_STREAM
-          path = parts[1]
-        elif parts[0] == 'dgram' and len(parts) == 2:
-          socket_type = socket.SOCK_DGRAM
-          path = parts[1]
-      else:
-        # Get absolute path.
-        path = os.path.abspath(os.path.expanduser(path))
-    else:
-      raise TypeError('path: str required but got %s.' % type(path))
-
-    # Git trace2 requires a directory to write log to.
-
-    # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
-    if not (path_is_socket or os.path.isdir(path)):
-      return None
-
-    if path_is_socket:
-      if socket_type == socket.SOCK_STREAM or socket_type is None:
-        try:
-          with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
-            sock.connect(path)
-            self._WriteLog(sock.sendall)
-          return f'af_unix:stream:{path}'
-        except OSError as err:
-          # If we tried to connect to a DGRAM socket using STREAM, ignore the
-          # attempt and continue to DGRAM below. Otherwise, issue a warning.
-          if err.errno != errno.EPROTOTYPE:
-            print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
-            return None
-      if socket_type == socket.SOCK_DGRAM or socket_type is None:
-        try:
-          with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
-            self._WriteLog(lambda bs: sock.sendto(bs, path))
-            return f'af_unix:dgram:{path}'
-        except OSError as err:
-          print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
-          return None
-      # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
-      # (SOCK_DGRAM).
-      print('repo: warning: git trace2 logging failed: could not write to socket', file=sys.stderr)
-      return None
-
-    # Path is an absolute path
-    # Use NamedTemporaryFile to generate a unique filename as required by git trace2.
-    try:
-      with tempfile.NamedTemporaryFile(mode='xb', prefix=self._sid, dir=path,
-                                       delete=False) as f:
-        # TODO(https://crbug.com/gerrit/13706): Support writing events as they
-        # occur.
-        self._WriteLog(f.write)
-        log_path = f.name
-    except FileExistsError as err:
-      print('repo: warning: git trace2 logging failed: %r' % err,
-            file=sys.stderr)
-      return None
-    return log_path

git_trace2_event_log_base.py

@@ -0,0 +1,362 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provide event logging in the git trace2 EVENT format.
+
+The git trace2 EVENT format is defined at:
+https://www.kernel.org/pub/software/scm/git/docs/technical/api-trace2.html#_event_format
+https://git-scm.com/docs/api-trace2#_the_event_format_target
+
+  Usage:
+
+  git_trace_log = EventLog()
+  git_trace_log.StartEvent()
+  ...
+  git_trace_log.ExitEvent()
+  git_trace_log.Write()
+"""
+
+
+import datetime
+import errno
+import json
+import os
+import socket
+import sys
+import tempfile
+import threading
+
+
+# Timeout when sending events via socket (applies to connect, send)
+SOCK_TIMEOUT = 0.5  # in seconds
+# BaseEventLog __init__ Counter that is consistent within the same process
+p_init_count = 0
+
+
+class BaseEventLog:
+    """Event log that records events that occurred during a repo invocation.
+
+    Events are written to the log as a consecutive JSON entries, one per line.
+    Entries follow the git trace2 EVENT format.
+
+    Each entry contains the following common keys:
+    - event: The event name
+    - sid: session-id - Unique string to allow process instance to be
+          identified.
+    - thread: The thread name.
+    - time: is the UTC time of the event.
+
+    Valid 'event' names and event specific fields are documented here:
+    https://git-scm.com/docs/api-trace2#_event_format
+    """
+
+    def __init__(
+        self, env=None, repo_source_version=None, add_init_count=False
+    ):
+        """Initializes the event log."""
+        global p_init_count
+        p_init_count += 1
+        self._log = []
+        self.verbose = False
+        # Try to get session-id (sid) from environment (setup in repo launcher).
+        KEY = "GIT_TRACE2_PARENT_SID"
+        if env is None:
+            env = os.environ
+
+        self.start = datetime.datetime.now(datetime.timezone.utc)
+
+        # Save both our sid component and the complete sid.
+        # We use our sid component (self._sid) as the unique filename prefix and
+        # the full sid (self._full_sid) in the log itself.
+        self._sid = (
+            f"repo-{self.start.strftime('%Y%m%dT%H%M%SZ')}-P{os.getpid():08x}"
+        )
+
+        if add_init_count:
+            self._sid = f"{self._sid}-{p_init_count}"
+
+        parent_sid = env.get(KEY)
+        # Append our sid component to the parent sid (if it exists).
+        if parent_sid is not None:
+            self._full_sid = parent_sid + "/" + self._sid
+        else:
+            self._full_sid = self._sid
+
+        # Set/update the environment variable.
+        # Environment handling across systems is messy.
+        try:
+            env[KEY] = self._full_sid
+        except UnicodeEncodeError:
+            env[KEY] = self._full_sid.encode()
+
+        if repo_source_version is not None:
+            # Add a version event to front of the log.
+            self._AddVersionEvent(repo_source_version)
+
+    @property
+    def full_sid(self):
+        return self._full_sid
+
+    def _AddVersionEvent(self, repo_source_version):
+        """Adds a 'version' event at the beginning of current log."""
+        version_event = self._CreateEventDict("version")
+        version_event["evt"] = "2"
+        version_event["exe"] = repo_source_version
+        self._log.insert(0, version_event)
+
+    def _CreateEventDict(self, event_name):
+        """Returns a dictionary with common keys/values for git trace2 events.
+
+        Args:
+            event_name: The event name.
+
+        Returns:
+            Dictionary with the common event fields populated.
+        """
+        return {
+            "event": event_name,
+            "sid": self._full_sid,
+            "thread": threading.current_thread().name,
+            "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        }
+
+    def StartEvent(self, argv):
+        """Append a 'start' event to the current log."""
+        start_event = self._CreateEventDict("start")
+        start_event["argv"] = argv
+        self._log.append(start_event)
+
+    def ExitEvent(self, result):
+        """Append an 'exit' event to the current log.
+
+        Args:
+            result: Exit code of the event
+        """
+        exit_event = self._CreateEventDict("exit")
+
+        # Consider 'None' success (consistent with event_log result handling).
+        if result is None:
+            result = 0
+        exit_event["code"] = result
+        time_delta = datetime.datetime.now(datetime.timezone.utc) - self.start
+        exit_event["t_abs"] = time_delta.total_seconds()
+        self._log.append(exit_event)
+
+    def CommandEvent(self, name, subcommands):
+        """Append a 'command' event to the current log.
+
+        Args:
+            name: Name of the primary command (ex: repo, git)
+            subcommands: List of the sub-commands (ex: version, init, sync)
+        """
+        command_event = self._CreateEventDict("cmd_name")
+        name = f"{name}-"
+        name += "-".join(subcommands)
+        command_event["name"] = name
+        command_event["hierarchy"] = name
+        self._log.append(command_event)
+
+    def LogConfigEvents(self, config, event_dict_name):
+        """Append a |event_dict_name| event for each config key in |config|.
+
+        Args:
+            config: Configuration dictionary.
+            event_dict_name: Name of the event dictionary for items to be logged
+                under.
+        """
+        for param, value in config.items():
+            event = self._CreateEventDict(event_dict_name)
+            event["param"] = param
+            event["value"] = value
+            self._log.append(event)
+
+    def DefParamRepoEvents(self, config):
+        """Append 'def_param' events for repo config keys to the current log.
+
+        This appends one event for each repo.* config key.
+
+        Args:
+            config: Repo configuration dictionary
+        """
+        # Only output the repo.* config parameters.
+        repo_config = {k: v for k, v in config.items() if k.startswith("repo.")}
+        self.LogConfigEvents(repo_config, "def_param")
+
+    def GetDataEventName(self, value):
+        """Returns 'data-json' if the value is an array else returns 'data'."""
+        return "data-json" if value[0] == "[" and value[-1] == "]" else "data"
+
+    def LogDataConfigEvents(self, config, prefix):
+        """Append a 'data' event for each entry in |config| to the current log.
+
+        For each keyX and valueX of the config, "key" field of the event is
+        '|prefix|/keyX' and the "value" of the "key" field is valueX.
+
+        Args:
+            config: Configuration dictionary.
+            prefix: Prefix for each key that is logged.
+        """
+        for key, value in config.items():
+            event = self._CreateEventDict(self.GetDataEventName(value))
+            event["key"] = f"{prefix}/{key}"
+            event["value"] = value
+            self._log.append(event)
+
+    def ErrorEvent(self, msg, fmt=None):
+        """Append a 'error' event to the current log."""
+        error_event = self._CreateEventDict("error")
+        if fmt is None:
+            fmt = msg
+        error_event["msg"] = f"RepoErrorEvent:{msg}"
+        error_event["fmt"] = f"RepoErrorEvent:{fmt}"
+        self._log.append(error_event)
+
+    def _WriteLog(self, write_fn):
+        """Writes the log out using a provided writer function.
+
+        Generate compact JSON output for each item in the log, and write it
+        using write_fn.
+
+        Args:
+            write_fn: A function that accepts byts and writes them to a
+                destination.
+        """
+
+        for e in self._log:
+            # Dump in compact encoding mode.
+            # See 'Compact encoding' in Python docs:
+            # https://docs.python.org/3/library/json.html#module-json
+            write_fn(
+                json.dumps(e, indent=None, separators=(",", ":")).encode(
+                    "utf-8"
+                )
+                + b"\n"
+            )
+
+    def Write(self, path=None):
+        """Writes the log out to a file or socket.
+
+        Log is only written if 'path' or 'git config --get trace2.eventtarget'
+        provide a valid path (or socket) to write logs to.
+
+        Logging filename format follows the git trace2 style of being a unique
+        (exclusive writable) file.
+
+        Args:
+            path: Path to where logs should be written. The path may have a
+                prefix of the form "af_unix:[{stream|dgram}:]", in which case
+                the path is treated as a Unix domain socket. See
+                https://git-scm.com/docs/api-trace2#_enabling_a_target for
+                details.
+
+        Returns:
+            log_path: Path to the log file or socket if log is written,
+                otherwise None
+        """
+        log_path = None
+        # If no logging path is specified, exit.
+        if path is None:
+            return None
+
+        path_is_socket = False
+        socket_type = None
+        if isinstance(path, str):
+            parts = path.split(":", 1)
+            if parts[0] == "af_unix" and len(parts) == 2:
+                path_is_socket = True
+                path = parts[1]
+                parts = path.split(":", 1)
+                if parts[0] == "stream" and len(parts) == 2:
+                    socket_type = socket.SOCK_STREAM
+                    path = parts[1]
+                elif parts[0] == "dgram" and len(parts) == 2:
+                    socket_type = socket.SOCK_DGRAM
+                    path = parts[1]
+            else:
+                # Get absolute path.
+                path = os.path.abspath(os.path.expanduser(path))
+        else:
+            raise TypeError("path: str required but got %s." % type(path))
+
+        # Git trace2 requires a directory to write log to.
+
+        # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
+        if not (path_is_socket or os.path.isdir(path)):
+            return None
+
+        if path_is_socket:
+            if socket_type == socket.SOCK_STREAM or socket_type is None:
+                try:
+                    with socket.socket(
+                        socket.AF_UNIX, socket.SOCK_STREAM
+                    ) as sock:
+                        sock.settimeout(SOCK_TIMEOUT)
+                        sock.connect(path)
+                        self._WriteLog(sock.sendall)
+                    return f"af_unix:stream:{path}"
+                except OSError as err:
+                    # If we tried to connect to a DGRAM socket using STREAM,
+                    # ignore the attempt and continue to DGRAM below. Otherwise,
+                    # issue a warning.
+                    if err.errno != errno.EPROTOTYPE:
+                        if self.verbose:
+                            print(
+                                "repo: warning: git trace2 logging failed:",
+                                f"{err}",
+                                file=sys.stderr,
+                            )
+                        return None
+            if socket_type == socket.SOCK_DGRAM or socket_type is None:
+                try:
+                    with socket.socket(
+                        socket.AF_UNIX, socket.SOCK_DGRAM
+                    ) as sock:
+                        self._WriteLog(lambda bs: sock.sendto(bs, path))
+                        return f"af_unix:dgram:{path}"
+                except OSError as err:
+                    if self.verbose:
+                        print(
+                            f"repo: warning: git trace2 logging failed: {err}",
+                            file=sys.stderr,
+                        )
+                    return None
+            # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
+            # (SOCK_DGRAM).
+            if self.verbose:
+                print(
+                    "repo: warning: git trace2 logging failed: could not"
+                    "write to socket",
+                    file=sys.stderr,
+                )
+            return None
+
+        # Path is an absolute path
+        # Use NamedTemporaryFile to generate a unique filename as required by
+        # git trace2.
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="xb", prefix=self._sid, dir=path, delete=False
+            ) as f:
+                # TODO(https://crbug.com/gerrit/13706): Support writing events
+                # as they occur.
+                self._WriteLog(f.write)
+                log_path = f.name
+        except FileExistsError as err:
+            if self.verbose:
+                print(
+                    "repo: warning: git trace2 logging failed: %r" % err,
+                    file=sys.stderr,
+                )
+            return None
+        return log_path

gitc_utils.py

@@ -1,156 +0,0 @@
-# Copyright (C) 2015 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import multiprocessing
-import platform
-import re
-import sys
-import time
-
-import git_command
-import git_config
-import wrapper
-
-from error import ManifestParseError
-
-NUM_BATCH_RETRIEVE_REVISIONID = 32
-
-
-def get_gitc_manifest_dir():
-  return wrapper.Wrapper().get_gitc_manifest_dir()
-
-
-def parse_clientdir(gitc_fs_path):
-  return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
-
-
-def _get_project_revision(args):
-  """Worker for _set_project_revisions to lookup one project remote."""
-  (i, url, expr) = args
-  gitcmd = git_command.GitCommand(
-      None, ['ls-remote', url, expr], capture_stdout=True, cwd='/tmp')
-  rc = gitcmd.Wait()
-  return (i, rc, gitcmd.stdout.split('\t', 1)[0])
-
-
-def _set_project_revisions(projects):
-  """Sets the revisionExpr for a list of projects.
-
-  Because of the limit of open file descriptors allowed, length of projects
-  should not be overly large. Recommend calling this function multiple times
-  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
-
-  Args:
-    projects: List of project objects to set the revionExpr for.
-  """
-  # Retrieve the commit id for each project based off of it's current
-  # revisionExpr and it is not already a commit id.
-  with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool:
-    results_iter = pool.imap_unordered(
-        _get_project_revision,
-        ((i, project.remote.url, project.revisionExpr)
-         for i, project in enumerate(projects)
-         if not git_config.IsId(project.revisionExpr)),
-        chunksize=8)
-    for (i, rc, revisionExpr) in results_iter:
-      project = projects[i]
-      if rc:
-        print('FATAL: Failed to retrieve revisionExpr for %s' % project.name)
-        pool.terminate()
-        sys.exit(1)
-      if not revisionExpr:
-        pool.terminate()
-        raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
-                                 (project.remote.url, project.revisionExpr))
-      project.revisionExpr = revisionExpr
-
-
-def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
-  """Generate a manifest for shafsd to use for this GITC client.
-
-  Args:
-    gitc_manifest: Current gitc manifest, or None if there isn't one yet.
-    manifest: A GitcManifest object loaded with the current repo manifest.
-    paths: List of project paths we want to update.
-  """
-
-  print('Generating GITC Manifest by fetching revision SHAs for each '
-        'project.')
-  if paths is None:
-    paths = list(manifest.paths.keys())
-
-  groups = [x for x in re.split(r'[,\s]+', manifest.GetGroupsStr()) if x]
-
-  # Convert the paths to projects, and filter them to the matched groups.
-  projects = [manifest.paths[p] for p in paths]
-  projects = [p for p in projects if p.MatchesGroups(groups)]
-
-  if gitc_manifest is not None:
-    for path, proj in manifest.paths.items():
-      if not proj.MatchesGroups(groups):
-        continue
-
-      if not proj.upstream and not git_config.IsId(proj.revisionExpr):
-        proj.upstream = proj.revisionExpr
-
-      if path not in gitc_manifest.paths:
-        # Any new projects need their first revision, even if we weren't asked
-        # for them.
-        projects.append(proj)
-      elif path not in paths:
-        # And copy revisions from the previous manifest if we're not updating
-        # them now.
-        gitc_proj = gitc_manifest.paths[path]
-        if gitc_proj.old_revision:
-          proj.revisionExpr = None
-          proj.old_revision = gitc_proj.old_revision
-        else:
-          proj.revisionExpr = gitc_proj.revisionExpr
-
-  _set_project_revisions(projects)
-
-  if gitc_manifest is not None:
-    for path, proj in gitc_manifest.paths.items():
-      if proj.old_revision and path in paths:
-        # If we updated a project that has been started, keep the old-revision
-        # updated.
-        repo_proj = manifest.paths[path]
-        repo_proj.old_revision = repo_proj.revisionExpr
-        repo_proj.revisionExpr = None
-
-  # Convert URLs from relative to absolute.
-  for _name, remote in manifest.remotes.items():
-    remote.fetchUrl = remote.resolvedFetchUrl
-
-  # Save the manifest.
-  save_manifest(manifest)
-
-
-def save_manifest(manifest, client_dir=None):
-  """Save the manifest file in the client_dir.
-
-  Args:
-    manifest: Manifest object to save.
-    client_dir: Client directory to save the manifest in.
-  """
-  if not client_dir:
-    manifest_file = manifest.manifestFile
-  else:
-    manifest_file = os.path.join(client_dir, '.manifest')
-  with open(manifest_file, 'w') as f:
-    manifest.Save(f, groups=manifest.GetGroupsStr())
-  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
-  # Give the GITC filesystem time to register the manifest changes.
-  time.sleep(3)

hooks.py

@@ -12,11 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import errno
-import json
 import os
 import re
-import subprocess
 import sys
 import traceback
 import urllib.parse
@@ -25,485 +22,507 @@ from error import HookError
 from git_refs import HEAD
 
 
-class RepoHook(object):
+# The API we've documented to hook authors.  Keep in sync with repo-hooks.md.
-  """A RepoHook contains information about a script to run as a hook.
+_API_ARGS = {
+    "pre-upload": {"project_list", "worktree_list"},
+    "post-sync": {"repo_topdir", "sync_duration_seconds"},
+}
 
-  Hooks are used to run a python script before running an upload (for instance,
-  to run presubmit checks).  Eventually, we may have hooks for other actions.
 
-  This shouldn't be confused with files in the 'repo/hooks' directory.  Those
+class RepoHook:
-  files are copied into each '.git/hooks' folder for each project.  Repo-level
+    """A RepoHook contains information about a script to run as a hook.
-  hooks are associated instead with repo actions.
 
-  Hooks are always python.  When a hook is run, we will load the hook into the
+    Hooks are used to run a python script before running an upload (for
-  interpreter and execute its main() function.
+    instance, to run presubmit checks).  Eventually, we may have hooks for other
+    actions.
 
-  Combinations of hook option flags:
+    This shouldn't be confused with files in the 'repo/hooks' directory.  Those
-  - no-verify=False, verify=False (DEFAULT):
+    files are copied into each '.git/hooks' folder for each project.  Repo-level
-    If stdout is a tty, can prompt about running hooks if needed.
+    hooks are associated instead with repo actions.
-    If user denies running hooks, the action is cancelled. If stdout is
-    not a tty and we would need to prompt about hooks, action is
-    cancelled.
-  - no-verify=False, verify=True:
-    Always run hooks with no prompt.
-  - no-verify=True, verify=False:
-    Never run hooks, but run action anyway (AKA bypass hooks).
-  - no-verify=True, verify=True:
-    Invalid
-  """
 
-  def __init__(self,
+    Hooks are always python.  When a hook is run, we will load the hook into the
-               hook_type,
+    interpreter and execute its main() function.
-               hooks_project,
-               repo_topdir,
-               manifest_url,
-               bypass_hooks=False,
-               allow_all_hooks=False,
-               ignore_hooks=False,
-               abort_if_user_denies=False):
-    """RepoHook constructor.
 
-    Params:
+    Combinations of hook option flags:
-      hook_type: A string representing the type of hook.  This is also used
+    - no-verify=False, verify=False (DEFAULT):
-          to figure out the name of the file containing the hook.  For
+      If stdout is a tty, can prompt about running hooks if needed.
-          example: 'pre-upload'.
+      If user denies running hooks, the action is cancelled. If stdout is
-      hooks_project: The project containing the repo hooks.
+      not a tty and we would need to prompt about hooks, action is
-          If you have a manifest, this is manifest.repo_hooks_project.
+      cancelled.
-          OK if this is None, which will make the hook a no-op.
+    - no-verify=False, verify=True:
-      repo_topdir: The top directory of the repo client checkout.
+      Always run hooks with no prompt.
-          This is the one containing the .repo directory. Scripts will
+    - no-verify=True, verify=False:
-          run with CWD as this directory.
+      Never run hooks, but run action anyway (AKA bypass hooks).
-          If you have a manifest, this is manifest.topdir.
+    - no-verify=True, verify=True:
-      manifest_url: The URL to the manifest git repo.
+      Invalid
-      bypass_hooks: If True, then 'Do not run the hook'.
-      allow_all_hooks: If True, then 'Run the hook without prompting'.
-      ignore_hooks: If True, then 'Do not abort action if hooks fail'.
-      abort_if_user_denies: If True, we'll abort running the hook if the user
-          doesn't allow us to run the hook.
     """
-    self._hook_type = hook_type
-    self._hooks_project = hooks_project
-    self._repo_topdir = repo_topdir
-    self._manifest_url = manifest_url
-    self._bypass_hooks = bypass_hooks
-    self._allow_all_hooks = allow_all_hooks
-    self._ignore_hooks = ignore_hooks
-    self._abort_if_user_denies = abort_if_user_denies
 
-    # Store the full path to the script for convenience.
+    def __init__(
-    if self._hooks_project:
+        self,
-      self._script_fullpath = os.path.join(self._hooks_project.worktree,
+        hook_type,
-                                           self._hook_type + '.py')
+        hooks_project,
-    else:
+        repo_topdir,
-      self._script_fullpath = None
+        manifest_url,
+        bug_url=None,
+        bypass_hooks=False,
+        allow_all_hooks=False,
+        ignore_hooks=False,
+        abort_if_user_denies=False,
+    ):
+        """RepoHook constructor.
 
-  def _GetHash(self):
+        Params:
-    """Return a hash of the contents of the hooks directory.
+            hook_type: A string representing the type of hook. This is also used
+                to figure out the name of the file containing the hook. For
+                example: 'pre-upload'.
+            hooks_project: The project containing the repo hooks.
+                If you have a manifest, this is manifest.repo_hooks_project.
+                OK if this is None, which will make the hook a no-op.
+            repo_topdir: The top directory of the repo client checkout.
+                This is the one containing the .repo directory. Scripts will
+                run with CWD as this directory.
+                If you have a manifest, this is manifest.topdir.
+            manifest_url: The URL to the manifest git repo.
+            bug_url: The URL to report issues.
+            bypass_hooks: If True, then 'Do not run the hook'.
+            allow_all_hooks: If True, then 'Run the hook without prompting'.
+            ignore_hooks: If True, then 'Do not abort action if hooks fail'.
+            abort_if_user_denies: If True, we'll abort running the hook if the
+                user doesn't allow us to run the hook.
+        """
+        self._hook_type = hook_type
+        self._hooks_project = hooks_project
+        self._repo_topdir = repo_topdir
+        self._manifest_url = manifest_url
+        self._bug_url = bug_url
+        self._bypass_hooks = bypass_hooks
+        self._allow_all_hooks = allow_all_hooks
+        self._ignore_hooks = ignore_hooks
+        self._abort_if_user_denies = abort_if_user_denies
 
-    We'll just use git to do this.  This hash has the property that if anything
+        # Store the full path to the script for convenience.
-    changes in the directory we will return a different has.
+        self._script_fullpath = None
+        if self._hooks_project and self._hooks_project.worktree:
+            self._script_fullpath = os.path.join(
+                self._hooks_project.worktree, self._hook_type + ".py"
+            )
 
-    SECURITY CONSIDERATION:
+    def _GetHash(self):
-      This hash only represents the contents of files in the hook directory, not
+        """Return a hash of the contents of the hooks directory.
-      any other files imported or called by hooks.  Changes to imported files
-      can change the script behavior without affecting the hash.
 
-    Returns:
+        We'll just use git to do this.  This hash has the property that if
-      A string representing the hash.  This will always be ASCII so that it can
+        anything changes in the directory we will return a different has.
-      be printed to the user easily.
-    """
-    assert self._hooks_project, "Must have hooks to calculate their hash."
 
-    # We will use the work_git object rather than just calling GetRevisionId().
+        SECURITY CONSIDERATION:
-    # That gives us a hash of the latest checked in version of the files that
+            This hash only represents the contents of files in the hook
-    # the user will actually be executing.  Specifically, GetRevisionId()
+            directory, not any other files imported or called by hooks.  Changes
-    # doesn't appear to change even if a user checks out a different version
+            to imported files can change the script behavior without affecting
-    # of the hooks repo (via git checkout) nor if a user commits their own revs.
+            the hash.
-    #
-    # NOTE: Local (non-committed) changes will not be factored into this hash.
-    # I think this is OK, since we're really only worried about warning the user
-    # about upstream changes.
-    return self._hooks_project.work_git.rev_parse(HEAD)
 
-  def _GetMustVerb(self):
+        Returns:
-    """Return 'must' if the hook is required; 'should' if not."""
+            A string representing the hash.  This will always be ASCII so that
-    if self._abort_if_user_denies:
+            it can be printed to the user easily.
-      return 'must'
+        """
-    else:
+        assert self._hooks_project, "Must have hooks to calculate their hash."
-      return 'should'
 
-  def _CheckForHookApproval(self):
+        # We will use the work_git object rather than just calling
-    """Check to see whether this hook has been approved.
+        # GetRevisionId(). That gives us a hash of the latest checked in version
+        # of the files that the user will actually be executing.  Specifically,
+        # GetRevisionId() doesn't appear to change even if a user checks out a
+        # different version of the hooks repo (via git checkout) nor if a user
+        # commits their own revs.
+        #
+        # NOTE: Local (non-committed) changes will not be factored into this
+        # hash. I think this is OK, since we're really only worried about
+        # warning the user about upstream changes.
+        return self._hooks_project.work_git.rev_parse(HEAD)
 
-    We'll accept approval of manifest URLs if they're using secure transports.
+    def _GetMustVerb(self):
-    This way the user can say they trust the manifest hoster.  For insecure
+        """Return 'must' if the hook is required; 'should' if not."""
-    hosts, we fall back to checking the hash of the hooks repo.
+        if self._abort_if_user_denies:
+            return "must"
+        else:
+            return "should"
 
-    Note that we ask permission for each individual hook even though we use
+    def _CheckForHookApproval(self):
-    the hash of all hooks when detecting changes.  We'd like the user to be
+        """Check to see whether this hook has been approved.
-    able to approve / deny each hook individually.  We only use the hash of all
-    hooks because there is no other easy way to detect changes to local imports.
 
-    Returns:
+        We'll accept approval of manifest URLs if they're using secure
-      True if this hook is approved to run; False otherwise.
+        transports. This way the user can say they trust the manifest hoster.
+        For insecure hosts, we fall back to checking the hash of the hooks repo.
 
-    Raises:
+        Note that we ask permission for each individual hook even though we use
-      HookError: Raised if the user doesn't approve and abort_if_user_denies
+        the hash of all hooks when detecting changes.  We'd like the user to be
-          was passed to the consturctor.
+        able to approve / deny each hook individually.  We only use the hash of
-    """
+        all hooks because there is no other easy way to detect changes to local
-    if self._ManifestUrlHasSecureScheme():
+        imports.
-      return self._CheckForHookApprovalManifest()
-    else:
-      return self._CheckForHookApprovalHash()
 
-  def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt,
+        Returns:
-                                  changed_prompt):
+            True if this hook is approved to run; False otherwise.
-    """Check for approval for a particular attribute and hook.
 
-    Args:
+        Raises:
-      subkey: The git config key under [repo.hooks.<hook_type>] to store the
+            HookError: Raised if the user doesn't approve and
-          last approved string.
+                abort_if_user_denies was passed to the consturctor.
-      new_val: The new value to compare against the last approved one.
+        """
-      main_prompt: Message to display to the user to ask for approval.
+        if self._ManifestUrlHasSecureScheme():
-      changed_prompt: Message explaining why we're re-asking for approval.
+            return self._CheckForHookApprovalManifest()
+        else:
+            return self._CheckForHookApprovalHash()
 
-    Returns:
+    def _CheckForHookApprovalHelper(
-      True if this hook is approved to run; False otherwise.
+        self, subkey, new_val, main_prompt, changed_prompt
+    ):
+        """Check for approval for a particular attribute and hook.
 
-    Raises:
+        Args:
-      HookError: Raised if the user doesn't approve and abort_if_user_denies
+            subkey: The git config key under [repo.hooks.<hook_type>] to store
-          was passed to the consturctor.
+                the last approved string.
-    """
+            new_val: The new value to compare against the last approved one.
-    hooks_config = self._hooks_project.config
+            main_prompt: Message to display to the user to ask for approval.
-    git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey)
+            changed_prompt: Message explaining why we're re-asking for approval.
 
-    # Get the last value that the user approved for this hook; may be None.
+        Returns:
-    old_val = hooks_config.GetString(git_approval_key)
+            True if this hook is approved to run; False otherwise.
 
-    if old_val is not None:
+        Raises:
-      # User previously approved hook and asked not to be prompted again.
+            HookError: Raised if the user doesn't approve and
-      if new_val == old_val:
+                abort_if_user_denies was passed to the consturctor.
-        # Approval matched.  We're done.
+        """
-        return True
+        hooks_config = self._hooks_project.config
-      else:
+        git_approval_key = f"repo.hooks.{self._hook_type}.{subkey}"
-        # Give the user a reason why we're prompting, since they last told
-        # us to "never ask again".
-        prompt = 'WARNING: %s\n\n' % (changed_prompt,)
-    else:
-      prompt = ''
 
-    # Prompt the user if we're not on a tty; on a tty we'll assume "no".
+        # Get the last value that the user approved for this hook; may be None.
-    if sys.stdout.isatty():
+        old_val = hooks_config.GetString(git_approval_key)
-      prompt += main_prompt + ' (yes/always/NO)? '
-      response = input(prompt).lower()
-      print()
 
-      # User is doing a one-time approval.
+        if old_val is not None:
-      if response in ('y', 'yes'):
+            # User previously approved hook and asked not to be prompted again.
-        return True
+            if new_val == old_val:
-      elif response == 'always':
+                # Approval matched.  We're done.
-        hooks_config.SetString(git_approval_key, new_val)
+                return True
-        return True
+            else:
+                # Give the user a reason why we're prompting, since they last
+                # told us to "never ask again".
+                prompt = f"WARNING: {changed_prompt}\n\n"
+        else:
+            prompt = ""
 
-    # For anything else, we'll assume no approval.
+        # Prompt the user if we're not on a tty; on a tty we'll assume "no".
-    if self._abort_if_user_denies:
+        if sys.stdout.isatty():
-      raise HookError('You must allow the %s hook or use --no-verify.' %
+            prompt += main_prompt + " (yes/always/NO)? "
-                      self._hook_type)
+            response = input(prompt).lower()
+            print()
 
-    return False
+            # User is doing a one-time approval.
+            if response in ("y", "yes"):
+                return True
+            elif response == "always":
+                hooks_config.SetString(git_approval_key, new_val)
+                return True
 
-  def _ManifestUrlHasSecureScheme(self):
+        # For anything else, we'll assume no approval.
-    """Check if the URI for the manifest is a secure transport."""
+        if self._abort_if_user_denies:
-    secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc')
+            raise HookError(
-    parse_results = urllib.parse.urlparse(self._manifest_url)
+                "You must allow the %s hook or use --no-verify."
-    return parse_results.scheme in secure_schemes
+                % self._hook_type
+            )
 
-  def _CheckForHookApprovalManifest(self):
+        return False
-    """Check whether the user has approved this manifest host.
 
-    Returns:
+    def _ManifestUrlHasSecureScheme(self):
-      True if this hook is approved to run; False otherwise.
+        """Check if the URI for the manifest is a secure transport."""
-    """
+        secure_schemes = (
-    return self._CheckForHookApprovalHelper(
+            "file",
-        'approvedmanifest',
+            "https",
-        self._manifest_url,
+            "ssh",
-        'Run hook scripts from %s' % (self._manifest_url,),
+            "persistent-https",
-        'Manifest URL has changed since %s was allowed.' % (self._hook_type,))
+            "sso",
+            "rpc",
+        )
+        parse_results = urllib.parse.urlparse(self._manifest_url)
+        return parse_results.scheme in secure_schemes
 
-  def _CheckForHookApprovalHash(self):
+    def _CheckForHookApprovalManifest(self):
-    """Check whether the user has approved the hooks repo.
+        """Check whether the user has approved this manifest host.
 
-    Returns:
+        Returns:
-      True if this hook is approved to run; False otherwise.
+            True if this hook is approved to run; False otherwise.
-    """
+        """
-    prompt = ('Repo %s run the script:\n'
+        return self._CheckForHookApprovalHelper(
-              '  %s\n'
+            "approvedmanifest",
-              '\n'
+            self._manifest_url,
-              'Do you want to allow this script to run')
+            f"Run hook scripts from {self._manifest_url}",
-    return self._CheckForHookApprovalHelper(
+            f"Manifest URL has changed since {self._hook_type} was allowed.",
-        'approvedhash',
+        )
-        self._GetHash(),
-        prompt % (self._GetMustVerb(), self._script_fullpath),
-        'Scripts have changed since %s was allowed.' % (self._hook_type,))
 
-  @staticmethod
+    def _CheckForHookApprovalHash(self):
-  def _ExtractInterpFromShebang(data):
+        """Check whether the user has approved the hooks repo.
-    """Extract the interpreter used in the shebang.
 
-    Try to locate the interpreter the script is using (ignoring `env`).
+        Returns:
+            True if this hook is approved to run; False otherwise.
+        """
+        prompt = (
+            "Repo %s run the script:\n"
+            "  %s\n"
+            "\n"
+            "Do you want to allow this script to run"
+        )
+        return self._CheckForHookApprovalHelper(
+            "approvedhash",
+            self._GetHash(),
+            prompt % (self._GetMustVerb(), self._script_fullpath),
+            f"Scripts have changed since {self._hook_type} was allowed.",
+        )
 
-    Args:
+    @staticmethod
-      data: The file content of the script.
+    def _ExtractInterpFromShebang(data):
+        """Extract the interpreter used in the shebang.
 
-    Returns:
+        Try to locate the interpreter the script is using (ignoring `env`).
-      The basename of the main script interpreter, or None if a shebang is not
-      used or could not be parsed out.
-    """
-    firstline = data.splitlines()[:1]
-    if not firstline:
-      return None
 
-    # The format here can be tricky.
+        Args:
-    shebang = firstline[0].strip()
+            data: The file content of the script.
-    m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang)
-    if not m:
-      return None
 
-    # If the using `env`, find the target program.
+        Returns:
-    interp = m.group(1)
+            The basename of the main script interpreter, or None if a shebang is
-    if os.path.basename(interp) == 'env':
+            not used or could not be parsed out.
-      interp = m.group(2)
+        """
+        firstline = data.splitlines()[:1]
+        if not firstline:
+            return None
 
-    return interp
+        # The format here can be tricky.
+        shebang = firstline[0].strip()
+        m = re.match(r"^#!\s*([^\s]+)(?:\s+([^\s]+))?", shebang)
+        if not m:
+            return None
 
-  def _ExecuteHookViaReexec(self, interp, context, **kwargs):
+        # If the using `env`, find the target program.
-    """Execute the hook script through |interp|.
+        interp = m.group(1)
+        if os.path.basename(interp) == "env":
+            interp = m.group(2)
 
-    Note: Support for this feature should be dropped ~Jun 2021.
+        return interp
 
-    Args:
+    def _ExecuteHookViaImport(self, data, context, **kwargs):
-      interp: The Python program to run.
+        """Execute the hook code in |data| directly.
-      context: Basic Python context to execute the hook inside.
-      kwargs: Arbitrary arguments to pass to the hook script.
 
-    Raises:
+        Args:
-      HookError: When the hooks failed for any reason.
+            data: The code of the hook to execute.
-    """
+            context: Basic Python context to execute the hook inside.
-    # This logic needs to be kept in sync with _ExecuteHookViaImport below.
+            kwargs: Arbitrary arguments to pass to the hook script.
-    script = """
-import json, os, sys
-path = '''%(path)s'''
-kwargs = json.loads('''%(kwargs)s''')
-context = json.loads('''%(context)s''')
-sys.path.insert(0, os.path.dirname(path))
-data = open(path).read()
-exec(compile(data, path, 'exec'), context)
-context['main'](**kwargs)
-""" % {
-        'path': self._script_fullpath,
-        'kwargs': json.dumps(kwargs),
-        'context': json.dumps(context),
-    }
 
-    # We pass the script via stdin to avoid OS argv limits.  It also makes
+        Raises:
-    # unhandled exception tracebacks less verbose/confusing for users.
+            HookError: When the hooks failed for any reason.
-    cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())']
+        """
-    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+        # Exec, storing global context in the context dict.  We catch exceptions
-    proc.communicate(input=script.encode('utf-8'))
+        # and convert to a HookError w/ just the failing traceback.
-    if proc.returncode:
-      raise HookError('Failed to run %s hook.' % (self._hook_type,))
-
-  def _ExecuteHookViaImport(self, data, context, **kwargs):
-    """Execute the hook code in |data| directly.
-
-    Args:
-      data: The code of the hook to execute.
-      context: Basic Python context to execute the hook inside.
-      kwargs: Arbitrary arguments to pass to the hook script.
-
-    Raises:
-      HookError: When the hooks failed for any reason.
-    """
-    # Exec, storing global context in the context dict.  We catch exceptions
-    # and convert to a HookError w/ just the failing traceback.
-    try:
-      exec(compile(data, self._script_fullpath, 'exec'), context)
-    except Exception:
-      raise HookError('%s\nFailed to import %s hook; see traceback above.' %
-                      (traceback.format_exc(), self._hook_type))
-
-    # Running the script should have defined a main() function.
-    if 'main' not in context:
-      raise HookError('Missing main() in: "%s"' % self._script_fullpath)
-
-    # Call the main function in the hook.  If the hook should cause the
-    # build to fail, it will raise an Exception.  We'll catch that convert
-    # to a HookError w/ just the failing traceback.
-    try:
-      context['main'](**kwargs)
-    except Exception:
-      raise HookError('%s\nFailed to run main() for %s hook; see traceback '
-                      'above.' % (traceback.format_exc(), self._hook_type))
-
-  def _ExecuteHook(self, **kwargs):
-    """Actually execute the given hook.
-
-    This will run the hook's 'main' function in our python interpreter.
-
-    Args:
-      kwargs: Keyword arguments to pass to the hook.  These are often specific
-          to the hook type.  For instance, pre-upload hooks will contain
-          a project_list.
-    """
-    # Keep sys.path and CWD stashed away so that we can always restore them
-    # upon function exit.
-    orig_path = os.getcwd()
-    orig_syspath = sys.path
-
-    try:
-      # Always run hooks with CWD as topdir.
-      os.chdir(self._repo_topdir)
-
-      # Put the hook dir as the first item of sys.path so hooks can do
-      # relative imports.  We want to replace the repo dir as [0] so
-      # hooks can't import repo files.
-      sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
-
-      # Initial global context for the hook to run within.
-      context = {'__file__': self._script_fullpath}
-
-      # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
-      # We don't actually want hooks to define their main with this argument--
-      # it's there to remind them that their hook should always take **kwargs.
-      # For instance, a pre-upload hook should be defined like:
-      #   def main(project_list, **kwargs):
-      #
-      # This allows us to later expand the API without breaking old hooks.
-      kwargs = kwargs.copy()
-      kwargs['hook_should_take_kwargs'] = True
-
-      # See what version of python the hook has been written against.
-      data = open(self._script_fullpath).read()
-      interp = self._ExtractInterpFromShebang(data)
-      reexec = False
-      if interp:
-        prog = os.path.basename(interp)
-        if prog.startswith('python2') and sys.version_info.major != 2:
-          reexec = True
-        elif prog.startswith('python3') and sys.version_info.major == 2:
-          reexec = True
-
-      # Attempt to execute the hooks through the requested version of Python.
-      if reexec:
         try:
-          self._ExecuteHookViaReexec(interp, context, **kwargs)
+            exec(compile(data, self._script_fullpath, "exec"), context)
-        except OSError as e:
+        except Exception:
-          if e.errno == errno.ENOENT:
+            raise HookError(
-            # We couldn't find the interpreter, so fallback to importing.
+                "%s\nFailed to import %s hook; see traceback above."
-            reexec = False
+                % (traceback.format_exc(), self._hook_type)
-          else:
+            )
-            raise
 
-      # Run the hook by importing directly.
+        # Running the script should have defined a main() function.
-      if not reexec:
+        if "main" not in context:
-        self._ExecuteHookViaImport(data, context, **kwargs)
+            raise HookError('Missing main() in: "%s"' % self._script_fullpath)
-    finally:
-      # Restore sys.path and CWD.
-      sys.path = orig_syspath
-      os.chdir(orig_path)
 
-  def _CheckHook(self):
+        # Call the main function in the hook.  If the hook should cause the
-    # Bail with a nice error if we can't find the hook.
+        # build to fail, it will raise an Exception.  We'll catch that convert
-    if not os.path.isfile(self._script_fullpath):
+        # to a HookError w/ just the failing traceback.
-      raise HookError('Couldn\'t find repo hook: %s' % self._script_fullpath)
+        try:
+            context["main"](**kwargs)
+        except Exception:
+            raise HookError(
+                "%s\nFailed to run main() for %s hook; see traceback "
+                "above." % (traceback.format_exc(), self._hook_type)
+            )
 
-  def Run(self, **kwargs):
+    def _ExecuteHook(self, **kwargs):
-    """Run the hook.
+        """Actually execute the given hook.
 
-    If the hook doesn't exist (because there is no hooks project or because
+        This will run the hook's 'main' function in our python interpreter.
-    this particular hook is not enabled), this is a no-op.
 
-    Args:
+        Args:
-      user_allows_all_hooks: If True, we will never prompt about running the
+            kwargs: Keyword arguments to pass to the hook.  These are often
-          hook--we'll just assume it's OK to run it.
+                specific to the hook type.  For instance, pre-upload hooks will
-      kwargs: Keyword arguments to pass to the hook.  These are often specific
+                contain a project_list.
-          to the hook type.  For instance, pre-upload hooks will contain
+        """
-          a project_list.
+        # Keep sys.path and CWD stashed away so that we can always restore them
+        # upon function exit.
+        orig_path = os.getcwd()
+        orig_syspath = sys.path
 
-    Returns:
+        try:
-      True: On success or ignore hooks by user-request
+            # Always run hooks with CWD as topdir.
-      False: The hook failed. The caller should respond with aborting the action.
+            os.chdir(self._repo_topdir)
-        Some examples in which False is returned:
-        * Finding the hook failed while it was enabled, or
-        * the user declined to run a required hook (from _CheckForHookApproval)
-        In all these cases the user did not pass the proper arguments to
-        ignore the result through the option combinations as listed in
-        AddHookOptionGroup().
-    """
-    # Do not do anything in case bypass_hooks is set, or
-    # no-op if there is no hooks project or if hook is disabled.
-    if (self._bypass_hooks or
-        not self._hooks_project or
-        self._hook_type not in self._hooks_project.enabled_repo_hooks):
-      return True
 
-    passed = True
+            # Put the hook dir as the first item of sys.path so hooks can do
-    try:
+            # relative imports.  We want to replace the repo dir as [0] so
-      self._CheckHook()
+            # hooks can't import repo files.
+            sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
 
-      # Make sure the user is OK with running the hook.
+            # Initial global context for the hook to run within.
-      if self._allow_all_hooks or self._CheckForHookApproval():
+            context = {"__file__": self._script_fullpath}
-        # Run the hook with the same version of python we're using.
-        self._ExecuteHook(**kwargs)
-    except SystemExit as e:
-      passed = False
-      print('ERROR: %s hooks exited with exit code: %s' % (self._hook_type, str(e)),
-            file=sys.stderr)
-    except HookError as e:
-      passed = False
-      print('ERROR: %s' % str(e), file=sys.stderr)
 
-    if not passed and self._ignore_hooks:
+            # Add 'hook_should_take_kwargs' to the arguments to be passed to
-      print('\nWARNING: %s hooks failed, but continuing anyways.' % self._hook_type,
+            # main. We don't actually want hooks to define their main with this
-            file=sys.stderr)
+            # argument--it's there to remind them that their hook should always
-      passed = True
+            # take **kwargs.
+            # For instance, a pre-upload hook should be defined like:
+            #   def main(project_list, **kwargs):
+            #
+            # This allows us to later expand the API without breaking old hooks.
+            kwargs = kwargs.copy()
+            kwargs["hook_should_take_kwargs"] = True
 
-    return passed
+            # See what version of python the hook has been written against.
+            data = open(self._script_fullpath).read()
+            interp = self._ExtractInterpFromShebang(data)
+            if interp:
+                prog = os.path.basename(interp)
+                if prog.startswith("python2"):
+                    raise HookError("Python 2 is not supported")
 
-  @classmethod
+            # Run the hook by importing directly.
-  def FromSubcmd(cls, manifest, opt, *args, **kwargs):
+            self._ExecuteHookViaImport(data, context, **kwargs)
-    """Method to construct the repo hook class
+        finally:
+            # Restore sys.path and CWD.
+            sys.path = orig_syspath
+            os.chdir(orig_path)
 
-    Args:
+    def _CheckHook(self):
-      manifest: The current active manifest for this command from which we
+        # Bail with a nice error if we can't find the hook.
-          extract a couple of fields.
+        if not os.path.isfile(self._script_fullpath):
-      opt: Contains the commandline options for the action of this hook.
+            raise HookError(
-          It should contain the options added by AddHookOptionGroup() in which
+                "Couldn't find repo hook: %s" % self._script_fullpath
-          we are interested in RepoHook execution.
+            )
-    """
-    for key in ('bypass_hooks', 'allow_all_hooks', 'ignore_hooks'):
-      kwargs.setdefault(key, getattr(opt, key))
-    kwargs.update({
-        'hooks_project': manifest.repo_hooks_project,
-        'repo_topdir': manifest.topdir,
-        'manifest_url': manifest.manifestProject.GetRemote('origin').url,
-    })
-    return cls(*args, **kwargs)
 
-  @staticmethod
+    def Run(self, **kwargs):
-  def AddOptionGroup(parser, name):
+        """Run the hook.
-    """Help options relating to the various hooks."""
 
-    # Note that verify and no-verify are NOT opposites of each other, which
+        If the hook doesn't exist (because there is no hooks project or because
-    # is why they store to different locations. We are using them to match
+        this particular hook is not enabled), this is a no-op.
-    # 'git commit' syntax.
+
-    group = parser.add_option_group(name + ' hooks')
+        Args:
-    group.add_option('--no-verify',
+            user_allows_all_hooks: If True, we will never prompt about running
-                     dest='bypass_hooks', action='store_true',
+                the hook--we'll just assume it's OK to run it.
-                     help='Do not run the %s hook.' % name)
+            kwargs: Keyword arguments to pass to the hook.  These are often
-    group.add_option('--verify',
+                specific to the hook type.  For instance, pre-upload hooks will
-                     dest='allow_all_hooks', action='store_true',
+                contain a project_list.
-                     help='Run the %s hook without prompting.' % name)
+
-    group.add_option('--ignore-hooks',
+        Returns:
-                     action='store_true',
+            True: On success or ignore hooks by user-request
-                     help='Do not abort if %s hooks fail.' % name)
+            False: The hook failed. The caller should respond with aborting the
+                action. Some examples in which False is returned:
+                * Finding the hook failed while it was enabled, or
+                * the user declined to run a required hook (from
+                    _CheckForHookApproval)
+                In all these cases the user did not pass the proper arguments to
+                ignore the result through the option combinations as listed in
+                AddHookOptionGroup().
+        """
+        # Make sure our own callers use the documented API.
+        exp_kwargs = _API_ARGS.get(self._hook_type, set())
+        got_kwargs = set(kwargs.keys())
+        if exp_kwargs != got_kwargs:
+            print(
+                "repo internal error: "
+                f"hook '{self._hook_type}' called incorrectly\n"
+                f"  got:      {sorted(got_kwargs)}\n"
+                f"  expected: {sorted(exp_kwargs)}\n"
+                f"Please file a bug: {self._bug_url}",
+                file=sys.stderr,
+            )
+            return False
+
+        # Do not do anything in case bypass_hooks is set, or
+        # no-op if there is no hooks project or if hook is disabled.
+        if (
+            self._bypass_hooks
+            or not self._hooks_project
+            or not self._script_fullpath
+            or self._hook_type not in self._hooks_project.enabled_repo_hooks
+        ):
+            return True
+
+        passed = True
+        try:
+            self._CheckHook()
+
+            # Make sure the user is OK with running the hook.
+            if self._allow_all_hooks or self._CheckForHookApproval():
+                # Run the hook with the same version of python we're using.
+                self._ExecuteHook(**kwargs)
+        except SystemExit as e:
+            passed = False
+            print(
+                "ERROR: %s hooks exited with exit code: %s"
+                % (self._hook_type, str(e)),
+                file=sys.stderr,
+            )
+        except HookError as e:
+            passed = False
+            print("ERROR: %s" % str(e), file=sys.stderr)
+
+        if not passed and self._ignore_hooks:
+            print(
+                "\nWARNING: %s hooks failed, but continuing anyways."
+                % self._hook_type,
+                file=sys.stderr,
+            )
+            passed = True
+
+        return passed
+
+    @classmethod
+    def FromSubcmd(cls, manifest, opt, *args, **kwargs):
+        """Method to construct the repo hook class
+
+        Args:
+            manifest: The current active manifest for this command from which we
+                extract a couple of fields.
+            opt: Contains the commandline options for the action of this hook.
+                It should contain the options added by AddHookOptionGroup() in
+                which we are interested in RepoHook execution.
+        """
+        for key in ("bypass_hooks", "allow_all_hooks", "ignore_hooks"):
+            kwargs.setdefault(key, getattr(opt, key))
+        kwargs.update(
+            {
+                "hooks_project": manifest.repo_hooks_project,
+                "repo_topdir": manifest.topdir,
+                "manifest_url": manifest.manifestProject.GetRemote(
+                    "origin"
+                ).url,
+                "bug_url": manifest.contactinfo.bugurl,
+            }
+        )
+        return cls(*args, **kwargs)
+
+    @staticmethod
+    def AddOptionGroup(parser, name):
+        """Help options relating to the various hooks."""
+
+        # Note that verify and no-verify are NOT opposites of each other, which
+        # is why they store to different locations. We are using them to match
+        # 'git commit' syntax.
+        group = parser.add_option_group(name + " hooks")
+        group.add_option(
+            "--no-verify",
+            dest="bypass_hooks",
+            action="store_true",
+            help="Do not run the %s hook." % name,
+        )
+        group.add_option(
+            "--verify",
+            dest="allow_all_hooks",
+            action="store_true",
+            help="Run the %s hook without prompting." % name,
+        )
+        group.add_option(
+            "--ignore-hooks",
+            action="store_true",
+            help="Do not abort if %s hooks fail." % name,
+        )

hooks/commit-msg

@@ -1,5 +1,8 @@
 #!/bin/sh
-# From Gerrit Code Review 3.1.3
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://gerrit.googlesource.com/gerrit/
+# This is synced from commit: 62f5bbea67f6dafa6e22a601a0c298214c510caf
+# DO NOT EDIT THIS FILE
 #
 # Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
@@ -17,6 +20,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -u
+
 # avoid [[ which is not POSIX sh.
 if test "$#" != 1 ; then
   echo "$0 requires an argument."
@@ -29,17 +34,33 @@ if test ! -f "$1" ; then
 fi
 
 # Do not create a change id if requested
-if test "false" = "`git config --bool --get gerrit.createChangeId`" ; then
+case "$(git config --get gerrit.createChangeId)" in
-  exit 0
+  false)
+    exit 0
+    ;;
+  always)
+    ;;
+  *)
+    # Do not create a change id for squash/fixup commits.
+    if head -n1 "$1" | LC_ALL=C grep -q '^[a-z][a-z]*! '; then
+      exit 0
+    fi
+    ;;
+esac
+
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+  refhash="$(git rev-parse HEAD)"
+else
+  refhash="$(git hash-object -t tree /dev/null)"
 fi
 
-# $RANDOM will be undefined if not using bash, so don't use set -u
+random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin)
-random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin)
 dest="$1.tmp.${random}"
 
-trap 'rm -f "${dest}"' EXIT
+trap 'rm -f "$dest" "$dest-2"' EXIT
 
-if ! git stripspace --strip-comments < "$1" > "${dest}" ; then
+if ! cat "$1" | sed -e '/>8/q' | git stripspace --strip-comments > "${dest}" ; then
    echo "cannot strip comments from $1"
    exit 1
 fi
@@ -49,11 +70,40 @@ if test ! -s "${dest}" ; then
   exit 1
 fi
 
+reviewurl="$(git config --get gerrit.reviewUrl)"
+if test -n "${reviewurl}" ; then
+  token="Link"
+  value="${reviewurl%/}/id/I$random"
+  pattern=".*/id/I[0-9a-f]\{40\}"
+else
+  token="Change-Id"
+  value="I$random"
+  pattern=".*"
+fi
+
+if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then
+  exit 0
+fi
+
+# There must be a Signed-off-by trailer for the code below to work. Insert a
+# sentinel at the end to make sure there is one.
 # Avoid the --in-place option which only appeared in Git 2.8
-# Avoid the --if-exists option which only appeared in Git 2.15
+if ! git interpret-trailers \
-if ! git -c trailer.ifexists=doNothing interpret-trailers \
+         --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then
-      --trailer "Change-Id: I${random}" < "$1" > "${dest}" ; then
+  echo "cannot insert Signed-off-by sentinel line in $1"
-  echo "cannot insert change-id line in $1"
+  exit 1
+fi
+
+# Make sure the trailer appears before any Signed-off-by trailers by inserting
+# it as if it was a Signed-off-by trailer and then use sed to remove the
+# Signed-off-by prefix and the Signed-off-by sentinel line.
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --where option which only appeared in Git 2.15
+if ! git -c trailer.where=before interpret-trailers \
+         --trailer "Signed-off-by: $token: $value" < "$dest-2" |
+     sed -e "s/^Signed-off-by: \($token: \)/\1/" \
+         -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
+  echo "cannot insert $token line in $1"
   exit 1
 fi
 

hooks/pre-auto-gc

@@ -1,33 +1,25 @@
 #!/bin/sh
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://github.com/git/git
+# This is synced from commit: 00e10ef10e161a913893b8cb33aa080d4ca5baa6
+# DO NOT EDIT THIS FILE
 #
 # An example hook script to verify if you are on battery, in case you
-# are running Windows, Linux or OS X. Called by git-gc --auto with no
+# are running Linux or OS X. Called by git-gc --auto with no arguments.
-# arguments. The hook should exit with non-zero status after issuing an
+# The hook should exit with non-zero status after issuing an appropriate
-# appropriate message if it wants to stop the auto repacking.
+# message if it wants to stop the auto repacking.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# This hook is stored in the contrib/hooks directory. Your distribution
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# may have put this somewhere else. If you want to use this hook, you
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# should make this script executable then link to it in the repository
-# GNU General Public License for more details.
+# you would like to use it in.
 #
-# You should have received a copy of the GNU General Public License
+# For example, if the hook is stored in
-# along with this program; if not, write to the Free Software
+# /usr/share/git-core/contrib/hooks/pre-auto-gc-battery:
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
-
+# cd /path/to/your/repository.git
-if uname -s | grep -q "_NT-"
+# ln -sf /usr/share/git-core/contrib/hooks/pre-auto-gc-battery \
-then
+#	hooks/pre-auto-gc
-	if test -x $SYSTEMROOT/System32/Wbem/wmic
-	then
-		STATUS=$(wmic path win32_battery get batterystatus /format:list | tr -d '\r\n')
-		[ "$STATUS" = "BatteryStatus=2" ] && exit 0 || exit 1
-	fi
-	exit 0
-fi
 
 if test -x /sbin/on_ac_power && (/sbin/on_ac_power;test $? -ne 1)
 then
@@ -48,11 +40,6 @@ elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
 	grep -q "drawing from 'AC Power'"
 then
 	exit 0
-elif test -d /sys/bus/acpi/drivers/battery && test 0 = \
-  "$(find /sys/bus/acpi/drivers/battery/ -type l | wc -l)";
-then
-	# No battery exists.
-	exit 0
 fi
 
 echo "Auto packing deferred; not on AC"

man/repo-forall.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo forall" "Repo Manual"
+.TH REPO "1" "May 2026" "repo forall" "Repo Manual"
 .SH NAME
 repo \- repo forall - manual page for repo forall
 .SH SYNOPSIS
@@ -120,6 +120,11 @@ git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly as written in
 the manifest.
 .PP
+REPO_UPSTREAM is the name of the upstream branch as specified in the manifest.
+.PP
+REPO_DEST_BRANCH is the name of the destination branch for code review, as
+specified in the manifest.
+.PP
 REPO_COUNT is the total number of projects being iterated.
 .PP
 REPO_I is the current (1\-based) iteration count. Can be used in conjunction with

man/repo-gc.1

@@ -1,21 +1,28 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo gitc-delete" "Repo Manual"
+.TH REPO "1" "April 2025" "repo gc" "Repo Manual"
 .SH NAME
-repo \- repo gitc-delete - manual page for repo gitc-delete
+repo \- repo gc - manual page for repo gc
 .SH SYNOPSIS
 .B repo
-\fI\,gitc-delete\/\fR
+\fI\,gc\/\fR
 .SH DESCRIPTION
 Summary
 .PP
-Delete a GITC Client.
+Cleaning up internal repo and Git state.
 .SH OPTIONS
 .TP
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-f\fR, \fB\-\-force\fR
+\fB\-n\fR, \fB\-\-dry\-run\fR
-force the deletion (no prompt)
+do everything except actually delete
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
+.TP
+\fB\-\-repack\fR
+repack all projects that use partial clone with
+filter=blob:none
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -37,8 +44,4 @@ only operate on this (sub)manifest
 \fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
 operate on this manifest and its submanifests
 .PP
-Run `repo help gitc\-delete` to view the detailed manual.
+Run `repo help gc` to view the detailed manual.
-.SH DETAILS
-.PP
-This subcommand deletes the current GITC client, deleting the GITC manifest and
-all locally downloaded sources.

man/repo-gitc-init.1

@@ -1,170 +0,0 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo gitc-init" "Repo Manual"
-.SH NAME
-repo \- repo gitc-init - manual page for repo gitc-init
-.SH SYNOPSIS
-.B repo
-\fI\,gitc-init \/\fR[\fI\,options\/\fR] [\fI\,client name\/\fR]
-.SH DESCRIPTION
-Summary
-.PP
-Initialize a GITC Client.
-.SH OPTIONS
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-show this help message and exit
-.SS Logging options:
-.TP
-\fB\-v\fR, \fB\-\-verbose\fR
-show all output
-.TP
-\fB\-q\fR, \fB\-\-quiet\fR
-only show errors
-.SS Manifest options:
-.TP
-\fB\-u\fR URL, \fB\-\-manifest\-url\fR=\fI\,URL\/\fR
-manifest repository location
-.TP
-\fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
-manifest branch or revision (use HEAD for default)
-.TP
-\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
-initial manifest file
-.TP
-\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
-restrict manifest projects to ones with specified
-group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
-.TP
-\fB\-p\fR PLATFORM, \fB\-\-platform\fR=\fI\,PLATFORM\/\fR
-restrict manifest projects to ones with a specified
-platform group [auto|all|none|linux|darwin|...]
-.TP
-\fB\-\-submodules\fR
-sync any submodules associated with the manifest repo
-.TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.SS Manifest (only) checkout options:
-.TP
-\fB\-\-current\-branch\fR
-fetch only current manifest branch from server
-.TP
-\fB\-\-no\-current\-branch\fR
-fetch all manifest branches from server
-.TP
-\fB\-\-tags\fR
-fetch tags in the manifest
-.TP
-\fB\-\-no\-tags\fR
-don't fetch tags in the manifest
-.SS Checkout modes:
-.TP
-\fB\-\-mirror\fR
-create a replica of the remote repositories rather
-than a client working directory
-.TP
-\fB\-\-archive\fR
-checkout an archive instead of a git repository for
-each project. See git archive.
-.TP
-\fB\-\-worktree\fR
-use git\-worktree to manage projects
-.SS Project checkout optimizations:
-.TP
-\fB\-\-reference\fR=\fI\,DIR\/\fR
-location of mirror directory
-.TP
-\fB\-\-dissociate\fR
-dissociate from reference mirrors after clone
-.TP
-\fB\-\-depth\fR=\fI\,DEPTH\/\fR
-create a shallow clone with given depth; see git clone
-.TP
-\fB\-\-partial\-clone\fR
-perform partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-no\-partial\-clone\fR
-disable use of partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-partial\-clone\-exclude\fR=\fI\,PARTIAL_CLONE_EXCLUDE\/\fR
-exclude the specified projects (a comma\-delimited
-project names) from partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-clone\-filter\fR=\fI\,CLONE_FILTER\/\fR
-filter for use with \fB\-\-partial\-clone\fR [default:
-blob:none]
-.TP
-\fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects;
-implies \fB\-c\fR
-.TP
-\fB\-\-no\-use\-superproject\fR
-disable use of manifest superprojects
-.TP
-\fB\-\-clone\-bundle\fR
-enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
-not \fB\-\-partial\-clone\fR)
-.TP
-\fB\-\-no\-clone\-bundle\fR
-disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
-\fB\-\-partial\-clone\fR)
-.TP
-\fB\-\-git\-lfs\fR
-enable Git LFS support
-.TP
-\fB\-\-no\-git\-lfs\fR
-disable Git LFS support
-.SS repo Version options:
-.TP
-\fB\-\-repo\-url\fR=\fI\,URL\/\fR
-repo repository location ($REPO_URL)
-.TP
-\fB\-\-repo\-rev\fR=\fI\,REV\/\fR
-repo branch or revision ($REPO_REV)
-.TP
-\fB\-\-no\-repo\-verify\fR
-do not verify repo source code
-.SS Other options:
-.TP
-\fB\-\-config\-name\fR
-Always prompt for name/e\-mail
-.SS GITC options:
-.TP
-\fB\-f\fR MANIFEST_FILE, \fB\-\-manifest\-file\fR=\fI\,MANIFEST_FILE\/\fR
-Optional manifest file to use for this GITC client.
-.TP
-\fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
-Name of the gitc_client instance to create or modify.
-.SS Multi\-manifest:
-.TP
-\fB\-\-outer\-manifest\fR
-operate starting at the outermost manifest
-.TP
-\fB\-\-no\-outer\-manifest\fR
-do not operate on outer manifests
-.TP
-\fB\-\-this\-manifest\-only\fR
-only operate on this (sub)manifest
-.TP
-\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
-operate on this manifest and its submanifests
-.PP
-Run `repo help gitc\-init` to view the detailed manual.
-.SH DETAILS
-.PP
-The 'repo gitc\-init' command is ran to initialize a new GITC client for use with
-the GITC file system.
-.PP
-This command will setup the client directory, initialize repo, just like repo
-init does, and then downloads the manifest collection and installs it in the
-\&.repo/directory of the GITC client.
-.PP
-Once this is done, a GITC manifest is generated by pulling the HEAD SHA for each
-project and generates the properly formatted XML file and installs it as
-\&.manifest in the GITC client directory.
-.PP
-The \fB\-c\fR argument is required to specify the GITC client name.
-.PP
-The optional \fB\-f\fR argument can be used to specify the manifest file to use for
-this GITC client.

man/repo-info.1

@@ -1,10 +1,10 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo info" "Repo Manual"
+.TH REPO "1" "May 2026" "repo info" "Repo Manual"
 .SH NAME
 repo \- repo info - manual page for repo info
 .SH SYNOPSIS
 .B repo
-\fI\,info \/\fR[\fI\,-dl\/\fR] [\fI\,-o \/\fR[\fI\,-c\/\fR]] [\fI\,<project>\/\fR...]
+\fI\,info \/\fR[\fI\,-dl\/\fR] [\fI\,-o \/\fR[\fI\,-c\/\fR]] [\fI\,--format=<format>\/\fR] [\fI\,<project>\/\fR...]
 .SH DESCRIPTION
 Summary
 .PP
@@ -14,6 +14,10 @@ Get info on the manifest branch, current branch or unmerged branches
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
 \fB\-d\fR, \fB\-\-diff\fR
 show full info and commit diff including remote
 branches
@@ -21,6 +25,18 @@ branches
 \fB\-o\fR, \fB\-\-overview\fR
 show overview of all local commits
 .TP
+\fB\-\-include\-summary\fR
+include manifest summary (default: true)
+.TP
+\fB\-\-no\-include\-summary\fR
+exclude manifest summary
+.TP
+\fB\-\-include\-projects\fR
+include project details (default: true)
+.TP
+\fB\-\-no\-include\-projects\fR
+exclude project details
+.TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
 consider only checked out branches
 .TP
@@ -29,6 +45,9 @@ consider all local branches
 .TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 disable all remote operations
+.TP
+\fB\-\-format\fR=\fI\,FORMAT\/\fR
+output format: text, json (default: text)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo init" "Repo Manual"
+.TH REPO "1" "September 2024" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -28,6 +28,11 @@ manifest repository location
 \fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
 manifest branch or revision (use HEAD for default)
 .TP
+\fB\-\-manifest\-upstream\-branch\fR=\fI\,BRANCH\/\fR
+when a commit is provided to \fB\-\-manifest\-branch\fR, this
+is the name of the git ref in which the commit can be
+found
+.TP
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
@@ -45,10 +50,15 @@ sync any submodules associated with the manifest repo
 \fB\-\-standalone\-manifest\fR
 download the manifest as a static file rather then
 create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -158,6 +168,10 @@ The optional \fB\-b\fR argument can be used to select the manifest branch to che
 and use. If no branch is specified, the remote's default branch is used. This is
 equivalent to using \fB\-b\fR HEAD.
 .PP
+The optional \fB\-\-manifest\-upstream\-branch\fR argument can be used when a commit is
+provided to \fB\-\-manifest\-branch\fR (or \fB\-b\fR), to specify the name of the git ref in
+which the commit can be found.
+.PP
 The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
 used. If no manifest is specified, the manifest default.xml will be used.
 .PP

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo manifest" "Repo Manual"
+.TH REPO "1" "April 2026" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -30,8 +30,8 @@ if in \fB\-r\fR mode, do not write the dest\-branch field
 (only of use if the branch names for a sha1 manifest
 are sensitive)
 .TP
-\fB\-\-json\fR
+\fB\-\-format\fR=\fI\,FORMAT\/\fR
-output manifest in JSON format (experimental)
+output format: xml, json (default: xml)
 .TP
 \fB\-\-pretty\fR
 format output for humans to read
@@ -78,6 +78,10 @@ set to the ref we were on when the manifest was generated. The 'dest\-branch'
 attribute is set to indicate the remote ref to push changes to via 'repo
 upload'.
 .PP
+Multiple output formats are supported via \fB\-\-format\fR. The default output is XML,
+and formats are generally "condensed". Use \fB\-\-pretty\fR for more human\-readable
+variations.
+.PP
 repo Manifest Format
 .PP
 A repo manifest describes the structure of a repo client; that is the
@@ -127,6 +131,7 @@ include*)>
 <!ATTLIST default dest\-branch CDATA #IMPLIED>
 <!ATTLIST default upstream    CDATA #IMPLIED>
 <!ATTLIST default sync\-j      CDATA #IMPLIED>
+<!ATTLIST default sync\-j\-max  CDATA #IMPLIED>
 <!ATTLIST default sync\-c      CDATA #IMPLIED>
 <!ATTLIST default sync\-s      CDATA #IMPLIED>
 <!ATTLIST default sync\-tags   CDATA #IMPLIED>
@@ -135,7 +140,7 @@ include*)>
 <!ATTLIST manifest\-server url CDATA #REQUIRED>
 .IP
 <!ELEMENT submanifest EMPTY>
-<!ATTLIST submanifest name           ID #REQUIRED>
+<!ATTLIST submanifest name           ID    #REQUIRED>
 <!ATTLIST submanifest remote         IDREF #IMPLIED>
 <!ATTLIST submanifest project        CDATA #IMPLIED>
 <!ATTLIST submanifest manifest\-name  CDATA #IMPLIED>
@@ -160,15 +165,32 @@ IDREF #IMPLIED>
 .TP
 <!ATTLIST project revision
 CDATA #IMPLIED>
+.TP
+<!ATTLIST project dest\-branch
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project groups
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project sync\-c
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project sync\-s
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project sync\-tags
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project upstream
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project clone\-depth
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project force\-path
+CDATA #IMPLIED>
 .IP
-<!ATTLIST project dest\-branch CDATA #IMPLIED>
+<!ATTLIST project sync\-strategy CDATA #IMPLIED>
-<!ATTLIST project groups      CDATA #IMPLIED>
-<!ATTLIST project sync\-c      CDATA #IMPLIED>
-<!ATTLIST project sync\-s      CDATA #IMPLIED>
-<!ATTLIST project sync\-tags   CDATA #IMPLIED>
-<!ATTLIST project upstream CDATA #IMPLIED>
-<!ATTLIST project clone\-depth CDATA #IMPLIED>
-<!ATTLIST project force\-path CDATA #IMPLIED>
 .IP
 <!ELEMENT annotation EMPTY>
 <!ATTLIST annotation name  CDATA #REQUIRED>
@@ -180,23 +202,43 @@ CDATA #IMPLIED>
 <!ATTLIST copyfile dest CDATA #REQUIRED>
 .IP
 <!ELEMENT linkfile EMPTY>
-<!ATTLIST linkfile src CDATA #REQUIRED>
+<!ATTLIST linkfile src  CDATA #REQUIRED>
 <!ATTLIST linkfile dest CDATA #REQUIRED>
+.TP
+<!ELEMENT extend\-project (annotation*,
+copyfile*,
+linkfile*)>
+.TP
+<!ATTLIST extend\-project name
+CDATA #REQUIRED>
+.TP
+<!ATTLIST extend\-project path
+CDATA #IMPLIED>
+.TP
+<!ATTLIST extend\-project dest\-path
+CDATA #IMPLIED>
+.TP
+<!ATTLIST extend\-project groups
+CDATA #IMPLIED>
+.TP
+<!ATTLIST extend\-project revision
+CDATA #IMPLIED>
+.TP
+<!ATTLIST extend\-project remote
+CDATA #IMPLIED>
 .IP
-<!ELEMENT extend\-project EMPTY>
+<!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
-<!ATTLIST extend\-project name CDATA #REQUIRED>
+<!ATTLIST extend\-project upstream    CDATA #IMPLIED>
-<!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project base\-rev    CDATA #IMPLIED>
-<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
-<!ATTLIST extend\-project groups CDATA #IMPLIED>
-<!ATTLIST extend\-project revision CDATA #IMPLIED>
-<!ATTLIST extend\-project remote CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
-<!ATTLIST remove\-project name  CDATA #REQUIRED>
+<!ATTLIST remove\-project name     CDATA #IMPLIED>
-<!ATTLIST remove\-project optional  CDATA #IMPLIED>
+<!ATTLIST remove\-project path     CDATA #IMPLIED>
+<!ATTLIST remove\-project optional CDATA #IMPLIED>
+<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT repo\-hooks EMPTY>
-<!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
+<!ATTLIST repo\-hooks in\-project   CDATA #REQUIRED>
 <!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
 .IP
 <!ELEMENT superproject EMPTY>
@@ -205,11 +247,12 @@ CDATA #IMPLIED>
 <!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
-<!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
+<!ATTLIST contactinfo bugurl CDATA #REQUIRED>
 .IP
 <!ELEMENT include EMPTY>
-<!ATTLIST include name   CDATA #REQUIRED>
+<!ATTLIST include name     CDATA #REQUIRED>
-<!ATTLIST include groups CDATA #IMPLIED>
+<!ATTLIST include groups   CDATA #IMPLIED>
+<!ATTLIST include revision CDATA #IMPLIED>
 .PP
 ]>
 ```
@@ -284,7 +327,9 @@ when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to syn
 entire ref space. Project elements not setting their own `upstream` will inherit
 this value.
 .PP
-Attribute `sync\-j`: Number of parallel jobs to use when synching.
+Attribute `sync\-j`: Number of parallel jobs to use when syncing.
+.PP
+Attribute `sync\-j\-max`: Maximum number of parallel jobs to use when syncing.
 .PP
 Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
 the `revision` attribute) rather than the whole ref space. Project elements
@@ -300,25 +345,7 @@ Element manifest\-server
 At most one manifest\-server may be specified. The url attribute is used to
 specify the URL of a manifest server, which is an XML RPC service.
 .PP
-The manifest server should implement the following RPC methods:
+See the [smart sync documentation](./smart\-sync.md) for more details.
-.IP
-GetApprovedManifest(branch, target)
-.PP
-Return a manifest in which each project is pegged to a known good revision for
-the current branch and target. This is used by repo sync when the \fB\-\-smart\-sync\fR
-option is given.
-.PP
-The target to use is defined by environment variables TARGET_PRODUCT and
-TARGET_BUILD_VARIANT. These variables are used to create a string of the form
-$TARGET_PRODUCT\-$TARGET_BUILD_VARIANT, e.g. passion\-userdebug. If one of those
-variables or both are not present, the program will call GetApprovedManifest
-without the target parameter and the manifest server should choose a reasonable
-default target.
-.IP
-GetManifest(tag)
-.PP
-Return a manifest in which each project is pegged to the revision at the
-specified tag. This is used by repo sync when the \fB\-\-smart\-tag\fR option is given.
 .PP
 Element submanifest
 .PP
@@ -370,7 +397,7 @@ supplied, `revision` is used.
 .PP
 `path` may not be an absolute path or use "." or ".." path components.
 .PP
-Attribute `groups`: List of additional groups to which all projects in the
+Attribute `groups`: Set of additional groups to which all projects in the
 included submanifest belong. This appends and recurses, meaning all projects in
 submanifests carry all parent submanifest groups. Same syntax as the
 corresponding element of `project`.
@@ -432,7 +459,7 @@ Attribute `dest\-branch`: Name of a Git branch (e.g. `main`). When using `repo
 upload`, changes will be submitted for code review on this branch. If
 unspecified both here and in the default element, `revision` is used instead.
 .PP
-Attribute `groups`: List of groups to which this project belongs, whitespace or
+Attribute `groups`: Set of groups to which this project belongs, whitespace or
 comma separated. All projects belong to the group "all", and each project
 automatically belongs to a group of its name:`name` and path:`path`. E.g. for
 `<project name="monkeys" path="barrel\-of"/>`, that project definition is
@@ -459,6 +486,21 @@ mirror repository according to its `path` attribute (if supplied) rather than
 the `name` attribute. This attribute only applies to the local mirrors syncing,
 it will be ignored when syncing the projects in a client working directory.
 .PP
+Attribute `sync\-strategy`: Set the sync strategy used when fetching this
+project. Currently the only supported value is `stateless`. When set to
+`stateless`, repo will run a reflog expiration and aggressive garbage collection
+at the end of the sync process. This is useful for projects that contain large
+binary files and use `clone\-depth="1"`, where garbage can accumulate as binaries
+are added, deleted, or modified across successive syncs.
+.PP
+During a stateless sync, repo checks the following before cleaning up: 1. The
+project does not share an object directory with other projects. 2. The working
+tree is clean (no uncommitted changes, no untracked files). 3. There are no
+unpushed local commits. 4. There is no Git stash.
+.PP
+If any of these conditions are not met, repo falls back to a standard sync
+without garbage collection.
+.PP
 Element extend\-project
 .PP
 Modify the attributes of the named project.
@@ -468,6 +510,11 @@ of an existing project without completely replacing the existing project
 definition. This makes the local manifest more robust against changes to the
 original manifest.
 .PP
+The `extend\-project` element can also contain `annotation`, `copyfile`, and
+`linkfile` child elements. These are added to the project's definition. A
+`copyfile` or `linkfile` with a `dest` that already exists in the project will
+overwrite the original.
+.PP
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
@@ -476,8 +523,8 @@ repo client where the Git working directory for this project should be placed.
 This is used to move a project in the checkout by overriding the existing `path`
 setting.
 .PP
-Attribute `groups`: List of additional groups to which this project belongs.
+Attribute `groups`: Set of additional groups to which this project belongs. Same
-Same syntax as the corresponding element of `project`.
+syntax as the corresponding element of `project`.
 .PP
 Attribute `revision`: If specified, overrides the revision of the original
 project. Same syntax as the corresponding element of `project`.
@@ -485,21 +532,37 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original project.
 Same syntax as the corresponding element of `project`.
 .PP
+Attribute `dest\-branch`: If specified, overrides the dest\-branch of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `upstream`: If specified, overrides the upstream of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+extended. Manifest parse will fail and give a list of mismatch extends if the
+revisions being extended have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element annotation
 .PP
-Zero or more annotation elements may be specified as children of a project or
+Zero or more annotation elements may be specified as children of a project
-remote element. Each element describes a name\-value pair. For projects, this
+element, an extend\-project element, or a remote element. Each element describes
-name\-value pair will be exported into each project's environment during a
+a name\-value pair. For projects, this name\-value pair will be exported into each
-\&'forall' command, prefixed with `REPO__`. In addition, there is an optional
+project's environment during a 'forall' command, prefixed with `REPO__`. In
-attribute "keep" which accepts the case insensitive values "true" (default) or
+addition, there is an optional attribute "keep" which accepts the case
-"false". This attribute determines whether or not the annotation will be kept
+insensitive values "true" (default) or "false". This attribute determines
-when exported with the manifest subcommand.
+whether or not the annotation will be kept when exported with the manifest
+subcommand.
 .PP
 Element copyfile
 .PP
 Zero or more copyfile elements may be specified as children of a project
-element. Each element describes a src\-dest pair of files; the "src" file will be
+element, or an extend\-project element. Each element describes a src\-dest pair of
-copied to the "dest" place during `repo sync` command.
+files; the "src" file will be copied to the "dest" place during `repo sync`
+command.
 .PP
 "src" is project relative, "dest" is relative to the top of the tree. Copying
 from paths outside of the project or to paths outside of the repo client is not
@@ -510,10 +573,14 @@ Intermediate paths must not be symlinks either.
 .PP
 Parent directories of "dest" will be automatically created if missing.
 .PP
+The files are copied in the order they are specified in the manifests. If
+multiple elements specify the same source and destination, they will only be
+applied as one, based on the first occurence. Files are copied before any links
+specified via linkfile elements are created.
+.PP
 Element linkfile
 .PP
-It's just like copyfile and runs at the same time as copyfile but instead of
+It's just like copyfile, but instead of copying it creates a symlink.
-copying it creates a symlink.
 .PP
 The symlink is created at "dest" (relative to the top of the tree) and points to
 the path specified by "src" which is a path in the project.
@@ -523,18 +590,42 @@ Parent directories of "dest" will be automatically created if missing.
 The symlink target may be a file or directory, but it may not point outside of
 the repo client.
 .PP
+The links are created in the order they are specified in the manifests. If
+multiple elements specify the same source and destination, they will only be
+applied as one, based on the first occurence. Links are created after any files
+specified via copyfile elements are copied.
+.PP
 Element remove\-project
 .PP
-Deletes the named project from the internal manifest table, possibly allowing a
+Deletes a project from the internal manifest table, possibly allowing a
 subsequent project element in the same manifest file to replace the project with
 a different source.
 .PP
 This element is mostly useful in a local manifest file, where the user can
 remove a project, and possibly replace it with their own definition.
 .PP
+The project `name` or project `path` can be used to specify the remove target
+meaning one of them is required. If only name is specified, all projects with
+that name are removed.
+.PP
+If both name and path are specified, only projects with the same name and path
+are removed, meaning projects with the same name but in other locations are
+kept.
+.PP
+If only path is specified, a matching project is removed regardless of its name.
+Logic otherwise behaves like both are specified.
+.PP
 Attribute `optional`: Set to true to ignore remove\-project elements with no
 matching `project` element.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+removed. Manifest parse will fail and give a list of mismatch removes if the
+revisions being removed have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element repo\-hooks
 .PP
 NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.
@@ -595,11 +686,17 @@ repository's root.
 "name" may not be an absolute path or use "." or ".." path components. These
 restrictions are not enforced for [Local Manifests].
 .PP
-Attribute `groups`: List of additional groups to which all projects in the
+Attribute `groups`: Set of additional groups to which all projects in the
 included manifest belong. This appends and recurses, meaning all projects in
-included manifests carry all parent include groups. Same syntax as the
+included manifests carry all parent include groups. This also applies to all
+extend\-project elements in the included manifests. Same syntax as the
 corresponding element of `project`.
 .PP
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`)
+default to which all projects in the included manifest belong. This recurses,
+meaning it will apply to all projects in all manifests included as a result of
+this element.
+.PP
 Local Manifests
 .PP
 Additional remotes and projects may be added through local manifest files stored

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo smartsync" "Repo Manual"
+.TH REPO "1" "May 2026" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1). Ignored unless \fB\-\-no\-interleaved\fR is set
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8). Ignored unless \fB\-\-nointerleaved\fR is set
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -37,11 +37,20 @@ overwrite an existing git directory if it needs to
 point to a different object directory. WARNING: this
 may cause loss of data
 .TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
 \fB\-\-force\-remove\-dirty\fR
 force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP
@@ -49,6 +58,12 @@ only update working tree, don't fetch
 use the existing manifest checkout as\-is. (do not
 update to the latest revision)
 .TP
+\fB\-\-interleaved\fR
+fetch and checkout projects in parallel (default)
+.TP
+\fB\-\-no\-interleaved\fR
+fetch and checkout projects in phases
+.TP
 \fB\-n\fR, \fB\-\-network\-only\fR
 fetch only, don't update working tree
 .TP
@@ -86,6 +101,10 @@ implies \fB\-c\fR
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
 .TP
+\fB\-\-superproject\-revision\fR=\fI\,SUPERPROJECT_REVISION\/\fR
+sync to superproject revision (applies to outer
+manifest)
+.TP
 \fB\-\-tags\fR
 fetch tags
 .TP
@@ -105,6 +124,13 @@ delete refs that no longer exist on the remote
 .TP
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -129,6 +155,16 @@ operate on this manifest and its submanifests
 .TP
 \fB\-\-no\-repo\-verify\fR
 do not verify repo source code
+.SS post\-sync hooks:
+.TP
+\fB\-\-no\-verify\fR
+Do not run the post\-sync hook.
+.TP
+\fB\-\-verify\fR
+Run the post\-sync hook without prompting.
+.TP
+\fB\-\-ignore\-hooks\fR
+Do not abort if post\-sync hooks fail.
 .PP
 Run `repo help smartsync` to view the detailed manual.
 .SH DETAILS

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo sync" "Repo Manual"
+.TH REPO "1" "May 2026" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1). Ignored unless \fB\-\-no\-interleaved\fR is set
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8). Ignored unless \fB\-\-nointerleaved\fR is set
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -37,11 +37,20 @@ overwrite an existing git directory if it needs to
 point to a different object directory. WARNING: this
 may cause loss of data
 .TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
 \fB\-\-force\-remove\-dirty\fR
 force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP
@@ -49,6 +58,12 @@ only update working tree, don't fetch
 use the existing manifest checkout as\-is. (do not
 update to the latest revision)
 .TP
+\fB\-\-interleaved\fR
+fetch and checkout projects in parallel (default)
+.TP
+\fB\-\-no\-interleaved\fR
+fetch and checkout projects in phases
+.TP
 \fB\-n\fR, \fB\-\-network\-only\fR
 fetch only, don't update working tree
 .TP
@@ -86,6 +101,10 @@ implies \fB\-c\fR
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
 .TP
+\fB\-\-superproject\-revision\fR=\fI\,SUPERPROJECT_REVISION\/\fR
+sync to superproject revision (applies to outer
+manifest)
+.TP
 \fB\-\-tags\fR
 fetch tags
 .TP
@@ -106,6 +125,13 @@ delete refs that no longer exist on the remote
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
 .TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
+.TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good
 build
@@ -136,6 +162,16 @@ operate on this manifest and its submanifests
 .TP
 \fB\-\-no\-repo\-verify\fR
 do not verify repo source code
+.SS post\-sync hooks:
+.TP
+\fB\-\-no\-verify\fR
+Do not run the post\-sync hook.
+.TP
+\fB\-\-verify\fR
+Run the post\-sync hook without prompting.
+.TP
+\fB\-\-ignore\-hooks\fR
+Do not abort if post\-sync hooks fail.
 .PP
 Run `repo help sync` to view the detailed manual.
 .SH DETAILS
@@ -178,6 +214,11 @@ The \fB\-\-force\-sync\fR option can be used to overwrite existing git directori
 they have previously been linked to a different object directory. WARNING: This
 may cause data to be lost since refs may be removed when overwriting.
 .PP
+The \fB\-\-force\-checkout\fR option can be used to force git to switch revs even if the
+index or the working tree differs from HEAD, and if there are untracked files.
+WARNING: This may cause data to be lost since uncommitted changes may be
+removed.
+.PP
 The \fB\-\-force\-remove\-dirty\fR option can be used to remove previously used projects
 with uncommitted changes. WARNING: This may cause data to be lost since
 uncommitted changes may be removed with projects that no longer exist in the
@@ -200,6 +241,9 @@ to a sha1 revision if the sha1 revision does not already exist locally.
 The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
 remote.
 .PP
+The \fB\-\-auto\-gc\fR option can be used to trigger garbage collection on all projects.
+By default, repo does not run garbage collection.
+.PP
 SSH Connections
 .PP
 If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,

man/repo-upload.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo upload" "Repo Manual"
+.TH REPO "1" "June 2024" "repo upload" "Repo Manual"
 .SH NAME
 repo \- repo upload - manual page for repo upload
 .SH SYNOPSIS
@@ -18,8 +18,11 @@ show this help message and exit
 number of jobs to run in parallel (default: based on
 number of CPU cores)
 .TP
-\fB\-t\fR
+\fB\-t\fR, \fB\-\-topic\-branch\fR
-send local branch name to Gerrit Code Review
+set the topic to the local branch name
+.TP
+\fB\-\-topic\fR=\fI\,TOPIC\/\fR
+set topic for the change
 .TP
 \fB\-\-hashtag\fR=\fI\,HASHTAGS\/\fR, \fB\-\-ht\fR=\fI\,HASHTAGS\/\fR
 add hashtags (comma delimited) to the review
@@ -30,6 +33,9 @@ add local branch name as a hashtag
 \fB\-l\fR LABELS, \fB\-\-label\fR=\fI\,LABELS\/\fR
 add a label when uploading
 .TP
+\fB\-\-pd\fR=\fI\,PATCHSET_DESCRIPTION\/\fR, \fB\-\-patchset\-description\fR=\fI\,PATCHSET_DESCRIPTION\/\fR
+description for patchset
+.TP
 \fB\-\-re\fR=\fI\,REVIEWERS\/\fR, \fB\-\-reviewers\fR=\fI\,REVIEWERS\/\fR
 request reviews from these people
 .TP
@@ -54,6 +60,9 @@ upload as a private change (deprecated; use \fB\-\-wip\fR)
 \fB\-w\fR, \fB\-\-wip\fR
 upload as a work\-in\-progress change
 .TP
+\fB\-r\fR, \fB\-\-ready\fR
+mark change as ready (clears work\-in\-progress setting)
+.TP
 \fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
 additional push options to transmit
 .TP
@@ -66,6 +75,12 @@ do everything except actually upload the CL
 \fB\-y\fR, \fB\-\-yes\fR
 answer yes to all safe prompts
 .TP
+\fB\-\-ignore\-untracked\-files\fR
+ignore untracked files in the working copy
+.TP
+\fB\-\-no\-ignore\-untracked\-files\fR
+always ask about untracked files in the working copy
+.TP
 \fB\-\-no\-cert\-checks\fR
 disable verifying ssl certs (unsafe)
 .SS Logging options:
@@ -118,6 +133,12 @@ respective list of users, and emails are sent to any new users. Users passed as
 \fB\-\-reviewers\fR must already be registered with the code review system, or the
 upload will fail.
 .PP
+While most normal Gerrit options have dedicated command line options, direct
+access to the Gerit options is available via \fB\-\-push\-options\fR. This is useful when
+Gerrit has newer functionality that repo upload doesn't yet support, or doesn't
+have plans to support. See the Push Options documentation for more details:
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#push_options
+.PP
 Configuration
 .PP
 review.URL.autoupload:
@@ -183,6 +204,12 @@ review.URL.uploadnotify:
 Control e\-mail notifications when uploading.
 https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#notify
 .PP
+review.URL.uploadwarningthreshold:
+.PP
+Repo will warn you if you are attempting to upload a large number of commits in
+one or more branches. By default, the threshold is five commits. This option
+allows you to override the warning threshold to a different value.
+.PP
 References
 .PP
 Gerrit Code Review: https://www.gerritcodereview.com/

man/repo-wipe.1

@@ -0,0 +1,61 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "November 2025" "repo wipe" "Repo Manual"
+.SH NAME
+repo \- repo wipe - manual page for repo wipe
+.SH SYNOPSIS
+.B repo
+\fI\,wipe <project>\/\fR...
+.SH DESCRIPTION
+Summary
+.PP
+Wipe projects from the worktree
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-f\fR, \fB\-\-force\fR
+force wipe shared projects and uncommitted changes
+.TP
+\fB\-\-force\-uncommitted\fR
+force wipe even if there are uncommitted changes
+.TP
+\fB\-\-force\-shared\fR
+force wipe even if the project shares an object
+directory
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help wipe` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo wipe' command removes the specified projects from the worktree (the
+checked out source code) and deletes the project's git data from `.repo`.
+.PP
+This is a destructive operation and cannot be undone.
+.PP
+Projects can be specified either by name, or by a relative or absolute path to
+the project's local directory.
+.SH EXAMPLES
+.SS # Wipe the project "platform/build" by name:
+$ repo wipe platform/build
+.SS # Wipe the project at the path "build/make":
+$ repo wipe build/make

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo" "Repo Manual"
+.TH REPO "1" "November 2025" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -25,6 +25,10 @@ control color usage: auto, always, never
 \fB\-\-trace\fR
 trace git command execution (REPO_TRACE=1)
 .TP
+\fB\-\-trace\-to\-stderr\fR
+trace outputs go to stderr in addition to
+\&.repo/TRACE_FILE
+.TP
 \fB\-\-trace\-python\fR
 trace python command execution
 .TP
@@ -75,11 +79,8 @@ Download and checkout a change
 forall
 Run a shell command in each project
 .TP
-gitc\-delete
+gc
-Delete a GITC Client.
+Cleaning up internal repo and Git state.
-.TP
-gitc\-init
-Initialize a GITC Client.
 .TP
 grep
 Print lines matching a pattern
@@ -131,6 +132,9 @@ Upload changes for code review
 .TP
 version
 Display the version of repo
+.TP
+wipe
+Wipe projects from the worktree
 .PP
 See 'repo help <command>' for more information on a specific command.
-Bug reports: https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue
+Bug reports: https://issues.gerritcodereview.com/issues/new?component=1370071

pager.py

@@ -19,6 +19,7 @@ import sys
 
 import platform_utils
 
+
 active = False
 pager_process = None
 old_stdout = None
@@ -26,99 +27,104 @@ old_stderr = None
 
 
 def RunPager(globalConfig):
-  if not os.isatty(0) or not os.isatty(1):
+    if not os.isatty(0) or not os.isatty(1):
-    return
+        return
-  pager = _SelectPager(globalConfig)
+    pager = _SelectPager(globalConfig)
-  if pager == '' or pager == 'cat':
+    if pager == "" or pager == "cat":
-    return
+        return
 
-  if platform_utils.isWindows():
+    if platform_utils.isWindows():
-    _PipePager(pager)
+        _PipePager(pager)
-  else:
+    else:
-    _ForkPager(pager)
+        _ForkPager(pager)
 
 
 def TerminatePager():
-  global pager_process, old_stdout, old_stderr
+    global pager_process
-  if pager_process:
+    if pager_process:
-    sys.stdout.flush()
+        sys.stdout.flush()
-    sys.stderr.flush()
+        sys.stderr.flush()
-    pager_process.stdin.close()
+        pager_process.stdin.close()
-    pager_process.wait()
+        pager_process.wait()
-    pager_process = None
+        pager_process = None
-    # Restore initial stdout/err in case there is more output in this process
+        # Restore initial stdout/err in case there is more output in this
-    # after shutting down the pager process
+        # process after shutting down the pager process.
-    sys.stdout = old_stdout
+        sys.stdout = old_stdout
-    sys.stderr = old_stderr
+        sys.stderr = old_stderr
 
 
 def _PipePager(pager):
-  global pager_process, old_stdout, old_stderr
+    global pager_process, old_stdout, old_stderr
-  assert pager_process is None, "Only one active pager process at a time"
+    assert pager_process is None, "Only one active pager process at a time"
-  # Create pager process, piping stdout/err into its stdin
+    # Create pager process, piping stdout/err into its stdin.
-  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
+    try:
-                                   stderr=sys.stderr)
+        pager_process = subprocess.Popen(
-  old_stdout = sys.stdout
+            [pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr
-  old_stderr = sys.stderr
+        )
-  sys.stdout = pager_process.stdin
+    except FileNotFoundError:
-  sys.stderr = pager_process.stdin
+        sys.exit(f'fatal: cannot start pager "{pager}"')
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = pager_process.stdin
+    sys.stderr = pager_process.stdin
 
 
 def _ForkPager(pager):
-  global active
+    global active
-  # This process turns into the pager; a child it forks will
+    # This process turns into the pager; a child it forks will
-  # do the real processing and output back to the pager. This
+    # do the real processing and output back to the pager. This
-  # is necessary to keep the pager in control of the tty.
+    # is necessary to keep the pager in control of the tty.
-  #
+    try:
-  try:
+        r, w = os.pipe()
-    r, w = os.pipe()
+        pid = os.fork()
-    pid = os.fork()
+        if not pid:
-    if not pid:
+            os.dup2(w, 1)
-      os.dup2(w, 1)
+            os.dup2(w, 2)
-      os.dup2(w, 2)
+            os.close(r)
-      os.close(r)
+            os.close(w)
-      os.close(w)
+            active = True
-      active = True
+            return
-      return
 
-    os.dup2(r, 0)
+        os.dup2(r, 0)
-    os.close(r)
+        os.close(r)
-    os.close(w)
+        os.close(w)
 
-    _BecomePager(pager)
+        _BecomePager(pager)
-  except Exception:
+    except Exception:
-    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
+        print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
-    sys.exit(255)
+        sys.exit(255)
 
 
 def _SelectPager(globalConfig):
-  try:
+    try:
-    return os.environ['GIT_PAGER']
+        return os.environ["GIT_PAGER"]
-  except KeyError:
+    except KeyError:
-    pass
+        pass
 
-  pager = globalConfig.GetString('core.pager')
+    pager = globalConfig.GetString("core.pager")
-  if pager:
+    if pager:
-    return pager
+        return pager
 
-  try:
+    try:
-    return os.environ['PAGER']
+        return os.environ["PAGER"]
-  except KeyError:
+    except KeyError:
-    pass
+        pass
 
-  return 'less'
+    return "less"
 
 
 def _BecomePager(pager):
-  # Delaying execution of the pager until we have output
+    # Delaying execution of the pager until we have output
-  # ready works around a long-standing bug in popularly
+    # ready works around a long-standing bug in popularly
-  # available versions of 'less', a better 'more'.
+    # available versions of 'less', a better 'more'.
-  #
+    _a, _b, _c = select.select([0], [], [0])
-  _a, _b, _c = select.select([0], [], [0])
 
-  os.environ['LESS'] = 'FRSX'
+    # This matches the behavior of git, which sets $LESS to `FRX` if it is not
+    # set. See:
+    # https://git-scm.com/docs/git-config#Documentation/git-config.txt-corepager
+    os.environ.setdefault("LESS", "FRX")
 
-  try:
+    try:
-    os.execvp(pager, [pager])
+        os.execvp(pager, [pager])
-  except OSError:
+    except OSError:
-    os.execv('/bin/sh', ['sh', '-c', pager])
+        os.execv("/bin/sh", ["sh", "-c", pager])

platform_utils.py

@@ -20,246 +20,277 @@ import stat
 
 
 def isWindows():
-  """ Returns True when running with the native port of Python for Windows,
+    """Returns True when running with the native port of Python for Windows,
-  False when running on any other platform (including the Cygwin port of
+    False when running on any other platform (including the Cygwin port of
-  Python).
+    Python).
-  """
+    """
-  # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
+    # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
-  return platform.system() == "Windows"
+    return platform.system() == "Windows"
 
 
 def symlink(source, link_name):
-  """Creates a symbolic link pointing to source named link_name.
+    """Creates a symbolic link pointing to source named link_name.
-  Note: On Windows, source must exist on disk, as the implementation needs
+
-  to know whether to create a "File" or a "Directory" symbolic link.
+    Note: On Windows, source must exist on disk, as the implementation needs
-  """
+    to know whether to create a "File" or a "Directory" symbolic link.
-  if isWindows():
+    """
-    import platform_utils_win32
+    if isWindows():
-    source = _validate_winpath(source)
+        import platform_utils_win32
-    link_name = _validate_winpath(link_name)
+
-    target = os.path.join(os.path.dirname(link_name), source)
+        source = _validate_winpath(source)
-    if isdir(target):
+        link_name = _validate_winpath(link_name)
-      platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
+        target = os.path.join(os.path.dirname(link_name), source)
+        if isdir(target):
+            platform_utils_win32.create_dirsymlink(
+                _makelongpath(source), link_name
+            )
+        else:
+            platform_utils_win32.create_filesymlink(
+                _makelongpath(source), link_name
+            )
     else:
-      platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
+        return os.symlink(source, link_name)
-  else:
-    return os.symlink(source, link_name)
 
 
 def _validate_winpath(path):
-  path = os.path.normpath(path)
+    path = os.path.normpath(path)
-  if _winpath_is_valid(path):
+    if _winpath_is_valid(path):
-    return path
+        return path
-  raise ValueError("Path \"%s\" must be a relative path or an absolute "
+    raise ValueError(
-                   "path starting with a drive letter".format(path))
+        f'Path "{path}" must be a relative path or an absolute '
+        "path starting with a drive letter"
+    )
 
 
 def _winpath_is_valid(path):
-  """Windows only: returns True if path is relative (e.g. ".\\foo") or is
+    """Windows only: returns True if path is relative (e.g. ".\\foo") or is
-  absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
+    absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
-  is ambiguous (e.g. "x:foo" or "\\foo").
+    is ambiguous (e.g. "x:foo" or "\\foo").
-  """
+    """
-  assert isWindows()
+    assert isWindows()
-  path = os.path.normpath(path)
+    path = os.path.normpath(path)
-  drive, tail = os.path.splitdrive(path)
+    drive, tail = os.path.splitdrive(path)
-  if tail:
+    if tail:
-    if not drive:
+        if not drive:
-      return tail[0] != os.sep  # "\\foo" is invalid
+            return tail[0] != os.sep  # "\\foo" is invalid
+        else:
+            return tail[0] == os.sep  # "x:foo" is invalid
     else:
-      return tail[0] == os.sep  # "x:foo" is invalid
+        return not drive  # "x:" is invalid
-  else:
-    return not drive  # "x:" is invalid
 
 
 def _makelongpath(path):
-  """Return the input path normalized to support the Windows long path syntax
+    """Return the input path normalized to support the Windows long path syntax
-  ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+    ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
-  MAX_PATH limit.
+    MAX_PATH limit.
-  """
+    """
-  if isWindows():
+    if isWindows():
-    # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
+        # Note: MAX_PATH is 260, but, for directories, the maximum value is
-    if len(path) < 246:
+        # actually 246.
-      return path
+        if len(path) < 246:
-    if path.startswith(u"\\\\?\\"):
+            return path
-      return path
+        if path.startswith("\\\\?\\"):
-    if not os.path.isabs(path):
+            return path
-      return path
+        if not os.path.isabs(path):
-    # Append prefix and ensure unicode so that the special longpath syntax
+            return path
-    # is supported by underlying Win32 API calls
+        # Append prefix and ensure unicode so that the special longpath syntax
-    return u"\\\\?\\" + os.path.normpath(path)
+        # is supported by underlying Win32 API calls
-  else:
+        return "\\\\?\\" + os.path.normpath(path)
-    return path
+    else:
+        return path
 
 
 def rmtree(path, ignore_errors=False):
-  """shutil.rmtree(path) wrapper with support for long paths on Windows.
+    """shutil.rmtree(path) wrapper with support for long paths on Windows.
 
-  Availability: Unix, Windows."""
+    Availability: Unix, Windows.
-  onerror = None
+    """
-  if isWindows():
+    onerror = None
-    path = _makelongpath(path)
+    if isWindows():
-    onerror = handle_rmtree_error
+        path = _makelongpath(path)
-  shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
+        onerror = handle_rmtree_error
+    shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
 
 
 def handle_rmtree_error(function, path, excinfo):
-  # Allow deleting read-only files
+    # Allow deleting read-only files.
-  os.chmod(path, stat.S_IWRITE)
+    os.chmod(path, stat.S_IWRITE)
-  function(path)
+    function(path)
 
 
 def rename(src, dst):
-  """os.rename(src, dst) wrapper with support for long paths on Windows.
+    """os.rename(src, dst) wrapper with support for long paths on Windows.
 
-  Availability: Unix, Windows."""
+    Availability: Unix, Windows.
-  if isWindows():
+    """
-    # On Windows, rename fails if destination exists, see
+    if isWindows():
-    # https://docs.python.org/2/library/os.html#os.rename
+        # On Windows, rename fails if destination exists, see
-    try:
+        # https://docs.python.org/2/library/os.html#os.rename
-      os.rename(_makelongpath(src), _makelongpath(dst))
+        try:
-    except OSError as e:
+            os.rename(_makelongpath(src), _makelongpath(dst))
-      if e.errno == errno.EEXIST:
+        except OSError as e:
-        os.remove(_makelongpath(dst))
+            if e.errno == errno.EEXIST:
-        os.rename(_makelongpath(src), _makelongpath(dst))
+                os.remove(_makelongpath(dst))
-      else:
+                os.rename(_makelongpath(src), _makelongpath(dst))
-        raise
+            else:
-  else:
+                raise
-    shutil.move(src, dst)
+    else:
+        shutil.move(src, dst)
 
 
 def remove(path, missing_ok=False):
-  """Remove (delete) the file path. This is a replacement for os.remove that
+    """Remove (delete) the file path. This is a replacement for os.remove that
-  allows deleting read-only files on Windows, with support for long paths and
+    allows deleting read-only files on Windows, with support for long paths and
-  for deleting directory symbolic links.
+    for deleting directory symbolic links.
 
-  Availability: Unix, Windows."""
+    Availability: Unix, Windows.
-  longpath = _makelongpath(path) if isWindows() else path
+    """
-  try:
+    longpath = _makelongpath(path) if isWindows() else path
-    os.remove(longpath)
+    try:
-  except OSError as e:
-    if e.errno == errno.EACCES:
-      os.chmod(longpath, stat.S_IWRITE)
-      # Directory symbolic links must be deleted with 'rmdir'.
-      if islink(longpath) and isdir(longpath):
-        os.rmdir(longpath)
-      else:
         os.remove(longpath)
-    elif missing_ok and e.errno == errno.ENOENT:
+    except OSError as e:
-      pass
+        if e.errno == errno.EACCES:
-    else:
+            os.chmod(longpath, stat.S_IWRITE)
-      raise
+            # Directory symbolic links must be deleted with 'rmdir'.
+            if islink(longpath) and isdir(longpath):
+                os.rmdir(longpath)
+            else:
+                os.remove(longpath)
+        elif (
+            e.errno == errno.EROFS
+            and missing_ok
+            and not os.path.exists(longpath)
+        ):
+            pass
+        elif missing_ok and e.errno == errno.ENOENT:
+            pass
+        else:
+            raise
 
 
 def walk(top, topdown=True, onerror=None, followlinks=False):
-  """os.walk(path) wrapper with support for long paths on Windows.
+    """os.walk(path) wrapper with support for long paths on Windows.
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  if isWindows():
+    if isWindows():
-    return _walk_windows_impl(top, topdown, onerror, followlinks)
+        return _walk_windows_impl(top, topdown, onerror, followlinks)
-  else:
+    else:
-    return os.walk(top, topdown, onerror, followlinks)
+        return os.walk(top, topdown, onerror, followlinks)
 
 
 def _walk_windows_impl(top, topdown, onerror, followlinks):
-  try:
+    try:
-    names = listdir(top)
+        names = listdir(top)
-  except Exception as err:
+    except Exception as err:
-    if onerror is not None:
+        if onerror is not None:
-      onerror(err)
+            onerror(err)
-    return
+        return
 
-  dirs, nondirs = [], []
+    dirs, nondirs = [], []
-  for name in names:
+    for name in names:
-    if isdir(os.path.join(top, name)):
+        if isdir(os.path.join(top, name)):
-      dirs.append(name)
+            dirs.append(name)
-    else:
+        else:
-      nondirs.append(name)
+            nondirs.append(name)
 
-  if topdown:
+    if topdown:
-    yield top, dirs, nondirs
+        yield top, dirs, nondirs
-  for name in dirs:
+    for name in dirs:
-    new_path = os.path.join(top, name)
+        new_path = os.path.join(top, name)
-    if followlinks or not islink(new_path):
+        if followlinks or not islink(new_path):
-      for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
+            yield from _walk_windows_impl(
-        yield x
+                new_path, topdown, onerror, followlinks
-  if not topdown:
+            )
-    yield top, dirs, nondirs
+    if not topdown:
+        yield top, dirs, nondirs
 
 
 def listdir(path):
-  """os.listdir(path) wrapper with support for long paths on Windows.
+    """os.listdir(path) wrapper with support for long paths on Windows.
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  return os.listdir(_makelongpath(path))
+    return os.listdir(_makelongpath(path))
 
 
 def rmdir(path):
-  """os.rmdir(path) wrapper with support for long paths on Windows.
+    """os.rmdir(path) wrapper with support for long paths on Windows.
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  os.rmdir(_makelongpath(path))
+    os.rmdir(_makelongpath(path))
+
+
+def removedirs(path: str) -> None:
+    """Remove a directory tree of symlinks and empty directories.
+
+    Walks |path| bottom-up without following symlinks.  Removes symlinks
+    and empty directories.  Stops at (and preserves) regular files and
+    non-empty directories.  Silently succeeds if |path| does not exist.
+
+    Availability: Unix, Windows.
+    """
+    if islink(path):
+        remove(path)
+        return
+
+    if not isdir(path):
+        return
+
+    for dirpath, dirnames, filenames in walk(path, topdown=False):
+        for name in dirnames:
+            entry = os.path.join(dirpath, name)
+            if islink(entry):
+                remove(entry)
+            else:
+                try:
+                    rmdir(entry)
+                except OSError:
+                    pass
+        for name in filenames:
+            entry = os.path.join(dirpath, name)
+            if islink(entry):
+                remove(entry)
+
+    try:
+        rmdir(path)
+    except OSError:
+        pass
 
 
 def isdir(path):
-  """os.path.isdir(path) wrapper with support for long paths on Windows.
+    """os.path.isdir(path) wrapper with support for long paths on Windows.
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  return os.path.isdir(_makelongpath(path))
+    return os.path.isdir(_makelongpath(path))
 
 
 def islink(path):
-  """os.path.islink(path) wrapper with support for long paths on Windows.
+    """os.path.islink(path) wrapper with support for long paths on Windows.
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  if isWindows():
+    if isWindows():
-    import platform_utils_win32
+        import platform_utils_win32
-    return platform_utils_win32.islink(_makelongpath(path))
+
-  else:
+        return platform_utils_win32.islink(_makelongpath(path))
-    return os.path.islink(path)
+    else:
+        return os.path.islink(path)
 
 
 def readlink(path):
-  """Return a string representing the path to which the symbolic link
+    """Return a string representing the path to which the symbolic link
-  points. The result may be either an absolute or relative pathname;
+    points. The result may be either an absolute or relative pathname;
-  if it is relative, it may be converted to an absolute pathname using
+    if it is relative, it may be converted to an absolute pathname using
-  os.path.join(os.path.dirname(path), result).
+    os.path.join(os.path.dirname(path), result).
 
-  Availability: Windows, Unix.
+    Availability: Windows, Unix.
-  """
+    """
-  if isWindows():
+    if isWindows():
-    import platform_utils_win32
+        import platform_utils_win32
-    return platform_utils_win32.readlink(_makelongpath(path))
-  else:
-    return os.readlink(path)
 
-
+        return platform_utils_win32.readlink(_makelongpath(path))
-def realpath(path):
+    else:
-  """Return the canonical path of the specified filename, eliminating
+        return os.readlink(path)
-  any symbolic links encountered in the path.
-
-  Availability: Windows, Unix.
-  """
-  if isWindows():
-    current_path = os.path.abspath(path)
-    path_tail = []
-    for c in range(0, 100):  # Avoid cycles
-      if islink(current_path):
-        target = readlink(current_path)
-        current_path = os.path.join(os.path.dirname(current_path), target)
-      else:
-        basename = os.path.basename(current_path)
-        if basename == '':
-          path_tail.append(current_path)
-          break
-        path_tail.append(basename)
-        current_path = os.path.dirname(current_path)
-    path_tail.reverse()
-    result = os.path.normpath(os.path.join(*path_tail))
-    return result
-  else:
-    return os.path.realpath(path)

platform_utils_win32.py

@@ -12,14 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from ctypes import addressof
+from ctypes import byref
+from ctypes import c_buffer
+from ctypes import c_ubyte
+from ctypes import FormatError
+from ctypes import get_last_error
+from ctypes import Structure
+from ctypes import Union
+from ctypes import WinDLL
+from ctypes import WinError
+from ctypes.wintypes import BOOL
+from ctypes.wintypes import BOOLEAN
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import HANDLE
+from ctypes.wintypes import LPCWSTR
+from ctypes.wintypes import LPDWORD
+from ctypes.wintypes import LPVOID
+from ctypes.wintypes import ULONG
+from ctypes.wintypes import USHORT
+from ctypes.wintypes import WCHAR
 import errno
 
-from ctypes import WinDLL, get_last_error, FormatError, WinError, addressof
-from ctypes import c_buffer, c_ubyte, Structure, Union, byref
-from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE
-from ctypes.wintypes import WCHAR, USHORT, LPVOID, ULONG, LPDWORD
 
-kernel32 = WinDLL('kernel32', use_last_error=True)
+kernel32 = WinDLL("kernel32", use_last_error=True)
 
 UCHAR = c_ubyte
 
@@ -31,14 +47,17 @@ ERROR_PRIVILEGE_NOT_HELD = 1314
 # Win32 API entry points
 CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
 CreateSymbolicLinkW.restype = BOOLEAN
-CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
+CreateSymbolicLinkW.argtypes = (
-                                LPCWSTR,  # lpTargetFileName In
+    LPCWSTR,  # lpSymlinkFileName In
-                                DWORD)    # dwFlags In
+    LPCWSTR,  # lpTargetFileName In
+    DWORD,  # dwFlags In
+)
 
 # Symbolic link creation flags
 SYMBOLIC_LINK_FLAG_FILE = 0x00
 SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
-# symlink support for CreateSymbolicLink() starting with Windows 10 (1703, v10.0.14972)
+# symlink support for CreateSymbolicLink() starting with Windows 10 (1703,
+# v10.0.14972)
 SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
 
 GetFileAttributesW = kernel32.GetFileAttributesW
@@ -50,13 +69,15 @@ FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
 
 CreateFileW = kernel32.CreateFileW
 CreateFileW.restype = HANDLE
-CreateFileW.argtypes = (LPCWSTR,  # lpFileName In
+CreateFileW.argtypes = (
-                        DWORD,    # dwDesiredAccess In
+    LPCWSTR,  # lpFileName In
-                        DWORD,    # dwShareMode In
+    DWORD,  # dwDesiredAccess In
-                        LPVOID,   # lpSecurityAttributes In_opt
+    DWORD,  # dwShareMode In
-                        DWORD,    # dwCreationDisposition In
+    LPVOID,  # lpSecurityAttributes In_opt
-                        DWORD,    # dwFlagsAndAttributes In
+    DWORD,  # dwCreationDisposition In
-                        HANDLE)   # hTemplateFile In_opt
+    DWORD,  # dwFlagsAndAttributes In
+    HANDLE,  # hTemplateFile In_opt
+)
 
 CloseHandle = kernel32.CloseHandle
 CloseHandle.restype = BOOL
@@ -69,14 +90,16 @@ FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
 
 DeviceIoControl = kernel32.DeviceIoControl
 DeviceIoControl.restype = BOOL
-DeviceIoControl.argtypes = (HANDLE,   # hDevice In
+DeviceIoControl.argtypes = (
-                            DWORD,    # dwIoControlCode In
+    HANDLE,  # hDevice In
-                            LPVOID,   # lpInBuffer In_opt
+    DWORD,  # dwIoControlCode In
-                            DWORD,    # nInBufferSize In
+    LPVOID,  # lpInBuffer In_opt
-                            LPVOID,   # lpOutBuffer Out_opt
+    DWORD,  # nInBufferSize In
-                            DWORD,    # nOutBufferSize In
+    LPVOID,  # lpOutBuffer Out_opt
-                            LPDWORD,  # lpBytesReturned Out_opt
+    DWORD,  # nOutBufferSize In
-                            LPVOID)   # lpOverlapped Inout_opt
+    LPDWORD,  # lpBytesReturned Out_opt
+    LPVOID,  # lpOverlapped Inout_opt
+)
 
 # Device I/O control flags and options
 FSCTL_GET_REPARSE_POINT = 0x000900A8
@@ -86,124 +109,136 @@ MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
 
 
 class GENERIC_REPARSE_BUFFER(Structure):
-  _fields_ = (('DataBuffer', UCHAR * 1),)
+    _fields_ = (("DataBuffer", UCHAR * 1),)
 
 
 class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
-  _fields_ = (('SubstituteNameOffset', USHORT),
+    _fields_ = (
-              ('SubstituteNameLength', USHORT),
+        ("SubstituteNameOffset", USHORT),
-              ('PrintNameOffset', USHORT),
+        ("SubstituteNameLength", USHORT),
-              ('PrintNameLength', USHORT),
+        ("PrintNameOffset", USHORT),
-              ('Flags', ULONG),
+        ("PrintNameLength", USHORT),
-              ('PathBuffer', WCHAR * 1))
+        ("Flags", ULONG),
+        ("PathBuffer", WCHAR * 1),
+    )
 
-  @property
+    @property
-  def PrintName(self):
+    def PrintName(self):
-    arrayt = WCHAR * (self.PrintNameLength // 2)
+        arrayt = WCHAR * (self.PrintNameLength // 2)
-    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+        offset = type(self).PathBuffer.offset + self.PrintNameOffset
-    return arrayt.from_address(addressof(self) + offset).value
+        return arrayt.from_address(addressof(self) + offset).value
 
 
 class MOUNT_POINT_REPARSE_BUFFER(Structure):
-  _fields_ = (('SubstituteNameOffset', USHORT),
+    _fields_ = (
-              ('SubstituteNameLength', USHORT),
+        ("SubstituteNameOffset", USHORT),
-              ('PrintNameOffset', USHORT),
+        ("SubstituteNameLength", USHORT),
-              ('PrintNameLength', USHORT),
+        ("PrintNameOffset", USHORT),
-              ('PathBuffer', WCHAR * 1))
+        ("PrintNameLength", USHORT),
+        ("PathBuffer", WCHAR * 1),
+    )
 
-  @property
+    @property
-  def PrintName(self):
+    def PrintName(self):
-    arrayt = WCHAR * (self.PrintNameLength // 2)
+        arrayt = WCHAR * (self.PrintNameLength // 2)
-    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+        offset = type(self).PathBuffer.offset + self.PrintNameOffset
-    return arrayt.from_address(addressof(self) + offset).value
+        return arrayt.from_address(addressof(self) + offset).value
 
 
 class REPARSE_DATA_BUFFER(Structure):
-  class REPARSE_BUFFER(Union):
+    class REPARSE_BUFFER(Union):
-    _fields_ = (('SymbolicLinkReparseBuffer', SYMBOLIC_LINK_REPARSE_BUFFER),
+        _fields_ = (
-                ('MountPointReparseBuffer', MOUNT_POINT_REPARSE_BUFFER),
+            ("SymbolicLinkReparseBuffer", SYMBOLIC_LINK_REPARSE_BUFFER),
-                ('GenericReparseBuffer', GENERIC_REPARSE_BUFFER))
+            ("MountPointReparseBuffer", MOUNT_POINT_REPARSE_BUFFER),
-  _fields_ = (('ReparseTag', ULONG),
+            ("GenericReparseBuffer", GENERIC_REPARSE_BUFFER),
-              ('ReparseDataLength', USHORT),
+        )
-              ('Reserved', USHORT),
+
-              ('ReparseBuffer', REPARSE_BUFFER))
+    _fields_ = (
-  _anonymous_ = ('ReparseBuffer',)
+        ("ReparseTag", ULONG),
+        ("ReparseDataLength", USHORT),
+        ("Reserved", USHORT),
+        ("ReparseBuffer", REPARSE_BUFFER),
+    )
+    _anonymous_ = ("ReparseBuffer",)
 
 
 def create_filesymlink(source, link_name):
-  """Creates a Windows file symbolic link source pointing to link_name."""
+    """Creates a Windows file symbolic link source pointing to link_name."""
-  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+    _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
 
 
 def create_dirsymlink(source, link_name):
-  """Creates a Windows directory symbolic link source pointing to link_name.
+    """Creates a Windows directory symbolic link source pointing to link_name."""  # noqa: E501
-  """
+    _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
-  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
 
 
 def _create_symlink(source, link_name, dwFlags):
-  if not CreateSymbolicLinkW(link_name, source,
+    if not CreateSymbolicLinkW(
-                             dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE):
+        link_name,
-    # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0
+        source,
-    # "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972).
+        dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE,
-    # retry without it."
+    ):
-    if not CreateSymbolicLinkW(link_name, source, dwFlags):
+        # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0  # noqa: E501
-      code = get_last_error()
+        # "the unprivileged create flag is unsupported below Windows 10 (1703,
-      error_desc = FormatError(code).strip()
+        # v10.0.14972). retry without it."
-      if code == ERROR_PRIVILEGE_NOT_HELD:
+        if not CreateSymbolicLinkW(link_name, source, dwFlags):
-        raise OSError(errno.EPERM, error_desc, link_name)
+            code = get_last_error()
-      _raise_winerror(
+            error_desc = FormatError(code).strip()
-          code,
+            if code == ERROR_PRIVILEGE_NOT_HELD:
-          'Error creating symbolic link \"%s\"'.format(link_name))
+                raise OSError(errno.EPERM, error_desc, link_name)
+            _raise_winerror(code, f'Error creating symbolic link "{link_name}"')
 
 
 def islink(path):
-  result = GetFileAttributesW(path)
+    result = GetFileAttributesW(path)
-  if result == INVALID_FILE_ATTRIBUTES:
+    if result == INVALID_FILE_ATTRIBUTES:
-    return False
+        return False
-  return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
+    return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
 
 
 def readlink(path):
-  reparse_point_handle = CreateFileW(path,
+    reparse_point_handle = CreateFileW(
-                                     0,
+        path,
-                                     0,
+        0,
-                                     None,
+        0,
-                                     OPEN_EXISTING,
+        None,
-                                     FILE_FLAG_OPEN_REPARSE_POINT |
+        OPEN_EXISTING,
-                                     FILE_FLAG_BACKUP_SEMANTICS,
+        FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
-                                     None)
+        None,
-  if reparse_point_handle == INVALID_HANDLE_VALUE:
+    )
+    if reparse_point_handle == INVALID_HANDLE_VALUE:
+        _raise_winerror(
+            get_last_error(), f'Error opening symbolic link "{path}"'
+        )
+    target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
+    n_bytes_returned = DWORD()
+    io_result = DeviceIoControl(
+        reparse_point_handle,
+        FSCTL_GET_REPARSE_POINT,
+        None,
+        0,
+        target_buffer,
+        len(target_buffer),
+        byref(n_bytes_returned),
+        None,
+    )
+    CloseHandle(reparse_point_handle)
+    if not io_result:
+        _raise_winerror(
+            get_last_error(), f'Error reading symbolic link "{path}"'
+        )
+    rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
+    if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
+        return rdb.SymbolicLinkReparseBuffer.PrintName
+    elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
+        return rdb.MountPointReparseBuffer.PrintName
+    # Unsupported reparse point type.
     _raise_winerror(
-        get_last_error(),
+        ERROR_NOT_SUPPORTED, f'Error reading symbolic link "{path}"'
-        'Error opening symbolic link \"%s\"'.format(path))
+    )
-  target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
-  n_bytes_returned = DWORD()
-  io_result = DeviceIoControl(reparse_point_handle,
-                              FSCTL_GET_REPARSE_POINT,
-                              None,
-                              0,
-                              target_buffer,
-                              len(target_buffer),
-                              byref(n_bytes_returned),
-                              None)
-  CloseHandle(reparse_point_handle)
-  if not io_result:
-    _raise_winerror(
-        get_last_error(),
-        'Error reading symbolic link \"%s\"'.format(path))
-  rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
-  if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
-    return rdb.SymbolicLinkReparseBuffer.PrintName
-  elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
-    return rdb.MountPointReparseBuffer.PrintName
-  # Unsupported reparse point type
-  _raise_winerror(
-      ERROR_NOT_SUPPORTED,
-      'Error reading symbolic link \"%s\"'.format(path))
 
 
 def _raise_winerror(code, error_desc):
-  win_error_desc = FormatError(code).strip()
+    win_error_desc = FormatError(code).strip()
-  error_desc = "%s: %s".format(error_desc, win_error_desc)
+    error_desc = f"{error_desc}: {win_error_desc}"
-  raise WinError(code, error_desc)
+    raise WinError(code, error_desc)

progress.py

@@ -14,123 +14,240 @@
 
 import os
 import sys
-from time import time
+import time
-from repo_trace import IsTrace
 
-_NOT_TTY = not os.isatty(2)
+
+try:
+    import threading as _threading
+except ImportError:
+    import dummy_threading as _threading
+
+from repo_trace import IsTraceToStderr
+
+
+# Capture the original stderr stream. We use this exclusively for progress
+# updates to ensure we talk to the terminal even if stderr is redirected.
+_STDERR = sys.stderr
+_TTY = _STDERR.isatty()
 
 # This will erase all content in the current line (wherever the cursor is).
 # It does not move the cursor, so this is usually followed by \r to move to
 # column 0.
-CSI_ERASE_LINE = '\x1b[2K'
+CSI_ERASE_LINE = "\x1b[2K"
 
 # This will erase all content in the current line after the cursor.  This is
 # useful for partial updates & progress messages as the terminal can display
 # it better.
-CSI_ERASE_LINE_AFTER = '\x1b[K'
+CSI_ERASE_LINE_AFTER = "\x1b[K"
+
+
+def convert_to_hms(total):
+    """Converts a period of seconds to hours, minutes, and seconds."""
+    hours, rem = divmod(total, 3600)
+    mins, secs = divmod(rem, 60)
+    return int(hours), int(mins), secs
 
 
 def duration_str(total):
-  """A less noisy timedelta.__str__.
+    """A less noisy timedelta.__str__.
 
-  The default timedelta stringification contains a lot of leading zeros and
+    The default timedelta stringification contains a lot of leading zeros and
-  uses microsecond resolution.  This makes for noisy output.
+    uses microsecond resolution.  This makes for noisy output.
-  """
+    """
-  hours, rem = divmod(total, 3600)
+    hours, mins, secs = convert_to_hms(total)
-  mins, secs = divmod(rem, 60)
+    ret = f"{secs:.3f}s"
-  ret = '%.3fs' % (secs,)
+    if mins:
-  if mins:
+        ret = f"{mins}m{ret}"
-    ret = '%im%s' % (mins, ret)
+    if hours:
-  if hours:
+        ret = f"{hours}h{ret}"
-    ret = '%ih%s' % (hours, ret)
+    return ret
-  return ret
 
 
-class Progress(object):
+def elapsed_str(total):
-  def __init__(self, title, total=0, units='', print_newline=False, delay=True,
+    """Returns seconds in the format [H:]MM:SS.
-               quiet=False):
-    self._title = title
-    self._total = total
-    self._done = 0
-    self._start = time()
-    self._show = not delay
-    self._units = units
-    self._print_newline = print_newline
-    # Only show the active jobs section if we run more than one in parallel.
-    self._show_jobs = False
-    self._active = 0
 
-    # When quiet, never show any output.  It's a bit hacky, but reusing the
+    Does not display a leading zero for minutes if under 10 minutes. This should
-    # existing logic that delays initial output keeps the rest of the class
+    be used when displaying elapsed time in a progress indicator.
-    # clean.  Basically we set the start time to years in the future.
+    """
-    if quiet:
+    hours, mins, secs = convert_to_hms(total)
-      self._show = False
+    ret = f"{int(secs):>02d}"
-      self._start += 2**32
+    if total >= 3600:
-
+        # Show leading zeroes if over an hour.
-  def start(self, name):
+        ret = f"{mins:>02d}:{ret}"
-    self._active += 1
-    if not self._show_jobs:
-      self._show_jobs = self._active > 1
-    self.update(inc=0, msg='started ' + name)
-
-  def finish(self, name):
-    self.update(msg='finished ' + name)
-    self._active -= 1
-
-  def update(self, inc=1, msg=''):
-    self._done += inc
-
-    if _NOT_TTY or IsTrace():
-      return
-
-    if not self._show:
-      if 0.5 <= time() - self._start:
-        self._show = True
-      else:
-        return
-
-    if self._total <= 0:
-      sys.stderr.write('\r%s: %d,%s' % (
-          self._title,
-          self._done,
-          CSI_ERASE_LINE_AFTER))
-      sys.stderr.flush()
     else:
-      p = (100 * self._done) / self._total
+        ret = f"{mins}:{ret}"
-      if self._show_jobs:
+    if hours:
-        jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
+        ret = f"{hours}:{ret}"
-      else:
+    return ret
-        jobs = ''
-      sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % (
-          self._title,
-          p,
-          jobs,
-          self._done, self._units,
-          self._total, self._units,
-          ' ' if msg else '', msg,
-          CSI_ERASE_LINE_AFTER,
-          '\n' if self._print_newline else ''))
-      sys.stderr.flush()
 
-  def end(self):
-    if _NOT_TTY or IsTrace() or not self._show:
-      return
 
-    duration = duration_str(time() - self._start)
+def jobs_str(total):
-    if self._total <= 0:
+    return f"{total} job{'s' if total > 1 else ''}"
-      sys.stderr.write('\r%s: %d, done in %s%s\n' % (
+
-          self._title,
+
-          self._done,
+class Progress:
-          duration,
+    def __init__(
-          CSI_ERASE_LINE_AFTER))
+        self,
-      sys.stderr.flush()
+        title,
-    else:
+        total=0,
-      p = (100 * self._done) / self._total
+        units="",
-      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % (
+        delay=True,
-          self._title,
+        quiet=False,
-          p,
+        show_elapsed=False,
-          self._done, self._units,
+        elide=False,
-          self._total, self._units,
+    ):
-          duration,
+        self._title = title
-          CSI_ERASE_LINE_AFTER))
+        self._total = total
-      sys.stderr.flush()
+        self._done = 0
+        self._start = time.time()
+        self._show = not delay
+        self._units = units
+        self._elide = elide and _TTY
+        self._quiet = quiet
+        self._ended = False
+
+        # Only show the active jobs section if we run more than one in parallel.
+        self._show_jobs = False
+        self._active = 0
+
+        # Save the last message for displaying on refresh.
+        self._last_msg = None
+        self._show_elapsed = show_elapsed
+        self._update_event = _threading.Event()
+        self._update_thread = _threading.Thread(
+            target=self._update_loop,
+        )
+        self._update_thread.daemon = True
+
+        if not quiet and show_elapsed:
+            self._update_thread.start()
+
+    def update_total(self, new_total):
+        """Updates the total if the new total is larger."""
+        if new_total > self._total:
+            self._total = new_total
+
+    def _update_loop(self):
+        while True:
+            self.update(inc=0)
+            if self._update_event.wait(timeout=1):
+                return
+
+    def _write(self, s):
+        s = "\r" + s
+        if self._elide:
+            col = os.get_terminal_size(_STDERR.fileno()).columns
+            if len(s) > col:
+                s = s[: col - 1] + ".."
+        _STDERR.write(s)
+        _STDERR.flush()
+
+    def start(self, name):
+        self._active += 1
+        if not self._show_jobs:
+            self._show_jobs = self._active > 1
+        self.update(inc=0, msg="started " + name)
+
+    def finish(self, name):
+        self.update(msg="finished " + name)
+        self._active -= 1
+
+    def update(self, inc=1, msg=None):
+        """Updates the progress indicator.
+
+        Args:
+            inc: The number of items completed.
+            msg: The message to display. If None, use the last message.
+        """
+        if self._ended:
+            return
+        self._done += inc
+        if msg is None:
+            msg = self._last_msg
+        self._last_msg = msg
+
+        if not _TTY or IsTraceToStderr() or self._quiet:
+            return
+
+        elapsed_sec = time.time() - self._start
+        if not self._show:
+            if 0.5 <= elapsed_sec:
+                self._show = True
+            else:
+                return
+
+        if self._total <= 0:
+            self._write(
+                "%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER)
+            )
+        else:
+            p = (100 * self._done) / self._total
+            if self._show_jobs:
+                jobs = f"[{jobs_str(self._active)}] "
+            else:
+                jobs = ""
+            if self._show_elapsed:
+                elapsed = f" {elapsed_str(elapsed_sec)} |"
+            else:
+                elapsed = ""
+            self._write(
+                "%s: %2d%% %s(%d%s/%d%s)%s %s%s"
+                % (
+                    self._title,
+                    p,
+                    jobs,
+                    self._done,
+                    self._units,
+                    self._total,
+                    self._units,
+                    elapsed,
+                    msg,
+                    CSI_ERASE_LINE_AFTER,
+                )
+            )
+
+    def display_message(self, msg):
+        """Clears the current progress line and prints a message above it.
+
+        The progress bar is then redrawn on the next line.
+        """
+        if not _TTY or IsTraceToStderr() or self._quiet:
+            return
+
+        # Erase the current line, print the message with a newline,
+        # and then immediately redraw the progress bar on the new line.
+        _STDERR.write("\r" + CSI_ERASE_LINE)
+        _STDERR.write(msg + "\n")
+        _STDERR.flush()
+        self.update(inc=0)
+
+    def end(self):
+        if self._ended:
+            return
+        self._ended = True
+
+        self._update_event.set()
+        if not _TTY or IsTraceToStderr() or self._quiet:
+            return
+
+        duration = duration_str(time.time() - self._start)
+        if self._total <= 0:
+            self._write(
+                "%s: %d, done in %s%s\n"
+                % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER)
+            )
+        else:
+            p = (100 * self._done) / self._total
+            self._write(
+                "%s: %3d%% (%d%s/%d%s), done in %s%s\n"
+                % (
+                    self._title,
+                    p,
+                    self._done,
+                    self._units,
+                    self._total,
+                    self._units,
+                    duration,
+                    CSI_ERASE_LINE_AFTER,
+                )
+            )

pyproject.toml

@@ -0,0 +1,46 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[tool.black]
+line-length = 80
+target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] #, 'py312'
+
+# Config file for the isort python module.
+# This is used to enforce import sorting standards.
+#
+# https://pycqa.github.io/isort/docs/configuration/options.html
+[tool.isort]
+# Be compatible with `black` since it also matches what we want.
+profile = 'black'
+
+line_length = 80
+length_sort = false
+force_single_line = true
+lines_after_imports = 2
+from_first = false
+case_sensitive = false
+force_sort_within_sections = true
+order_by_type = false
+
+# Ignore generated files.
+extend_skip_glob = '*_pb2.py'
+
+# Allow importing multiple classes on a single line from these modules.
+# https://google.github.io/styleguide/pyguide#s2.2-imports
+single_line_exclusions = ['abc', 'collections.abc', 'typing']
+
+[tool.pytest.ini_options]
+markers = """
+    skip_cq: Skip tests in the CQ. Should be rarely used!
+"""

release/check-metadata.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool to check various metadata (e.g. licensing) in source files."""
+
+import argparse
+from pathlib import Path
+import re
+import sys
+
+import util
+
+
+_FILE_HEADER_RE = re.compile(
+    r"""# Copyright \(C\) 20[0-9]{2} The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2\.0 \(the "License"\);
+# you may not use this file except in compliance with the License\.
+# You may obtain a copy of the License at
+#
+#      http://www\.apache\.org/licenses/LICENSE-2\.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied\.
+# See the License for the specific language governing permissions and
+# limitations under the License\.
+"""
+)
+
+
+def check_license(path: Path, lines: list[str]) -> bool:
+    """Check license header."""
+    # Enforce licensing on configs & scripts.
+    if not (
+        path.suffix in (".bash", ".cfg", ".ini", ".py", ".toml")
+        or lines[0] in ("#!/bin/bash", "#!/bin/sh", "#!/usr/bin/env python3")
+    ):
+        return True
+
+    # Extract the file header.
+    header_lines = []
+    for line in lines:
+        if line.startswith("#"):
+            header_lines.append(line)
+        else:
+            break
+    if not header_lines:
+        print(
+            f"error: {path.relative_to(util.TOPDIR)}: "
+            "missing file header (copyright+licensing)",
+            file=sys.stderr,
+        )
+        return False
+
+    # Skip the shebang.
+    if header_lines[0].startswith("#!"):
+        header_lines.pop(0)
+
+    # If this file is imported into the tree, then leave it be.
+    if header_lines[0] == "# DO NOT EDIT THIS FILE":
+        return True
+
+    header = "".join(f"{x}\n" for x in header_lines)
+    if not _FILE_HEADER_RE.match(header):
+        print(
+            f"error: {path.relative_to(util.TOPDIR)}: "
+            "file header incorrectly formatted",
+            file=sys.stderr,
+        )
+        print(
+            "".join(f"> {x}\n" for x in header_lines), end="", file=sys.stderr
+        )
+        return False
+
+    return True
+
+
+def check_path(opts: argparse.Namespace, path: Path) -> bool:
+    """Check a single path."""
+    try:
+        data = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return True
+    lines = data.splitlines()
+    # NB: Use list comprehension and not a generator so we run all the checks.
+    return all(
+        [
+            check_license(path, lines),
+        ]
+    )
+
+
+def check_paths(opts: argparse.Namespace, paths: list[Path]) -> bool:
+    """Check all the paths."""
+    # NB: Use list comprehension and not a generator so we check all paths.
+    return all([check_path(opts, x) for x in paths])  # noqa: C419
+
+
+def find_files(opts: argparse.Namespace) -> list[Path]:
+    """Find all the files in the source tree."""
+    result = util.run(
+        opts,
+        ["git", "ls-tree", "-r", "-z", "--name-only", "HEAD"],
+        cwd=util.TOPDIR,
+        capture_output=True,
+        encoding="utf-8",
+    )
+    return [util.TOPDIR / x for x in result.stdout.split("\0")[:-1]]
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """Get a CLI parser."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        dest="dryrun",
+        action="store_true",
+        help="show everything that would be done",
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        help="the paths to scan",
+    )
+    return parser
+
+
+def main(argv: list[str]) -> int:
+    """The main func!"""
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    paths = opts.paths
+    if not opts.paths:
+        paths = find_files(opts)
+
+    return 0 if check_paths(opts, paths) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

release/sign-launcher.py

@@ -27,109 +27,142 @@ import sys
 import util
 
 
-def sign(opts):
+assert sys.version_info >= (3, 9), "Release framework requires Python 3.9+"
-  """Sign the launcher!"""
-  output = ''
-  for key in opts.keys:
-    # We use ! at the end of the key so that gpg uses this specific key.
-    # Otherwise it uses the key as a lookup into the overall key and uses the
-    # default signing key.  i.e. It will see that KEYID_RSA is a subkey of
-    # another key, and use the primary key to sign instead of the subkey.
-    cmd = ['gpg', '--homedir', opts.gpgdir, '-u', f'{key}!', '--batch', '--yes',
-           '--armor', '--detach-sign', '--output', '-', opts.launcher]
-    ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
-    output += ret.stdout
 
-  # Save the combined signatures into one file.
+
-  with open(f'{opts.launcher}.asc', 'w', encoding='utf-8') as fp:
+def sign(opts):
-    fp.write(output)
+    """Sign the launcher!"""
+    output = ""
+    for key in opts.keys:
+        # We use ! at the end of the key so that gpg uses this specific key.
+        # Otherwise it uses the key as a lookup into the overall key and uses
+        # the default signing key.  i.e. It will see that KEYID_RSA is a subkey
+        # of another key, and use the primary key to sign instead of the subkey.
+        cmd = [
+            "gpg",
+            "--homedir",
+            opts.gpgdir,
+            "-u",
+            f"{key}!",
+            "--batch",
+            "--yes",
+            "--armor",
+            "--detach-sign",
+            "--output",
+            "-",
+            opts.launcher,
+        ]
+        ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+        output += ret.stdout
+
+    # Save the combined signatures into one file.
+    with open(f"{opts.launcher}.asc", "w", encoding="utf-8") as fp:
+        fp.write(output)
 
 
 def check(opts):
-  """Check the signature."""
+    """Check the signature."""
-  util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc'])
+    util.run(opts, ["gpg", "--verify", f"{opts.launcher}.asc"])
 
 
 def get_version(opts):
-  """Get the version from |launcher|."""
+    """Get the version from |launcher|."""
-  # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+    # Make sure we don't search $PATH when signing the "repo" file in the cwd.
-  launcher = os.path.join('.', opts.launcher)
+    launcher = os.path.join(".", opts.launcher)
-  cmd = [launcher, '--version']
+    cmd = [launcher, "--version"]
-  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
-  m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout)
+    m = re.search(r"repo launcher version ([0-9.]+)", ret.stdout)
-  if not m:
+    if not m:
-    sys.exit(f'{opts.launcher}: unable to detect repo version')
+        sys.exit(f"{opts.launcher}: unable to detect repo version")
-  return m.group(1)
+    return m.group(1)
 
 
 def postmsg(opts, version):
-  """Helpful info to show at the end for release manager."""
+    """Helpful info to show at the end for release manager."""
-  print(f"""
+    print(
+        f"""
 Repo launcher bucket:
   gs://git-repo-downloads/
 
 You should first upload it with a specific version:
-  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gcloud storage cp --predefined-acl=publicRead {opts.launcher} gs://git-repo-downloads/repo-{version}
-  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
+  gcloud storage cp --predefined-acl=publicRead {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
 
 Then to make it the public default:
-  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
+  gcloud storage cp --predefined-acl=publicRead gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
-  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+  gcloud storage cp --predefined-acl=publicRead gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
 
 NB: If a rollback is necessary, the GS bucket archives old versions, and may be
     accessed by specifying their unique id number.
-  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gcloud storage ls --long --all-versions gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
-  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gcloud storage cp --predefined-acl=publicRead gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
-  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
+  gcloud storage cp --predefined-acl=publicRead gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
-""")
+"""  # noqa: E501
+    )
 
 
 def get_parser():
-  """Get a CLI parser."""
+    """Get a CLI parser."""
-  parser = argparse.ArgumentParser(description=__doc__)
+    parser = argparse.ArgumentParser(description=__doc__)
-  parser.add_argument('-n', '--dry-run',
+    parser.add_argument(
-                      dest='dryrun', action='store_true',
+        "-n",
-                      help='show everything that would be done')
+        "--dry-run",
-  parser.add_argument('--gpgdir',
+        dest="dryrun",
-                      default=os.path.join(util.HOMEDIR, '.gnupg', 'repo'),
+        action="store_true",
-                      help='path to dedicated gpg dir with release keys '
+        help="show everything that would be done",
-                           '(default: ~/.gnupg/repo/)')
+    )
-  parser.add_argument('--keyid', dest='keys', default=[], action='append',
+    parser.add_argument(
-                      help='alternative signing keys to use')
+        "--gpgdir",
-  parser.add_argument('launcher',
+        default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
-                      default=os.path.join(util.TOPDIR, 'repo'), nargs='?',
+        help="path to dedicated gpg dir with release keys "
-                      help='the launcher script to sign')
+        "(default: ~/.gnupg/repo/)",
-  return parser
+    )
+    parser.add_argument(
+        "--keyid",
+        dest="keys",
+        default=[],
+        action="append",
+        help="alternative signing keys to use",
+    )
+    parser.add_argument(
+        "launcher",
+        default=os.path.join(util.TOPDIR, "repo"),
+        nargs="?",
+        help="the launcher script to sign",
+    )
+    return parser
 
 
 def main(argv):
-  """The main func!"""
+    """The main func!"""
-  parser = get_parser()
+    parser = get_parser()
-  opts = parser.parse_args(argv)
+    opts = parser.parse_args(argv)
 
-  if not os.path.exists(opts.gpgdir):
+    if not os.path.exists(opts.gpgdir):
-    parser.error(f'--gpgdir does not exist: {opts.gpgdir}')
+        parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
-  if not os.path.exists(opts.launcher):
+    if not os.path.exists(opts.launcher):
-    parser.error(f'launcher does not exist: {opts.launcher}')
+        parser.error(f"launcher does not exist: {opts.launcher}")
 
-  opts.launcher = os.path.relpath(opts.launcher)
+    opts.launcher = os.path.relpath(opts.launcher)
-  print(f'Signing "{opts.launcher}" launcher script and saving to '
+    print(
-        f'"{opts.launcher}.asc"')
+        f'Signing "{opts.launcher}" launcher script and saving to '
+        f'"{opts.launcher}.asc"'
+    )
 
-  if opts.keys:
+    if opts.keys:
-    print(f'Using custom keys to sign: {" ".join(opts.keys)}')
+        print(f'Using custom keys to sign: {" ".join(opts.keys)}')
-  else:
+    else:
-    print('Using official Repo release keys to sign')
+        print("Using official Repo release keys to sign")
-    opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
+        opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
-    util.import_release_key(opts)
+        util.import_release_key(opts)
 
-  version = get_version(opts)
+    version = get_version(opts)
-  sign(opts)
+    sign(opts)
-  check(opts)
+    check(opts)
-  postmsg(opts, version)
+    postmsg(opts, version)
 
-  return 0
+    return 0
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
-  sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

release/sign-tag.py

@@ -30,51 +30,69 @@ import sys
 import util
 
 
+assert sys.version_info >= (3, 9), "Release framework requires Python 3.9+"
+
+
 # We currently sign with the old DSA key as it's been around the longest.
 # We should transition to RSA by Jun 2020, and ECC by Jun 2021.
 KEYID = util.KEYID_DSA
 
 # Regular expression to validate tag names.
-RE_VALID_TAG = r'^v([0-9]+[.])+[0-9]+$'
+RE_VALID_TAG = r"^v([0-9]+[.])+[0-9]+$"
 
 
 def sign(opts):
-  """Tag the commit & sign it!"""
+    """Tag the commit & sign it!"""
-  # We use ! at the end of the key so that gpg uses this specific key.
+    # We use ! at the end of the key so that gpg uses this specific key.
-  # Otherwise it uses the key as a lookup into the overall key and uses the
+    # Otherwise it uses the key as a lookup into the overall key and uses the
-  # default signing key.  i.e. It will see that KEYID_RSA is a subkey of
+    # default signing key.  i.e. It will see that KEYID_RSA is a subkey of
-  # another key, and use the primary key to sign instead of the subkey.
+    # another key, and use the primary key to sign instead of the subkey.
-  cmd = ['git', 'tag', '-s', opts.tag, '-u', f'{opts.key}!',
+    cmd = [
-         '-m', f'repo {opts.tag}', opts.commit]
+        "git",
+        "tag",
+        "-s",
+        opts.tag,
+        "-u",
+        f"{opts.key}!",
+        "-m",
+        f"repo {opts.tag}",
+        opts.commit,
+    ]
 
-  key = 'GNUPGHOME'
+    key = "GNUPGHOME"
-  print('+', f'export {key}="{opts.gpgdir}"')
+    print("+", f'export {key}="{opts.gpgdir}"')
-  oldvalue = os.getenv(key)
+    oldvalue = os.getenv(key)
-  os.putenv(key, opts.gpgdir)
+    os.putenv(key, opts.gpgdir)
-  util.run(opts, cmd)
+    util.run(opts, cmd)
-  if oldvalue is None:
+    if oldvalue is None:
-    os.unsetenv(key)
+        os.unsetenv(key)
-  else:
+    else:
-    os.putenv(key, oldvalue)
+        os.putenv(key, oldvalue)
 
 
 def check(opts):
-  """Check the signature."""
+    """Check the signature."""
-  util.run(opts, ['git', 'tag', '--verify', opts.tag])
+    util.run(opts, ["git", "tag", "--verify", opts.tag])
 
 
 def postmsg(opts):
-  """Helpful info to show at the end for release manager."""
+    """Helpful info to show at the end for release manager."""
-  cmd = ['git', 'rev-parse', 'remotes/origin/stable']
+    cmd = ["git", "rev-parse", "remotes/origin/stable"]
-  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
-  current_release = ret.stdout.strip()
+    current_release = ret.stdout.strip()
 
-  cmd = ['git', 'log', '--format=%h (%aN) %s', '--no-merges',
+    cmd = [
-         f'remotes/origin/stable..{opts.tag}']
+        "git",
-  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+        "log",
-  shortlog = ret.stdout.strip()
+        "--format=%h (%aN) %s",
+        "--no-merges",
+        f"remotes/origin/stable..{opts.tag}",
+    ]
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+    shortlog = ret.stdout.strip()
 
-  print(f"""
+    print(
+        f"""
 Here's the short log since the last release.
 {shortlog}
 
@@ -84,57 +102,69 @@ NB: People will start upgrading to this version immediately.
 
 To roll back a release:
   git push origin --force {current_release}:stable -n
-""")
+"""
+    )
 
 
 def get_parser():
-  """Get a CLI parser."""
+    """Get a CLI parser."""
-  parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-      description=__doc__,
+        description=__doc__,
-      formatter_class=argparse.RawDescriptionHelpFormatter)
+        formatter_class=argparse.RawDescriptionHelpFormatter,
-  parser.add_argument('-n', '--dry-run',
+    )
-                      dest='dryrun', action='store_true',
+    parser.add_argument(
-                      help='show everything that would be done')
+        "-n",
-  parser.add_argument('--gpgdir',
+        "--dry-run",
-                      default=os.path.join(util.HOMEDIR, '.gnupg', 'repo'),
+        dest="dryrun",
-                      help='path to dedicated gpg dir with release keys '
+        action="store_true",
-                           '(default: ~/.gnupg/repo/)')
+        help="show everything that would be done",
-  parser.add_argument('-f', '--force', action='store_true',
+    )
-                      help='force signing of any tag')
+    parser.add_argument(
-  parser.add_argument('--keyid', dest='key',
+        "--gpgdir",
-                      help='alternative signing key to use')
+        default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
-  parser.add_argument('tag',
+        help="path to dedicated gpg dir with release keys "
-                      help='the tag to create (e.g. "v2.0")')
+        "(default: ~/.gnupg/repo/)",
-  parser.add_argument('commit', default='HEAD', nargs='?',
+    )
-                      help='the commit to tag')
+    parser.add_argument(
-  return parser
+        "-f", "--force", action="store_true", help="force signing of any tag"
+    )
+    parser.add_argument(
+        "--keyid", dest="key", help="alternative signing key to use"
+    )
+    parser.add_argument("tag", help='the tag to create (e.g. "v2.0")')
+    parser.add_argument(
+        "commit", default="HEAD", nargs="?", help="the commit to tag"
+    )
+    return parser
 
 
 def main(argv):
-  """The main func!"""
+    """The main func!"""
-  parser = get_parser()
+    parser = get_parser()
-  opts = parser.parse_args(argv)
+    opts = parser.parse_args(argv)
 
-  if not os.path.exists(opts.gpgdir):
+    if not os.path.exists(opts.gpgdir):
-    parser.error(f'--gpgdir does not exist: {opts.gpgdir}')
+        parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
 
-  if not opts.force and not re.match(RE_VALID_TAG, opts.tag):
+    if not opts.force and not re.match(RE_VALID_TAG, opts.tag):
-    parser.error(f'tag "{opts.tag}" does not match regex "{RE_VALID_TAG}"; '
+        parser.error(
-                 'use --force to sign anyways')
+            f'tag "{opts.tag}" does not match regex "{RE_VALID_TAG}"; '
+            "use --force to sign anyways"
+        )
 
-  if opts.key:
+    if opts.key:
-    print(f'Using custom key to sign: {opts.key}')
+        print(f"Using custom key to sign: {opts.key}")
-  else:
+    else:
-    print('Using official Repo release key to sign')
+        print("Using official Repo release key to sign")
-    opts.key = KEYID
+        opts.key = KEYID
-    util.import_release_key(opts)
+        util.import_release_key(opts)
 
-  sign(opts)
+    sign(opts)
-  check(opts)
+    check(opts)
-  postmsg(opts)
+    postmsg(opts)
 
-  return 0
+    return 0
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
-  sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

release/update-hooks

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for updating hooks from their various upstreams."""
+
+import argparse
+import base64
+import json
+from pathlib import Path
+import sys
+from typing import List, Optional
+import urllib.request
+
+
+assert sys.version_info >= (3, 9), "Release framework requires Python 3.9+"
+
+
+TOPDIR = Path(__file__).resolve().parent.parent
+HOOKS_DIR = TOPDIR / "hooks"
+
+
+def update_hook_commit_msg() -> None:
+    """Update commit-msg hook from Gerrit."""
+    hook = HOOKS_DIR / "commit-msg"
+    print(
+        f"{hook.name}: Updating from https://gerrit.googlesource.com/gerrit/"
+        "+/HEAD/resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+
+    # Get the current commit.
+    url = "https://gerrit.googlesource.com/gerrit/+/HEAD?format=JSON"
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+    # Discard the xss protection.
+    data = data.split(b"\n", 1)[1]
+    data = json.loads(data)
+    commit = data["commit"]
+
+    # Fetch the data for that commit.
+    url = (
+        f"https://gerrit.googlesource.com/gerrit/+/{commit}/"
+        "resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+    with urllib.request.urlopen(f"{url}?format=TEXT") as fp:
+        data = fp.read()
+
+    # gitiles base64 encodes text data.
+    data = base64.b64decode(data)
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://gerrit.googlesource.com/gerrit/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def update_hook_pre_auto_gc() -> None:
+    """Update pre-auto-gc hook from git."""
+    hook = HOOKS_DIR / "pre-auto-gc"
+    print(
+        f"{hook.name}: Updating from https://github.com/git/git/"
+        "HEAD/contrib/hooks/pre-auto-gc-battery"
+    )
+
+    # Get the current commit.
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    url = "https://api.github.com/repos/git/git/git/refs/heads/master"
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req) as fp:
+        data = fp.read()
+    data = json.loads(data)
+
+    # Fetch the data for that commit.
+    commit = data["object"]["sha"]
+    url = (
+        f"https://raw.githubusercontent.com/git/git/{commit}/"
+        "contrib/hooks/pre-auto-gc-battery"
+    )
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://github.com/git/git/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def main(argv: Optional[List[str]] = None) -> Optional[int]:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.parse_args(argv)
+
+    update_hook_commit_msg()
+    update_hook_pre_auto_gc()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

release/update-manpages

@@ -18,85 +18,9 @@
 This is intended to be run before every official Repo release.
 """
 
-from pathlib import Path
-from functools import partial
-import argparse
-import multiprocessing
-import os
-import re
-import shutil
-import subprocess
 import sys
-import tempfile
 
-TOPDIR = Path(__file__).resolve().parent.parent
+import update_manpages
-MANDIR = TOPDIR.joinpath('man')
-
-# Load repo local modules.
-sys.path.insert(0, str(TOPDIR))
-from git_command import RepoSourceVersion
-import subcmds
-
-def worker(cmd, **kwargs):
-  subprocess.run(cmd, **kwargs)
-
-def main(argv):
-  parser = argparse.ArgumentParser(description=__doc__)
-  opts = parser.parse_args(argv)
-
-  if not shutil.which('help2man'):
-    sys.exit('Please install help2man to continue.')
-
-  # Let repo know we're generating man pages so it can avoid some dynamic
-  # behavior (like probing active number of CPUs).  We use a weird name &
-  # value to make it less likely for users to set this var themselves.
-  os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! '
-
-  # "repo branch" is an alias for "repo branches".
-  del subcmds.all_commands['branch']
-  (MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1')
-
-  version = RepoSourceVersion()
-  cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
-    '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
-    '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
-  cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
-    '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
-    '-h', '--help-all'])
-
-  with tempfile.TemporaryDirectory() as tempdir:
-    repo_dir = Path(tempdir) / '.repo'
-    repo_dir.mkdir()
-    (repo_dir / 'repo').symlink_to(TOPDIR)
-
-    # Run all cmd in parallel, and wait for them to finish.
-    with multiprocessing.Pool() as pool:
-      pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
-
-  regex = (
-      (r'(It was generated by help2man) [0-9.]+', '\g<1>.'),
-      (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
-      (r'^\.PP\nDescription', '.SH DETAILS'),
-  )
-  for tmp_path in MANDIR.glob('*.1.tmp'):
-    path = tmp_path.parent / tmp_path.stem
-    old_data = path.read_text() if path.exists() else ''
-
-    data = tmp_path.read_text()
-    tmp_path.unlink()
-
-    for pattern, replacement in regex:
-      data = re.sub(pattern, replacement, data, flags=re.M)
-
-    # If the only thing that changed was the date, don't refresh.  This avoids
-    # a lot of noise when only one file actually updates.
-    old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
-    new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
-    if old_data != new_data:
-      path.write_text(data)
 
 
-if __name__ == '__main__':
+sys.exit(update_manpages.main(sys.argv[1:]))
-  sys.exit(main(sys.argv[1:]))

release/update_manpages.py

@@ -0,0 +1,189 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for generating manual page for all repo commands.
+
+Most code lives in this module so it can be unittested.
+"""
+
+import argparse
+import functools
+import multiprocessing
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import List
+
+
+assert sys.version_info >= (3, 9), "Release framework requires Python 3.9+"
+
+
+THIS_FILE = Path(__file__).resolve()
+TOPDIR = THIS_FILE.parent.parent
+MANDIR = TOPDIR.joinpath("man")
+
+# Load repo local modules.
+sys.path.insert(0, str(TOPDIR))
+from git_command import RepoSourceVersion
+import subcmds
+
+
+def worker(cmd, **kwargs):
+    subprocess.run(cmd, **kwargs)
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """Get argument parser."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--check",
+        "--dry-run",
+        action="store_const",
+        const=True,
+        help="Check if changes are necessary; don't actually change files",
+    )
+    return parser
+
+
+def main(argv: List[str]) -> int:
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    help2man = ["help2man"]
+    cipd_help2man = TOPDIR / ".cipd_bin/bin/help2man"
+    if cipd_help2man.exists():
+        help2man = [cipd_help2man]
+    elif not shutil.which("help2man"):
+        sys.exit("Please install help2man to continue.")
+
+    # Let repo know we're generating man pages so it can avoid some dynamic
+    # behavior (like probing active number of CPUs).  We use a weird name &
+    # value to make it less likely for users to set this var themselves.
+    os.environ["_REPO_GENERATE_MANPAGES_"] = " indeed! "
+
+    # "repo branch" is an alias for "repo branches".
+    del subcmds.all_commands["branch"]
+    (MANDIR / "repo-branch.1").write_text(".so man1/repo-branches.1")
+
+    version = RepoSourceVersion()
+    cmdlist = [
+        [
+            *help2man,
+            "-N",
+            "-n",
+            f"repo {cmd} - manual page for repo {cmd}",
+            "-S",
+            f"repo {cmd}",
+            "-m",
+            "Repo Manual",
+            f"--version-string={version}",
+            "-o",
+            MANDIR.joinpath(f"repo-{cmd}.1.tmp"),
+            "./repo",
+            "-h",
+            f"help {cmd}",
+        ]
+        for cmd in subcmds.all_commands
+    ]
+    cmdlist.append(
+        [
+            *help2man,
+            "-N",
+            "-n",
+            "repository management tool built on top of git",
+            "-S",
+            "repo",
+            "-m",
+            "Repo Manual",
+            f"--version-string={version}",
+            "-o",
+            MANDIR.joinpath("repo.1.tmp"),
+            "./repo",
+            "-h",
+            "--help-all",
+        ]
+    )
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir = Path(tempdir)
+        repo_dir = tempdir / ".repo"
+        repo_dir.mkdir()
+        (repo_dir / "repo").symlink_to(TOPDIR)
+
+        # Create a repo wrapper using the active Python executable.  We can't
+        # pass this directly to help2man as it's too simple, so insert it via
+        # shebang.
+        data = (TOPDIR / "repo").read_text(encoding="utf-8")
+        tempbin = tempdir / "repo"
+        tempbin.write_text(f"#!{sys.executable}\n" + data, encoding="utf-8")
+        tempbin.chmod(0o755)
+
+        # Run all cmd in parallel, and wait for them to finish.
+        with multiprocessing.Pool() as pool:
+            pool.map(
+                functools.partial(worker, cwd=tempdir, check=True), cmdlist
+            )
+
+    ret = 0
+    for tmp_path in MANDIR.glob("*.1.tmp"):
+        path = tmp_path.parent / tmp_path.stem
+        old_data = path.read_text() if path.exists() else ""
+
+        data = tmp_path.read_text()
+        tmp_path.unlink()
+
+        data = replace_regex(data)
+
+        # If the only thing that changed was the date, don't refresh.  This
+        # avoids a lot of noise when only one file actually updates.
+        old_data = re.sub(
+            r'^(\.TH REPO "1" ")([^"]+)', r"\1", old_data, flags=re.M
+        )
+        new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r"\1", data, flags=re.M)
+        if old_data != new_data:
+            if opts.check:
+                ret = 1
+                print(
+                    f"{THIS_FILE.name}: {path.name}: "
+                    "man page needs regenerating",
+                    file=sys.stderr,
+                )
+            else:
+                path.write_text(data)
+
+    return ret
+
+
+def replace_regex(data):
+    """Replace semantically null regexes in the data.
+
+    Args:
+        data: manpage text.
+
+    Returns:
+        Updated manpage text.
+    """
+    regex = (
+        (r"(It was generated by help2man) [0-9.]+", r"\g<1>."),
+        (r"^\.IP\n(.*:)\n", r".SS \g<1>\n"),
+        (r"^\.PP\nDescription", r".SH DETAILS"),
+    )
+    for pattern, replacement in regex:
+        data = re.sub(pattern, replacement, data, flags=re.M)
+    return data

release/util.py

@@ -14,60 +14,63 @@
 
 """Random utility code for release tools."""
 
-import os
+from pathlib import Path
 import re
+import shlex
 import subprocess
 import sys
 
 
-assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
+assert sys.version_info >= (3, 6), "This module requires Python 3.6+"
 
 
-TOPDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+THIS_FILE = Path(__file__).resolve()
-HOMEDIR = os.path.expanduser('~')
+TOPDIR = THIS_FILE.parent.parent
+HOMEDIR = Path("~").expanduser()
 
 
 # These are the release keys we sign with.
-KEYID_DSA = '8BB9AD793E8E6153AF0F9A4416530D5E920F5C65'
+KEYID_DSA = "8BB9AD793E8E6153AF0F9A4416530D5E920F5C65"
-KEYID_RSA = 'A34A13BE8E76BFF46A0C022DA2E75A824AAB9624'
+KEYID_RSA = "A34A13BE8E76BFF46A0C022DA2E75A824AAB9624"
-KEYID_ECC = 'E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39'
+KEYID_ECC = "E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39"
 
 
 def cmdstr(cmd):
-  """Get a nicely quoted shell command."""
+    """Get a nicely quoted shell command."""
-  ret = []
+    return " ".join(shlex.quote(x) for x in cmd)
-  for arg in cmd:
-    if not re.match(r'^[a-zA-Z0-9/_.=-]+$', arg):
-      arg = f'"{arg}"'
-    ret.append(arg)
-  return ' '.join(ret)
 
 
 def run(opts, cmd, check=True, **kwargs):
-  """Helper around subprocess.run to include logging."""
+    """Helper around subprocess.run to include logging."""
-  print('+', cmdstr(cmd))
+    print("+", cmdstr(cmd))
-  if opts.dryrun:
+    if opts.dryrun:
-    cmd = ['true', '--'] + cmd
+        cmd = ["true", "--"] + cmd
-  try:
+    try:
-    return subprocess.run(cmd, check=check, **kwargs)
+        return subprocess.run(cmd, check=check, **kwargs)
-  except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError as e:
-    print(f'aborting: {e}', file=sys.stderr)
+        print(f"aborting: {e}", file=sys.stderr)
-    sys.exit(1)
+        sys.exit(1)
 
 
 def import_release_key(opts):
-  """Import the public key of the official release repo signing key."""
+    """Import the public key of the official release repo signing key."""
-  # Extract the key from our repo launcher.
+    # Extract the key from our repo launcher.
-  launcher = getattr(opts, 'launcher', os.path.join(TOPDIR, 'repo'))
+    launcher = getattr(opts, "launcher", TOPDIR / "repo")
-  print(f'Importing keys from "{launcher}" launcher script')
+    print(f'Importing keys from "{launcher}" launcher script')
-  with open(launcher, encoding='utf-8') as fp:
+    with open(launcher, encoding="utf-8") as fp:
-    data = fp.read()
+        data = fp.read()
 
-  keys = re.findall(
+    keys = re.findall(
-      r'\n-----BEGIN PGP PUBLIC KEY BLOCK-----\n[^-]*'
+        r"\n-----BEGIN PGP PUBLIC KEY BLOCK-----\n[^-]*"
-      r'\n-----END PGP PUBLIC KEY BLOCK-----\n', data, flags=re.M)
+        r"\n-----END PGP PUBLIC KEY BLOCK-----\n",
-  run(opts, ['gpg', '--import'], input='\n'.join(keys).encode('utf-8'))
+        data,
+        flags=re.M,
+    )
+    run(opts, ["gpg", "--import"], input="\n".join(keys).encode("utf-8"))
 
-  print('Marking keys as fully trusted')
+    print("Marking keys as fully trusted")
-  run(opts, ['gpg', '--import-ownertrust'],
+    run(
-      input=f'{KEYID_DSA}:6:\n'.encode('utf-8'))
+        opts,
+        ["gpg", "--import-ownertrust"],
+        input=f"{KEYID_DSA}:6:\n".encode("utf-8"),
+    )

repo_logging.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logic for printing user-friendly logs in repo."""
+
+import logging
+
+from color import Coloring
+from error import RepoExitError
+
+
+SEPARATOR = "=" * 80
+MAX_PRINT_ERRORS = 5
+
+
+class _ConfigMock:
+    """Default coloring config to use when Logging.config is not set."""
+
+    def __init__(self):
+        self.default_values = {"color.ui": "auto"}
+
+    def GetString(self, x):
+        return self.default_values.get(x, None)
+
+
+class _LogColoring(Coloring):
+    """Coloring outstream for logging."""
+
+    def __init__(self, config):
+        super().__init__(config, "logs")
+        self.error = self.nofmt_colorer("error", fg="red")
+        self.warning = self.nofmt_colorer("warn", fg="yellow")
+        self.levelMap = {
+            "WARNING": self.warning,
+            "ERROR": self.error,
+        }
+
+
+class _LogColoringFormatter(logging.Formatter):
+    """Coloring formatter for logging."""
+
+    def __init__(self, config=None, *args, **kwargs):
+        self.config = config if config else _ConfigMock()
+        self.colorer = _LogColoring(self.config)
+        super().__init__(*args, **kwargs)
+
+    def format(self, record):
+        """Formats |record| with color."""
+        msg = super().format(record)
+        colorer = self.colorer.levelMap.get(record.levelname)
+        return msg if not colorer else colorer(msg)
+
+
+class RepoLogger(logging.Logger):
+    """Repo Logging Module."""
+
+    def __init__(self, name: str, config=None, **kwargs):
+        super().__init__(name, **kwargs)
+        handler = logging.StreamHandler()
+        handler.setFormatter(_LogColoringFormatter(config))
+        self.addHandler(handler)
+
+    def log_aggregated_errors(self, err: RepoExitError):
+        """Print all aggregated logs."""
+        self.error(SEPARATOR)
+
+        if not err.aggregate_errors:
+            self.error("Repo command failed: %s", type(err).__name__)
+            self.error("\t%s", str(err))
+            return
+
+        self.error(
+            "Repo command failed due to the following `%s` errors:",
+            type(err).__name__,
+        )
+        self.error(
+            "\n".join(str(e) for e in err.aggregate_errors[:MAX_PRINT_ERRORS])
+        )
+
+        diff = len(err.aggregate_errors) - MAX_PRINT_ERRORS
+        if diff > 0:
+            self.error("+%d additional errors...", diff)

repo_trace.py

@@ -15,26 +15,157 @@
 """Logic for tracing repo interactions.
 
 Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+
+Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
+To also include trace outputs in stderr do `repo --trace_to_stderr ...`
 """
 
-import sys
+import contextlib
 import os
+import sys
+import tempfile
+import time
+
+import platform_utils
+
 
 # Env var to implicitly turn on tracing.
-REPO_TRACE = 'REPO_TRACE'
+REPO_TRACE = "REPO_TRACE"
 
-_TRACE = os.environ.get(REPO_TRACE) == '1'
+# Temporarily set tracing to always on unless user expicitly sets to 0.
+_TRACE = os.environ.get(REPO_TRACE) != "0"
+_TRACE_TO_STDERR = False
+_TRACE_FILE = None
+_TRACE_FILE_NAME = "TRACE_FILE"
+_MAX_SIZE = 70  # in MiB
+_NEW_COMMAND_SEP = "+++++++++++++++NEW COMMAND+++++++++++++++++++"
+
+
+def IsTraceToStderr():
+    """Whether traces are written to stderr."""
+    return _TRACE_TO_STDERR
 
 
 def IsTrace():
-  return _TRACE
+    """Whether tracing is enabled."""
+    return _TRACE
+
+
+def SetTraceToStderr():
+    """Enables tracing logging to stderr."""
+    global _TRACE_TO_STDERR
+    _TRACE_TO_STDERR = True
 
 
 def SetTrace():
-  global _TRACE
+    """Enables tracing."""
-  _TRACE = True
+    global _TRACE
+    _TRACE = True
 
 
-def Trace(fmt, *args):
+def _SetTraceFile(quiet):
-  if IsTrace():
+    """Sets the trace file location."""
-    print(fmt % args, file=sys.stderr)
+    global _TRACE_FILE
+    _TRACE_FILE = _GetTraceFile(quiet)
+
+
+class Trace(contextlib.ContextDecorator):
+    """Used to capture and save git traces."""
+
+    def _time(self):
+        """Generate nanoseconds of time in a py3.6 safe way"""
+        return int(time.time() * 1e9)
+
+    def __init__(self, fmt, *args, first_trace=False, quiet=True):
+        """Initialize the object.
+
+        Args:
+            fmt: The format string for the trace.
+            *args: Arguments to pass to formatting.
+            first_trace: Whether this is the first trace of a `repo` invocation.
+            quiet: Whether to suppress notification of trace file location.
+        """
+        if not IsTrace():
+            return
+        self._trace_msg = fmt % args
+
+        if not _TRACE_FILE:
+            _SetTraceFile(quiet)
+
+        if first_trace:
+            _ClearOldTraces()
+            self._trace_msg = f"{_NEW_COMMAND_SEP} {self._trace_msg}"
+
+    def __enter__(self):
+        if not IsTrace():
+            return self
+
+        print_msg = (
+            f"PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n"
+        )
+
+        with open(_TRACE_FILE, "a") as f:
+            print(print_msg, file=f)
+
+        if _TRACE_TO_STDERR:
+            print(print_msg, file=sys.stderr)
+
+        return self
+
+    def __exit__(self, *exc):
+        if not IsTrace():
+            return False
+
+        print_msg = (
+            f"PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n"
+        )
+
+        with open(_TRACE_FILE, "a") as f:
+            print(print_msg, file=f)
+
+        if _TRACE_TO_STDERR:
+            print(print_msg, file=sys.stderr)
+
+        return False
+
+
+def _GetTraceFile(quiet):
+    """Get the trace file or create one."""
+    # TODO: refactor to pass repodir to Trace.
+    repo_dir = os.path.dirname(os.path.dirname(__file__))
+    trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
+    if not quiet:
+        print(f"Trace outputs in {trace_file}", file=sys.stderr)
+    return trace_file
+
+
+def _ClearOldTraces():
+    """Clear the oldest commands if trace file is too big."""
+    try:
+        with open(_TRACE_FILE, errors="ignore") as f:
+            if os.path.getsize(f.name) / (1024 * 1024) <= _MAX_SIZE:
+                return
+            trace_lines = f.readlines()
+    except FileNotFoundError:
+        return
+
+    while sum(len(x) for x in trace_lines) / (1024 * 1024) > _MAX_SIZE:
+        for i, line in enumerate(trace_lines):
+            if "END:" in line and _NEW_COMMAND_SEP in line:
+                trace_lines = trace_lines[i + 1 :]
+                break
+        else:
+            # The last chunk is bigger than _MAX_SIZE, so just throw everything
+            # away.
+            trace_lines = []
+
+    while trace_lines and trace_lines[-1] == "\n":
+        trace_lines = trace_lines[:-1]
+    # Write to a temporary file with a unique name in the same filesystem
+    # before replacing the original trace file.
+    temp_dir, temp_prefix = os.path.split(_TRACE_FILE)
+    with tempfile.NamedTemporaryFile(
+        "w", dir=temp_dir, prefix=temp_prefix, delete=False
+    ) as f:
+        f.writelines(trace_lines)
+    platform_utils.rename(f.name, _TRACE_FILE)

requirements.json

@@ -46,12 +46,14 @@
 
   # Supported git versions.
   #
-  # git-1.7.2 is in Debian Squeeze.
-  # git-1.7.9 is in Ubuntu Precise.
   # git-1.9.1 is in Ubuntu Trusty.
-  # git-1.7.10 is in Debian Wheezy.
+  # git-2.1.4 is in Debian Jessie.
+  # git-2.7.4 is in Ubuntu Xenial.
+  # git-2.11.0 is in Debian Stretch.
+  # git-2.17.0 is in Ubuntu Bionic.
+  # git-2.20.1 is in Debian Buster.
   "git": {
-    "hard": [1, 7, 2],
+    "hard": [1, 9, 1],
-    "soft": [1, 9, 1]
+    "soft": [2, 7, 4]
   }
 }

run_tests

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright 2019 The Android Open Source Project
+# Copyright (C) 2019 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,49 +13,190 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Wrapper to run pytest with the right settings."""
+"""Wrapper to run linters and pytest with the right settings."""
 
-import errno
+import functools
 import os
+import shlex
 import shutil
 import subprocess
 import sys
 
 
-def find_pytest():
+# NB: While tests/* support Python >=3.6 to match requirements.json for `repo`,
-  """Try to locate a good version of pytest."""
+# the higher level runner logic does not need to be held back.
-  # If we're in a virtualenv, assume that it's provided the right pytest.
+assert sys.version_info >= (3, 9), "Test/release framework requires Python 3.9+"
-  if 'VIRTUAL_ENV' in os.environ:
-    return 'pytest'
 
-  # Use the Python 3 version if available.
-  ret = shutil.which('pytest-3')
-  if ret:
-    return ret
 
-  # Hopefully this is a Python 3 version.
+ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
-  ret = shutil.which('pytest')
-  if ret:
-    return ret
 
-  print('%s: unable to find pytest.' % (__file__,), file=sys.stderr)
+
-  print('%s: Try installing: sudo apt-get install python-pytest' % (__file__,),
+def log_cmd(cmd: str, argv: list[str]) -> None:
-        file=sys.stderr)
+    """Log a debug message to make history easier to track."""
+    print("+", cmd, shlex.join(argv), file=sys.stderr)
+
+
+@functools.lru_cache()
+def is_ci() -> bool:
+    """Whether we're running in our CI system."""
+    return os.getenv("LUCI_CQ") == "yes"
+
+
+def run_pytest(argv: list[str]) -> int:
+    """Returns the exit code from pytest."""
+    if is_ci():
+        argv = ["-m", "not skip_cq"] + argv
+
+    log_cmd("pytest", argv)
+    return subprocess.run(
+        [sys.executable, "-m", "pytest"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_pytest_py38(argv: list[str]) -> int:
+    """Returns the exit code from pytest under Python 3.8."""
+    if is_ci():
+        argv = ["-m", "not skip_cq"] + argv
+
+    log_cmd("[vpython 3.8] pytest", argv)
+    try:
+        return subprocess.run(
+            [
+                "vpython3",
+                "-vpython-spec",
+                "run_tests.vpython3.8",
+                "-m",
+                "pytest",
+            ]
+            + argv,
+            check=False,
+            cwd=ROOT_DIR,
+        ).returncode
+    except FileNotFoundError:
+        # Skip if the user doesn't have vpython from depot_tools.
+        return 0
+
+
+def run_black():
+    """Returns the exit code from black."""
+    argv = ["--version"]
+    log_cmd("black", argv)
+    subprocess.run(
+        [sys.executable, "-m", "black"] + argv,
+        check=True,
+        cwd=ROOT_DIR,
+    )
+
+    # Black by default only matches .py files.  We have to list standalone
+    # scripts manually.
+    extra_programs = [
+        "repo",
+        "run_tests",
+        "release/update-hooks",
+        "release/update-manpages",
+    ]
+    argv = ["--diff", "--check", ROOT_DIR] + extra_programs
+    log_cmd("black", argv)
+    return subprocess.run(
+        [sys.executable, "-m", "black"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_flake8():
+    """Returns the exit code from flake8."""
+    argv = ["--version"]
+    log_cmd("flake8", argv)
+    subprocess.run(
+        [sys.executable, "-m", "flake8"] + argv,
+        check=True,
+        cwd=ROOT_DIR,
+    )
+
+    argv = [ROOT_DIR]
+    log_cmd("flake8", argv)
+    return subprocess.run(
+        [sys.executable, "-m", "flake8"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_isort():
+    """Returns the exit code from isort."""
+    argv = ["--version-number"]
+    log_cmd("isort", argv)
+    subprocess.run(
+        [sys.executable, "-m", "isort"] + argv,
+        check=True,
+        cwd=ROOT_DIR,
+    )
+
+    argv = ["--check", ROOT_DIR]
+    log_cmd("isort", argv)
+    return subprocess.run(
+        [sys.executable, "-m", "isort"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_check_metadata():
+    """Returns the exit code from check-metadata."""
+    argv = []
+    log_cmd("release/check-metadata.py", argv)
+    return subprocess.run(
+        [sys.executable, "release/check-metadata.py"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_update_manpages() -> int:
+    """Returns the exit code from release/update-manpages."""
+    argv = ["--check"]
+    log_cmd("release/update-manpages", argv)
+    return subprocess.run(
+        [sys.executable, "release/update-manpages"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def cipd_bootstrap() -> None:
+    """Install packages via cipd if available."""
+    argv = ["ensure", "-root", ".cipd_bin", "-ensure-file", "cipd_manifest.txt"]
+    log_cmd("cipd", argv)
+    try:
+        subprocess.run(
+            ["cipd"] + argv,
+            check=True,
+            cwd=ROOT_DIR,
+        )
+    except FileNotFoundError:
+        # Skip if the user doesn't have cipd from depot_tools.
+        return
 
 
 def main(argv):
-  """The main entry."""
+    """The main entry."""
-  # Add the repo tree to PYTHONPATH as the tests expect to be able to import
+    cipd_bootstrap()
-  # modules directly.
+    checks = (
-  pythonpath = os.path.dirname(os.path.realpath(__file__))
+        functools.partial(run_pytest, argv),
-  oldpythonpath = os.environ.get('PYTHONPATH', None)
+        functools.partial(run_pytest_py38, argv),
-  if oldpythonpath is not None:
+        run_black,
-    pythonpath += os.pathsep + oldpythonpath
+        run_flake8,
-  os.environ['PYTHONPATH'] = pythonpath
+        run_isort,
-
+        run_check_metadata,
-  pytest = find_pytest()
+        run_update_manpages,
-  return subprocess.run([pytest] + argv, check=False).returncode
+    )
+    # Run all the tests all the time to get full feedback.  Don't exit on the
+    # first error as that makes it more difficult to iterate in the CQ.
+    return 1 if sum(c() for c in checks) else 0
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
-  sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

run_tests.vpython3

@@ -0,0 +1,135 @@
+# This is a vpython "spec" file.
+#
+# Read more about `vpython` and how to modify this file here:
+#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
+# List of available wheels:
+#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
+
+python_version: "3.11"
+
+wheel: <
+  name: "infra/python/wheels/pytest-py3"
+  version: "version:8.3.4"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/py-py2_py3"
+  version: "version:1.11.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/iniconfig-py3"
+  version: "version:1.1.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/packaging-py3"
+  version: "version:23.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pluggy-py3"
+  version: "version:1.5.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/toml-py3"
+  version: "version:0.10.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pyparsing-py3"
+  version: "version:3.0.7"
+>
+
+# Required by pytest==8.3.4 and flake8-bugbear==24.12.12
+wheel: <
+  name: "infra/python/wheels/attrs-py3"
+  version: "version:24.2.0"
+>
+
+# NB: Keep in sync with constraints.txt.
+wheel: <
+  name: "infra/python/wheels/black-py3"
+  version: "version:25.1.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/mypy-extensions-py3"
+  version: "version:0.4.3"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/tomli-py3"
+  version: "version:2.0.1"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/platformdirs-py3"
+  version: "version:2.5.2"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/pathspec-py3"
+  version: "version:0.9.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/typing-extensions-py3"
+  version: "version:4.3.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/click-py3"
+  version: "version:8.0.3"
+>
+
+wheel: <
+  name: "infra/python/wheels/flake8-py2_py3"
+  version: "version:6.0.0"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/mccabe-py2_py3"
+  version: "version:0.7.0"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/pyflakes-py2_py3"
+  version: "version:3.0.1"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/pycodestyle-py2_py3"
+  version: "version:2.10.0"
+>
+
+wheel: <
+  name: "infra/python/wheels/flake8-bugbear-py3"
+  version: "version:24.12.12"
+>
+
+wheel: <
+  name: "infra/python/wheels/flake8-comprehensions-py3"
+  version: "version:3.16.0"
+>
+
+wheel: <
+  name: "infra/python/wheels/isort-py3"
+  version: "version:5.10.1"
+>

run_tests.vpython3.8

@@ -0,0 +1,67 @@
+# This is a vpython "spec" file.
+#
+# Read more about `vpython` and how to modify this file here:
+#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
+# List of available wheels:
+#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
+
+python_version: "3.8"
+
+wheel: <
+  name: "infra/python/wheels/pytest-py3"
+  version: "version:8.3.4"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/py-py2_py3"
+  version: "version:1.11.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/iniconfig-py3"
+  version: "version:1.1.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/packaging-py3"
+  version: "version:23.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pluggy-py3"
+  version: "version:1.5.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/toml-py3"
+  version: "version:0.10.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/tomli-py3"
+  version: "version:2.1.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pyparsing-py3"
+  version: "version:3.0.7"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/attrs-py2_py3"
+  version: "version:21.4.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/exceptiongroup-py3"
+  version: "version:1.1.2"
+>

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-# Copyright 2019 The Android Open Source Project
+# Copyright (C) 2019 The Android Open Source Project
 #
-# Licensed under the Apache License, Version 2.0 (the 'License");
+# Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -16,6 +16,7 @@
 """Python packaging for repo."""
 
 import os
+
 import setuptools
 
 
@@ -23,39 +24,39 @@ TOPDIR = os.path.dirname(os.path.abspath(__file__))
 
 
 # Rip out the first intro paragraph.
-with open(os.path.join(TOPDIR, 'README.md')) as fp:
+with open(os.path.join(TOPDIR, "README.md")) as fp:
     lines = fp.read().splitlines()[2:]
-    end = lines.index('')
+    end = lines.index("")
-    long_description = ' '.join(lines[0:end])
+    long_description = " ".join(lines[0:end])
 
 
 # https://packaging.python.org/tutorials/packaging-projects/
 setuptools.setup(
-    name='repo',
+    name="repo",
-    version='2',
+    version="2",
-    maintainer='Various',
+    maintainer="Various",
-    maintainer_email='repo-discuss@googlegroups.com',
+    maintainer_email="repo-discuss@googlegroups.com",
-    description='Repo helps manage many Git repositories',
+    description="Repo helps manage many Git repositories",
     long_description=long_description,
-    long_description_content_type='text/plain',
+    long_description_content_type="text/plain",
-    url='https://gerrit.googlesource.com/git-repo/',
+    url="https://gerrit.googlesource.com/git-repo/",
     project_urls={
-        'Bug Tracker': 'https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo',
+        "Bug Tracker": "https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071",  # noqa: E501
     },
     # https://pypi.org/classifiers/
     classifiers=[
-        'Development Status :: 6 - Mature',
+        "Development Status :: 6 - Mature",
-        'Environment :: Console',
+        "Environment :: Console",
-        'Intended Audience :: Developers',
+        "Intended Audience :: Developers",
-        'License :: OSI Approved :: Apache Software License',
+        "License :: OSI Approved :: Apache Software License",
-        'Natural Language :: English',
+        "Natural Language :: English",
-        'Operating System :: MacOS :: MacOS X',
+        "Operating System :: MacOS :: MacOS X",
-        'Operating System :: Microsoft :: Windows :: Windows 10',
+        "Operating System :: Microsoft :: Windows :: Windows 10",
-        'Operating System :: POSIX :: Linux',
+        "Operating System :: POSIX :: Linux",
-        'Programming Language :: Python :: 3',
+        "Programming Language :: Python :: 3",
-        'Programming Language :: Python :: 3 :: Only',
+        "Programming Language :: Python :: 3 :: Only",
-        'Topic :: Software Development :: Version Control :: Git',
+        "Topic :: Software Development :: Version Control :: Git",
     ],
-    python_requires='>=3.6',
+    python_requires=">=3.6",
-    packages=['subcmds'],
+    packages=["subcmds"],
 )

ssh.py

@@ -24,257 +24,343 @@ import sys
 import tempfile
 import time
 
+from git_command import git
 import platform_utils
 from repo_trace import Trace
 
 
-PROXY_PATH = os.path.join(os.path.dirname(__file__), 'git_ssh')
+PROXY_PATH = os.path.join(os.path.dirname(__file__), "git_ssh")
 
 
 def _run_ssh_version():
-  """run ssh -V to display the version number"""
+    """run ssh -V to display the version number"""
-  return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()
+    return subprocess.check_output(
+        ["ssh", "-V"], stderr=subprocess.STDOUT
+    ).decode()
 
 
 def _parse_ssh_version(ver_str=None):
-  """parse a ssh version string into a tuple"""
+    """parse a ssh version string into a tuple"""
-  if ver_str is None:
+    if ver_str is None:
-    ver_str = _run_ssh_version()
+        ver_str = _run_ssh_version()
-  m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
+    m = re.match(r"^OpenSSH_([0-9.]+)(p[0-9]+)?[\s,]", ver_str)
-  if m:
+    if m:
-    return tuple(int(x) for x in m.group(1).split('.'))
+        return tuple(int(x) for x in m.group(1).split("."))
-  else:
+    else:
-    return ()
+        return ()
 
 
 @functools.lru_cache(maxsize=None)
 def version():
-  """return ssh version as a tuple"""
+    """Return ssh version as a tuple.
-  try:
+
-    return _parse_ssh_version()
+    If ssh is not available, a FileNotFoundError will be raised.
-  except FileNotFoundError:
+    """
-    print('fatal: ssh not installed', file=sys.stderr)
+    try:
-    sys.exit(1)
+        return _parse_ssh_version()
-  except subprocess.CalledProcessError:
+    except subprocess.CalledProcessError as e:
-    print('fatal: unable to detect ssh version', file=sys.stderr)
+        print(
-    sys.exit(1)
+            "fatal: unable to detect ssh version"
+            f" (code={e.returncode}, output={e.stdout})",
+            file=sys.stderr,
+        )
+        sys.exit(1)
 
 
-URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_SCP = re.compile(r"^([^@:]*@?[^:/]{1,}):")
-URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
+URI_ALL = re.compile(r"^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/")
 
 
 class ProxyManager:
-  """Manage various ssh clients & masters that we spawn.
+    """Manage various ssh clients & masters that we spawn.
 
-  This will take care of sharing state between multiprocessing children, and
+    This will take care of sharing state between multiprocessing children, and
-  make sure that if we crash, we don't leak any of the ssh sessions.
+    make sure that if we crash, we don't leak any of the ssh sessions.
 
-  The code should work with a single-process scenario too, and not add too much
+    The code should work with a single-process scenario too, and not add too
-  overhead due to the manager.
+    much overhead due to the manager.
-  """
-
-  # Path to the ssh program to run which will pass our master settings along.
-  # Set here more as a convenience API.
-  proxy = PROXY_PATH
-
-  def __init__(self, manager):
-    # Protect access to the list of active masters.
-    self._lock = multiprocessing.Lock()
-    # List of active masters (pid).  These will be spawned on demand, and we are
-    # responsible for shutting them all down at the end.
-    self._masters = manager.list()
-    # Set of active masters indexed by "host:port" information.
-    # The value isn't used, but multiprocessing doesn't provide a set class.
-    self._master_keys = manager.dict()
-    # Whether ssh masters are known to be broken, so we give up entirely.
-    self._master_broken = manager.Value('b', False)
-    # List of active ssh sesssions.  Clients will be added & removed as
-    # connections finish, so this list is just for safety & cleanup if we crash.
-    self._clients = manager.list()
-    # Path to directory for holding master sockets.
-    self._sock_path = None
-
-  def __enter__(self):
-    """Enter a new context."""
-    return self
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    """Exit a context & clean up all resources."""
-    self.close()
-
-  def add_client(self, proc):
-    """Track a new ssh session."""
-    self._clients.append(proc.pid)
-
-  def remove_client(self, proc):
-    """Remove a completed ssh session."""
-    try:
-      self._clients.remove(proc.pid)
-    except ValueError:
-      pass
-
-  def add_master(self, proc):
-    """Track a new master connection."""
-    self._masters.append(proc.pid)
-
-  def _terminate(self, procs):
-    """Kill all |procs|."""
-    for pid in procs:
-      try:
-        os.kill(pid, signal.SIGTERM)
-        os.waitpid(pid, 0)
-      except OSError:
-        pass
-
-    # The multiprocessing.list() API doesn't provide many standard list()
-    # methods, so we have to manually clear the list.
-    while True:
-      try:
-        procs.pop(0)
-      except:
-        break
-
-  def close(self):
-    """Close this active ssh session.
-
-    Kill all ssh clients & masters we created, and nuke the socket dir.
     """
-    self._terminate(self._clients)
-    self._terminate(self._masters)
 
-    d = self.sock(create=False)
+    # Path to the ssh program to run which will pass our master settings along.
-    if d:
+    # Set here more as a convenience API.
-      try:
+    proxy = PROXY_PATH
-        platform_utils.rmdir(os.path.dirname(d))
-      except OSError:
-        pass
 
-  def _open_unlocked(self, host, port=None):
+    def __init__(self, manager):
-    """Make sure a ssh master session exists for |host| & |port|.
+        # Protect access to the list of active masters.
+        self._lock = multiprocessing.Lock()
+        # List of active masters (pid).  These will be spawned on demand, and we
+        # are responsible for shutting them all down at the end.
+        self._masters = manager.list()
+        # Set of active masters indexed by "host:port" information.
+        # The value isn't used, but multiprocessing doesn't provide a set class.
+        self._master_keys = manager.dict()
+        # Whether ssh masters are known to be broken, so we give up entirely.
+        self._master_broken = manager.Value("b", False)
+        # List of active ssh sesssions.  Clients will be added & removed as
+        # connections finish, so this list is just for safety & cleanup if we
+        # crash.
+        self._clients = manager.list()
+        # Path to directory for holding master sockets.
+        self._sock_path = None
+        # See if ssh is usable.
+        self._ssh_installed = False
 
-    If one doesn't exist already, we'll create it.
+    def __enter__(self):
+        """Enter a new context."""
+        # Check which version of ssh is available.
+        try:
+            version()
+            self._ssh_installed = True
+        except FileNotFoundError:
+            self._ssh_installed = False
 
-    We won't grab any locks, so the caller has to do that.  This helps keep the
+        return self
-    business logic of actually creating the master separate from grabbing locks.
-    """
-    # Check to see whether we already think that the master is running; if we
-    # think it's already running, return right away.
-    if port is not None:
-      key = '%s:%s' % (host, port)
-    else:
-      key = host
 
-    if key in self._master_keys:
+    def __exit__(self, exc_type, exc_value, traceback):
-      return True
+        """Exit a context & clean up all resources."""
+        self.close()
 
-    if self._master_broken.value or 'GIT_SSH' in os.environ:
+    def add_client(self, proc):
-      # Failed earlier, so don't retry.
+        """Track a new ssh session."""
-      return False
+        self._clients.append(proc.pid)
 
-    # We will make two calls to ssh; this is the common part of both calls.
+    def remove_client(self, proc):
-    command_base = ['ssh', '-o', 'ControlPath %s' % self.sock(), host]
+        """Remove a completed ssh session."""
-    if port is not None:
+        try:
-      command_base[1:1] = ['-p', str(port)]
+            self._clients.remove(proc.pid)
+        except ValueError:
+            pass
 
-    # Since the key wasn't in _master_keys, we think that master isn't running.
+    def add_master(self, proc):
-    # ...but before actually starting a master, we'll double-check.  This can
+        """Track a new master connection."""
-    # be important because we can't tell that that 'git@myhost.com' is the same
+        self._masters.append(proc.pid)
-    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
-    check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
 
-      if not isnt_running:
+    def _terminate(self, procs):
-        # Our double-check found that the master _was_ infact running.  Add to
+        """Kill all |procs|."""
-        # the list of keys.
+        for pid in procs:
+            try:
+                os.kill(pid, signal.SIGTERM)
+                os.waitpid(pid, 0)
+            except OSError:
+                pass
+
+        # The multiprocessing.list() API doesn't provide many standard list()
+        # methods, so we have to manually clear the list.
+        while True:
+            try:
+                procs.pop(0)
+            except IndexError:
+                break
+
+    def close(self):
+        """Close this active ssh session.
+
+        Kill all ssh clients & masters we created, and nuke the socket dir.
+        """
+        self._terminate(self._clients)
+        self._terminate(self._masters)
+
+        d = self.sock(create=False)
+        if d:
+            try:
+                platform_utils.rmdir(os.path.dirname(d))
+            except OSError:
+                pass
+
+    def _open_unlocked(self, host, port=None):
+        """Make sure a ssh master session exists for |host| & |port|.
+
+        If one doesn't exist already, we'll create it.
+
+        We won't grab any locks, so the caller has to do that.  This helps keep
+        the business logic of actually creating the master separate from
+        grabbing locks.
+        """
+        # Check to see whether we already think that the master is running; if
+        # we think it's already running, return right away.
+        if port is not None:
+            key = f"{host}:{port}"
+        else:
+            key = host
+
+        if key in self._master_keys:
+            return True
+
+        if self._master_broken.value or "GIT_SSH" in os.environ:
+            # Failed earlier, so don't retry.
+            return False
+
+        # We will make two calls to ssh; this is the common part of both calls.
+        command_base = ["ssh", "-o", "ControlPath %s" % self.sock(), host]
+        if port is not None:
+            command_base[1:1] = ["-p", str(port)]
+
+        # Since the key wasn't in _master_keys, we think that master isn't
+        # running... but before actually starting a master, we'll double-check.
+        # This can be important because we can't tell that that 'git@myhost.com'
+        # is the same as 'myhost.com' where "User git" is setup in the user's
+        # ~/.ssh/config file.
+        check_command = command_base + ["-O", "check"]
+        with Trace("Call to ssh (check call): %s", " ".join(check_command)):
+            try:
+                check_process = subprocess.Popen(
+                    check_command,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+                check_process.communicate()  # read output, but ignore it...
+                isnt_running = check_process.wait()
+
+                if not isnt_running:
+                    # Our double-check found that the master _was_ infact
+                    # running.  Add to the list of keys.
+                    self._master_keys[key] = True
+                    return True
+            except Exception:
+                # Ignore excpetions.  We we will fall back to the normal command
+                # and print to the log there.
+                pass
+
+        # Git protocol V2 is a new feature in git 2.18.0, made default in
+        # git 2.26.0
+        # It is faster and more efficient than V1.
+        # To enable it when using SSH, the environment variable GIT_PROTOCOL
+        # must be set in the SSH side channel when establishing the connection
+        # to the git server.
+        # See https://git-scm.com/docs/protocol-v2#_ssh_and_file_transport
+        # Normally git does this by itself. But here, where the SSH connection
+        # is established manually over ControlMaster via the repo-tool, it must
+        # be passed in explicitly instead.
+        # Based on https://git-scm.com/docs/gitprotocol-pack#_extra_parameters,
+        # GIT_PROTOCOL is considered an "Extra Parameter" and must be ignored
+        # by servers that do not understand it. This means that it is safe to
+        # set it even when connecting to older servers.
+        # It should also be safe to set the environment variable for older
+        # local git versions, since it is only part of the ssh side channel.
+        git_protocol_version = _get_git_protocol_version()
+        ssh_git_protocol_args = [
+            "-o",
+            f"SetEnv GIT_PROTOCOL=version={git_protocol_version}",
+        ]
+
+        command = (
+            command_base[:1]
+            + ["-M", "-N", *ssh_git_protocol_args]
+            + command_base[1:]
+        )
+        p = None
+        try:
+            with Trace("Call to ssh: %s", " ".join(command)):
+                p = subprocess.Popen(command)
+        except Exception as e:
+            self._master_broken.value = True
+            print(
+                "\nwarn: cannot enable ssh control master for %s:%s\n%s"
+                % (host, port, str(e)),
+                file=sys.stderr,
+            )
+            return False
+
+        time.sleep(1)
+        ssh_died = p.poll() is not None
+        if ssh_died:
+            return False
+
+        self.add_master(p)
         self._master_keys[key] = True
         return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
 
-    command = command_base[:1] + ['-M', '-N'] + command_base[1:]
+    def _open(self, host, port=None):
+        """Make sure a ssh master session exists for |host| & |port|.
+
+        If one doesn't exist already, we'll create it.
+
+        This will obtain any necessary locks to avoid inter-process races.
+        """
+        # Bail before grabbing the lock if we already know that we aren't going
+        # to try creating new masters below.
+        if sys.platform in ("win32", "cygwin"):
+            return False
+
+        # Acquire the lock.  This is needed to prevent opening multiple masters
+        # for the same host when we're running "repo sync -jN" (for N > 1) _and_
+        # the manifest <remote fetch="ssh://xyz"> specifies a different host
+        # from the one that was passed to repo init.
+        with self._lock:
+            return self._open_unlocked(host, port)
+
+    def preconnect(self, url):
+        """If |uri| will create a ssh connection, setup the ssh master for it."""  # noqa: E501
+        if not self._ssh_installed:
+            return False
+
+        m = URI_ALL.match(url)
+        if m:
+            scheme = m.group(1)
+            host = m.group(2)
+            if ":" in host:
+                host, port = host.split(":")
+            else:
+                port = None
+            if scheme in ("ssh", "git+ssh", "ssh+git"):
+                return self._open(host, port)
+            return False
+
+        m = URI_SCP.match(url)
+        if m:
+            host = m.group(1)
+            return self._open(host)
+
+        return False
+
+    def sock(self, create=True):
+        """Return the path to the ssh socket dir.
+
+        This has all the master sockets so clients can talk to them.
+        """
+        if not self._ssh_installed:
+            return None
+
+        if self._sock_path is None:
+            if not create:
+                return None
+            tmp_dir = "/tmp"
+            if not os.path.exists(tmp_dir):
+                tmp_dir = tempfile.gettempdir()
+            if version() < (6, 7):
+                tokens = "%r@%h:%p"
+            else:
+                tokens = "%C"  # hash of %l%h%p%r
+            self._sock_path = os.path.join(
+                tempfile.mkdtemp("", "ssh-", tmp_dir), "master-" + tokens
+            )
+        return self._sock_path
+
+
+@functools.lru_cache(maxsize=1)
+def _get_git_protocol_version() -> str:
+    """Return the git protocol version.
+
+    The version is found by first reading the global git config.
+    If no git config for protocol version exists, try to deduce the default
+    protocol version based on the git version.
+
+    See https://git-scm.com/docs/gitprotocol-v2 for details.
+    """
     try:
-      Trace(': %s', ' '.join(command))
+        return subprocess.check_output(
-      p = subprocess.Popen(command)
+            ["git", "config", "--get", "--global", "protocol.version"],
-    except Exception as e:
+            encoding="utf-8",
-      self._master_broken.value = True
+            stderr=subprocess.PIPE,
-      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
+        ).strip()
-            % (host, port, str(e)), file=sys.stderr)
+    except subprocess.CalledProcessError as e:
-      return False
+        if e.returncode == 1:
-
+            # Exit code 1 means that the git config key was not found.
-    time.sleep(1)
+            # Try to imitate the defaults that git would have used.
-    ssh_died = (p.poll() is not None)
+            git_version = git.version_tuple()
-    if ssh_died:
+            if git_version >= (2, 26, 0):
-      return False
+                # Since git version 2.26, protocol v2 is the default.
-
+                return "2"
-    self.add_master(p)
+            return "1"
-    self._master_keys[key] = True
+        # Other exit codes indicate error with reading the config.
-    return True
+        raise
-
-  def _open(self, host, port=None):
-    """Make sure a ssh master session exists for |host| & |port|.
-
-    If one doesn't exist already, we'll create it.
-
-    This will obtain any necessary locks to avoid inter-process races.
-    """
-    # Bail before grabbing the lock if we already know that we aren't going to
-    # try creating new masters below.
-    if sys.platform in ('win32', 'cygwin'):
-      return False
-
-    # Acquire the lock.  This is needed to prevent opening multiple masters for
-    # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
-    # manifest <remote fetch="ssh://xyz"> specifies a different host from the
-    # one that was passed to repo init.
-    with self._lock:
-      return self._open_unlocked(host, port)
-
-  def preconnect(self, url):
-    """If |uri| will create a ssh connection, setup the ssh master for it."""
-    m = URI_ALL.match(url)
-    if m:
-      scheme = m.group(1)
-      host = m.group(2)
-      if ':' in host:
-        host, port = host.split(':')
-      else:
-        port = None
-      if scheme in ('ssh', 'git+ssh', 'ssh+git'):
-        return self._open(host, port)
-      return False
-
-    m = URI_SCP.match(url)
-    if m:
-      host = m.group(1)
-      return self._open(host)
-
-    return False
-
-  def sock(self, create=True):
-    """Return the path to the ssh socket dir.
-
-    This has all the master sockets so clients can talk to them.
-    """
-    if self._sock_path is None:
-      if not create:
-        return None
-      tmp_dir = '/tmp'
-      if not os.path.exists(tmp_dir):
-        tmp_dir = tempfile.gettempdir()
-      if version() < (6, 7):
-        tokens = '%r@%h:%p'
-      else:
-        tokens = '%C'  # hash of %l%h%p%r
-      self._sock_path = os.path.join(
-          tempfile.mkdtemp('', 'ssh-', tmp_dir),
-          'master-' + tokens)
-    return self._sock_path

subcmds/__init__.py

@@ -14,36 +14,35 @@
 
 import os
 
+
 # A mapping of the subcommand name to the class that implements it.
 all_commands = {}
+all_modules = []
 
 my_dir = os.path.dirname(__file__)
 for py in os.listdir(my_dir):
-  if py == '__init__.py':
+    if py == "__init__.py":
-    continue
+        continue
 
-  if py.endswith('.py'):
+    if py.endswith(".py"):
-    name = py[:-3]
+        name = py[:-3]
 
-    clsn = name.capitalize()
+        clsn = name.capitalize()
-    while clsn.find('_') > 0:
+        while clsn.find("_") > 0:
-      h = clsn.index('_')
+            h = clsn.index("_")
-      clsn = clsn[0:h] + clsn[h + 1:].capitalize()
+            clsn = clsn[0:h] + clsn[h + 1 :].capitalize()
 
-    mod = __import__(__name__,
+        mod = __import__(__name__, globals(), locals(), ["%s" % name])
-                     globals(),
+        mod = getattr(mod, name)
-                     locals(),
+        try:
-                     ['%s' % name])
+            cmd = getattr(mod, clsn)
-    mod = getattr(mod, name)
+        except AttributeError:
-    try:
+            raise SyntaxError(f"{__name__}/{py} does not define class {clsn}")
-      cmd = getattr(mod, clsn)
-    except AttributeError:
-      raise SyntaxError('%s/%s does not define class %s' % (
-          __name__, py, clsn))
 
-    name = name.replace('_', '-')
+        name = name.replace("_", "-")
-    cmd.NAME = name
+        cmd.NAME = name
-    all_commands[name] = cmd
+        all_commands[name] = cmd
+        all_modules.append(mod)
 
 # Add 'branch' as an alias for 'branches'.
-all_commands['branch'] = all_commands['branches']
+all_commands["branch"] = all_commands["branches"]

subcmds/abandon.py

@@ -12,20 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections import defaultdict
+import collections
 import functools
 import itertools
-import sys
 
-from command import Command, DEFAULT_LOCAL_JOBS
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
+from error import RepoError
+from error import RepoExitError
 from git_command import git
 from progress import Progress
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class AbandonError(RepoExitError):
+    """Exit error when abandon command fails."""
 
 
 class Abandon(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Permanently abandon a development branch"
+    helpSummary = "Permanently abandon a development branch"
-  helpUsage = """
+    helpUsage = """
 %prog [--all | <branchname>] [<project>...]
 
 This subcommand permanently abandons a development branch by
@@ -33,83 +43,119 @@ deleting it (and all its history) from your local repository.
 
 It is equivalent to "git branch -D <branchname>".
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('--all',
+        p.add_option(
-                 dest='all', action='store_true',
+            "--all",
-                 help='delete all branches in all projects')
+            action="store_true",
+            help="delete all branches in all projects",
+        )
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if not opt.all and not args:
+        if not opt.all and not args:
-      self.Usage()
+            self.Usage()
 
-    if not opt.all:
+        if not opt.all:
-      nb = args[0]
+            branches = args[0].split()
-      if not git.check_ref_format('heads/%s' % nb):
+            invalid_branches = [
-        self.OptionParser.error("'%s' is not a valid branch name" % nb)
+                x for x in branches if not git.check_ref_format(f"heads/{x}")
-    else:
+            ]
-      args.insert(0, "'All local branches'")
 
-  def _ExecuteOne(self, all_branches, nb, project):
+            if invalid_branches:
-    """Abandon one project."""
+                self.OptionParser.error(
-    if all_branches:
+                    f"{invalid_branches} are not valid branch names"
-      branches = project.GetBranches()
+                )
-    else:
-      branches = [nb]
-
-    ret = {}
-    for name in branches:
-      status = project.AbandonBranch(name)
-      if status is not None:
-        ret[name] = status
-    return (ret, project)
-
-  def Execute(self, opt, args):
-    nb = args[0]
-    err = defaultdict(list)
-    success = defaultdict(list)
-    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
-    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
-
-    def _ProcessResults(_pool, pm, states):
-      for (results, project) in states:
-        for branch, status in results.items():
-          if status:
-            success[branch].append(project)
-          else:
-            err[branch].append(project)
-        pm.update()
-
-    self.ExecuteInParallel(
-        opt.jobs,
-        functools.partial(self._ExecuteOne, opt.all, nb),
-        all_projects,
-        callback=_ProcessResults,
-        output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet))
-
-    width = max(itertools.chain(
-        [25], (len(x) for x in itertools.chain(success, err))))
-    if err:
-      for br in err.keys():
-        err_msg = "error: cannot abandon %s" % br
-        print(err_msg, file=sys.stderr)
-        for proj in err[br]:
-          print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
-      sys.exit(1)
-    elif not success:
-      print('error: no project has local branch(es) : %s' % nb,
-            file=sys.stderr)
-      sys.exit(1)
-    else:
-      # Everything below here is displaying status.
-      if opt.quiet:
-        return
-      print('Abandoned branches:')
-      for br in success.keys():
-        if len(all_projects) > 1 and len(all_projects) == len(success[br]):
-          result = "all project"
         else:
-          result = "%s" % (
+            args.insert(0, "'All local branches'")
-              ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
+
-        print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))
+    @classmethod
+    def _ExecuteOne(cls, all_branches, nb, project_idx):
+        """Abandon one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
+        if all_branches:
+            branches = project.GetBranches()
+        else:
+            branches = nb
+
+        ret = {}
+        errors = []
+        for name in branches:
+            status = None
+            try:
+                status = project.AbandonBranch(name)
+            except RepoError as e:
+                status = False
+                errors.append(e)
+            if status is not None:
+                ret[name] = status
+
+        return (ret, project_idx, errors)
+
+    def Execute(self, opt, args):
+        nb = args[0].split()
+        self.TryOverrideManifestWithSmartSync()
+        err = collections.defaultdict(list)
+        success = collections.defaultdict(list)
+        aggregate_errors = []
+        all_projects = self.GetProjects(
+            args[1:], all_manifests=not opt.this_manifest_only
+        )
+        _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+
+        def _ProcessResults(_pool, pm, states):
+            for results, project_idx, errors in states:
+                project = all_projects[project_idx]
+                for branch, status in results.items():
+                    if status:
+                        success[branch].append(project)
+                    else:
+                        err[branch].append(project)
+                aggregate_errors.extend(errors)
+                pm.update(msg="")
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, opt.all, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Abandon {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
+
+        width = max(
+            itertools.chain(
+                [25], (len(x) for x in itertools.chain(success, err))
+            )
+        )
+        if err:
+            for br in err.keys():
+                err_msg = "error: cannot abandon %s" % br
+                logger.error(err_msg)
+                for proj in err[br]:
+                    logger.error(" " * len(err_msg) + " | %s", _RelPath(proj))
+            raise AbandonError(aggregate_errors=aggregate_errors)
+        elif not success:
+            logger.error("error: no project has local branch(es) : %s", nb)
+            raise AbandonError(aggregate_errors=aggregate_errors)
+        else:
+            # Everything below here is displaying status.
+            if opt.quiet:
+                return
+            print("Abandoned branches:")
+            for br in success.keys():
+                if len(all_projects) > 1 and len(all_projects) == len(
+                    success[br]
+                ):
+                    result = "all project"
+                else:
+                    result = "%s" % (
+                        ("\n" + " " * width + "| ").join(
+                            _RelPath(p) for p in success[br]
+                        )
+                    )
+                print(f"{br}{' ' * (width - len(br))}| {result}\n")

subcmds/branches.py

@@ -16,55 +16,56 @@ import itertools
 import sys
 
 from color import Coloring
-from command import Command, DEFAULT_LOCAL_JOBS
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
 
 
 class BranchColoring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, 'branch')
+        Coloring.__init__(self, config, "branch")
-    self.current = self.printer('current', fg='green')
+        self.current = self.printer("current", fg="green")
-    self.local = self.printer('local')
+        self.local = self.printer("local")
-    self.notinproject = self.printer('notinproject', fg='red')
+        self.notinproject = self.printer("notinproject", fg="red")
 
 
-class BranchInfo(object):
+class BranchInfo:
-  def __init__(self, name):
+    def __init__(self, name):
-    self.name = name
+        self.name = name
-    self.current = 0
+        self.current = 0
-    self.published = 0
+        self.published = 0
-    self.published_equal = 0
+        self.published_equal = 0
-    self.projects = []
+        self.projects = []
 
-  def add(self, b):
+    def add(self, b):
-    if b.current:
+        if b.current:
-      self.current += 1
+            self.current += 1
-    if b.published:
+        if b.published:
-      self.published += 1
+            self.published += 1
-    if b.revision == b.published:
+        if b.revision == b.published:
-      self.published_equal += 1
+            self.published_equal += 1
-    self.projects.append(b)
+        self.projects.append(b)
 
-  @property
+    @property
-  def IsCurrent(self):
+    def IsCurrent(self):
-    return self.current > 0
+        return self.current > 0
 
-  @property
+    @property
-  def IsSplitCurrent(self):
+    def IsSplitCurrent(self):
-    return self.current != 0 and self.current != len(self.projects)
+        return self.current != 0 and self.current != len(self.projects)
 
-  @property
+    @property
-  def IsPublished(self):
+    def IsPublished(self):
-    return self.published > 0
+        return self.published > 0
 
-  @property
+    @property
-  def IsPublishedEqual(self):
+    def IsPublishedEqual(self):
-    return self.published_equal == len(self.projects)
+        return self.published_equal == len(self.projects)
 
 
 class Branches(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "View current topic branches"
+    helpSummary = "View current topic branches"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...]
 
 Summarizes the currently available topic branches.
@@ -95,111 +96,120 @@ the branch appears in, or does not appear in.  If no project list
 is shown, then the branch appears in all projects.
 
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def Execute(self, opt, args):
+    @classmethod
-    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+    def _ExpandProjectToBranches(cls, project_idx):
-    out = BranchColoring(self.manifest.manifestProject.config)
+        """Expands a project into a list of branch names & associated info.
-    all_branches = {}
-    project_cnt = len(projects)
 
-    def _ProcessResults(_pool, _output, results):
+        Args:
-      for name, b in itertools.chain.from_iterable(results):
+            project_idx: project.Project index
-        if name not in all_branches:
-          all_branches[name] = BranchInfo(name)
-        all_branches[name].add(b)
 
-    self.ExecuteInParallel(
+        Returns:
-        opt.jobs,
+            List[Tuple[str, git_config.Branch, int]]
-        expand_project_to_branches,
+        """
-        projects,
+        branches = []
-        callback=_ProcessResults)
+        project = cls.get_parallel_context()["projects"][project_idx]
+        for name, b in project.GetBranches().items():
+            branches.append((name, b, project_idx))
+        return branches
 
-    names = sorted(all_branches)
+    def Execute(self, opt, args):
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+        out = BranchColoring(self.manifest.manifestProject.config)
+        all_branches = {}
+        project_cnt = len(projects)
 
-    if not names:
+        def _ProcessResults(_pool, _output, results):
-      print('   (no branches)', file=sys.stderr)
+            for name, b, project_idx in itertools.chain.from_iterable(results):
-      return
+                b.project = projects[project_idx]
+                if name not in all_branches:
+                    all_branches[name] = BranchInfo(name)
+                all_branches[name].add(b)
 
-    width = 25
+        with self.ParallelContext():
-    for name in names:
+            self.get_parallel_context()["projects"] = projects
-      if width < len(name):
+            self.ExecuteInParallel(
-        width = len(name)
+                opt.jobs,
+                self._ExpandProjectToBranches,
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
 
-    for name in names:
+        names = sorted(all_branches)
-      i = all_branches[name]
-      in_cnt = len(i.projects)
 
-      if i.IsCurrent:
+        if not names:
-        current = '*'
+            print("   (no branches)", file=sys.stderr)
-        hdr = out.current
+            return
-      else:
-        current = ' '
-        hdr = out.local
 
-      if i.IsPublishedEqual:
+        width = 25
-        published = 'P'
+        for name in names:
-      elif i.IsPublished:
+            if width < len(name):
-        published = 'p'
+                width = len(name)
-      else:
-        published = ' '
 
-      hdr('%c%c %-*s' % (current, published, width, name))
+        for name in names:
-      out.write(' |')
+            i = all_branches[name]
+            in_cnt = len(i.projects)
 
-      _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+            if i.IsCurrent:
-      if in_cnt < project_cnt:
+                current = "*"
-        fmt = out.write
+                hdr = out.current
-        paths = []
-        non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
-          in_type = 'in'
-          for b in i.projects:
-            relpath = b.project.relpath
-            if not i.IsSplitCurrent or b.current:
-              paths.append(_RelPath(b.project))
             else:
-              non_cur_paths.append(_RelPath(b.project))
+                current = " "
-        else:
+                hdr = out.local
-          fmt = out.notinproject
-          in_type = 'not in'
-          have = set()
-          for b in i.projects:
-            have.add(_RelPath(b.project))
-          for p in projects:
-            if _RelPath(p) not in have:
-              paths.append(_RelPath(p))
 
-        s = ' %s %s' % (in_type, ', '.join(paths))
+            if i.IsPublishedEqual:
-        if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+                published = "P"
-          fmt = out.current if i.IsCurrent else fmt
+            elif i.IsPublished:
-          fmt(s)
+                published = "p"
-        else:
+            else:
-          fmt(' %s:' % in_type)
+                published = " "
-          fmt = out.current if i.IsCurrent else out.write
+
-          for p in paths:
+            # A branch name can contain a percent sign, so we need to escape it.
+            # Escape after f-string formatting to properly account for leading
+            # spaces.
+            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
+            out.write(" |")
+
+            _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+            if in_cnt < project_cnt:
+                fmt = out.write
+                paths = []
+                non_cur_paths = []
+                if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
+                    in_type = "in"
+                    for b in i.projects:
+                        relpath = _RelPath(b.project)
+                        if not i.IsSplitCurrent or b.current:
+                            paths.append(relpath)
+                        else:
+                            non_cur_paths.append(relpath)
+                else:
+                    fmt = out.notinproject
+                    in_type = "not in"
+                    have = set()
+                    for b in i.projects:
+                        have.add(_RelPath(b.project))
+                    for p in projects:
+                        if _RelPath(p) not in have:
+                            paths.append(_RelPath(p))
+
+                s = f" {in_type} {', '.join(paths)}"
+                if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+                    fmt = out.current if i.IsCurrent else fmt
+                    fmt(s)
+                else:
+                    fmt(" %s:" % in_type)
+                    fmt = out.current if i.IsCurrent else out.write
+                    for p in paths:
+                        out.nl()
+                        fmt(width * " " + "          %s" % p)
+                    fmt = out.write
+                    for p in non_cur_paths:
+                        out.nl()
+                        fmt(width * " " + "          %s" % p)
+            else:
+                out.write(" in all projects")
             out.nl()
-            fmt(width * ' ' + '          %s' % p)
-          fmt = out.write
-          for p in non_cur_paths:
-            out.nl()
-            fmt(width * ' ' + '          %s' % p)
-      else:
-        out.write(' in all projects')
-      out.nl()
-
-
-def expand_project_to_branches(project):
-  """Expands a project into a list of branch names & associated information.
-
-  Args:
-    project: project.Project
-
-  Returns:
-    List[Tuple[str, git_config.Branch]]
-  """
-  branches = []
-  for name, b in project.GetBranches().items():
-    b.project = project
-    branches.append((name, b))
-  return branches

subcmds/checkout.py

@@ -13,19 +13,41 @@
 # limitations under the License.
 
 import functools
-import sys
+from typing import NamedTuple
 
-from command import Command, DEFAULT_LOCAL_JOBS
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
+from error import GitError
+from error import RepoExitError
 from progress import Progress
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class CheckoutBranchResult(NamedTuple):
+    # Whether the Project is on the branch (i.e. branch exists and no errors)
+    result: bool
+    project_idx: int
+    error: Exception
+
+
+class CheckoutCommandError(RepoExitError):
+    """Exception thrown when checkout command fails."""
+
+
+class MissingBranchError(RepoExitError):
+    """Exception thrown when no project has specified branch."""
 
 
 class Checkout(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Checkout a branch for development"
+    helpSummary = "Checkout a branch for development"
-  helpUsage = """
+    helpUsage = """
 %prog <branchname> [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command checks out an existing branch that was previously
 created by 'repo start'.
 
@@ -33,43 +55,60 @@ The command is equivalent to:
 
   repo forall [<project>...] -c git checkout <branchname>
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if not args:
+        if not args:
-      self.Usage()
+            self.Usage()
 
-  def _ExecuteOne(self, nb, project):
+    @classmethod
-    """Checkout one project."""
+    def _ExecuteOne(cls, nb, project_idx):
-    return (project.CheckoutBranch(nb), project)
+        """Checkout one project."""
+        error = None
+        result = None
+        project = cls.get_parallel_context()["projects"][project_idx]
+        try:
+            result = project.CheckoutBranch(nb)
+        except GitError as e:
+            error = e
+        return CheckoutBranchResult(result, project_idx, error)
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    nb = args[0]
+        nb = args[0]
-    err = []
+        err = []
-    success = []
+        err_projects = []
-    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
+        success = []
+        all_projects = self.GetProjects(
+            args[1:], all_manifests=not opt.this_manifest_only
+        )
 
-    def _ProcessResults(_pool, pm, results):
+        def _ProcessResults(_pool, pm, results):
-      for status, project in results:
+            for result in results:
-        if status is not None:
+                project = all_projects[result.project_idx]
-          if status:
+                if result.error is not None:
-            success.append(project)
+                    err.append(result.error)
-          else:
+                    err_projects.append(project)
-            err.append(project)
+                elif result.result:
-        pm.update()
+                    success.append(project)
+                pm.update(msg="")
 
-    self.ExecuteInParallel(
+        with self.ParallelContext():
-        opt.jobs,
+            self.get_parallel_context()["projects"] = all_projects
-        functools.partial(self._ExecuteOne, nb),
+            self.ExecuteInParallel(
-        all_projects,
+                opt.jobs,
-        callback=_ProcessResults,
+                functools.partial(self._ExecuteOne, nb),
-        output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet))
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Checkout {nb}", len(all_projects), quiet=opt.quiet
+                ),
+            )
 
-    if err:
+        if err_projects:
-      for p in err:
+            for p in err_projects:
-        print("error: %s/: cannot checkout %s" % (p.relpath, nb),
+                logger.error("error: %s/: cannot checkout %s", p.relpath, nb)
-              file=sys.stderr)
+            raise CheckoutCommandError(aggregate_errors=err)
-      sys.exit(1)
+        elif not success:
-    elif not success:
+            msg = f"error: no project has branch {nb}"
-      print('error: no project has branch %s' % nb, file=sys.stderr)
+            logger.error(msg)
-      sys.exit(1)
+            raise MissingBranchError(msg)

subcmds/cherry_pick.py

@@ -14,97 +14,132 @@
 
 import re
 import sys
-from command import Command
-from git_command import GitCommand
 
-CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
+from command import Command
+from error import GitError
+from git_command import GitCommand
+from repo_logging import RepoLogger
+
+
+CHANGE_ID_RE = re.compile(r"^\s*Change-Id: I([0-9a-f]{40})\s*$")
+logger = RepoLogger(__file__)
 
 
 class CherryPick(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Cherry-pick a change."
+    helpSummary = "Cherry-pick a change."
-  helpUsage = """
+    helpUsage = """
 %prog <sha1>
 """
-  helpDescription = """
+    helpDescription = """
 '%prog' cherry-picks a change from one branch to another.
 The change id will be updated, and a reference to the old
 change id will be added.
 """
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if len(args) != 1:
+        if len(args) != 1:
-      self.Usage()
+            self.Usage()
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    reference = args[0]
+        reference = args[0]
 
-    p = GitCommand(None,
+        p = GitCommand(
-                   ['rev-parse', '--verify', reference],
+            None,
-                   capture_stdout=True,
+            ["rev-parse", "--verify", reference],
-                   capture_stderr=True)
+            capture_stdout=True,
-    if p.Wait() != 0:
+            capture_stderr=True,
-      print(p.stderr, file=sys.stderr)
+            verify_command=True,
-      sys.exit(1)
+        )
-    sha1 = p.stdout.strip()
+        try:
+            p.Wait()
+        except GitError:
+            logger.error(p.stderr)
+            raise
 
-    p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
+        sha1 = p.stdout.strip()
-    if p.Wait() != 0:
-      print("error: Failed to retrieve old commit message", file=sys.stderr)
-      sys.exit(1)
-    old_msg = self._StripHeader(p.stdout)
 
-    p = GitCommand(None,
+        p = GitCommand(
-                   ['cherry-pick', sha1],
+            None,
-                   capture_stdout=True,
+            ["cat-file", "commit", sha1],
-                   capture_stderr=True)
+            capture_stdout=True,
-    status = p.Wait()
+            verify_command=True,
+        )
 
-    print(p.stdout, file=sys.stdout)
+        try:
-    print(p.stderr, file=sys.stderr)
+            p.Wait()
+        except GitError:
+            logger.error("error: Failed to retrieve old commit message")
+            raise
 
-    if status == 0:
+        old_msg = self._StripHeader(p.stdout)
-      # The cherry-pick was applied correctly. We just need to edit the
-      # commit message.
-      new_msg = self._Reformat(old_msg, sha1)
 
-      p = GitCommand(None, ['commit', '--amend', '-F', '-'],
+        p = GitCommand(
-                     input=new_msg,
+            None,
-                     capture_stdout=True,
+            ["cherry-pick", sha1],
-                     capture_stderr=True)
+            capture_stdout=True,
-      if p.Wait() != 0:
+            capture_stderr=True,
-        print("error: Failed to update commit message", file=sys.stderr)
+            verify_command=True,
-        sys.exit(1)
+        )
 
-    else:
+        try:
-      print('NOTE: When committing (please see above) and editing the commit '
+            p.Wait()
-            'message, please remove the old Change-Id-line and add:')
+        except GitError as e:
-      print(self._GetReference(sha1), file=sys.stderr)
+            logger.error(e)
-      print(file=sys.stderr)
+            logger.warning(
+                "NOTE: When committing (please see above) and editing the "
+                "commit message, please remove the old Change-Id-line and "
+                "add:\n%s",
+                self._GetReference(sha1),
+            )
+            raise
 
-  def _IsChangeId(self, line):
+        if p.stdout:
-    return CHANGE_ID_RE.match(line)
+            print(p.stdout.strip(), file=sys.stdout)
+        if p.stderr:
+            print(p.stderr.strip(), file=sys.stderr)
 
-  def _GetReference(self, sha1):
+        # The cherry-pick was applied correctly. We just need to edit
-    return "(cherry picked from commit %s)" % sha1
+        # the commit message.
+        new_msg = self._Reformat(old_msg, sha1)
 
-  def _StripHeader(self, commit_msg):
+        p = GitCommand(
-    lines = commit_msg.splitlines()
+            None,
-    return "\n".join(lines[lines.index("") + 1:])
+            ["commit", "--amend", "-F", "-"],
+            input=new_msg,
+            capture_stdout=True,
+            capture_stderr=True,
+            verify_command=True,
+        )
+        try:
+            p.Wait()
+        except GitError:
+            logger.error("error: Failed to update commit message")
+            raise
 
-  def _Reformat(self, old_msg, sha1):
+    def _IsChangeId(self, line):
-    new_msg = []
+        return CHANGE_ID_RE.match(line)
 
-    for line in old_msg.splitlines():
+    def _GetReference(self, sha1):
-      if not self._IsChangeId(line):
+        return "(cherry picked from commit %s)" % sha1
-        new_msg.append(line)
 
-    # Add a blank line between the message and the change id/reference
+    def _StripHeader(self, commit_msg):
-    try:
+        lines = commit_msg.splitlines()
-      if new_msg[-1].strip() != "":
+        return "\n".join(lines[lines.index("") + 1 :])
-        new_msg.append("")
-    except IndexError:
-      pass
 
-    new_msg.append(self._GetReference(sha1))
+    def _Reformat(self, old_msg, sha1):
-    return "\n".join(new_msg)
+        new_msg = []
+
+        for line in old_msg.splitlines():
+            if not self._IsChangeId(line):
+                new_msg.append(line)
+
+        # Add a blank line between the message and the change id/reference.
+        try:
+            if new_msg[-1].strip() != "":
+                new_msg.append("")
+        except IndexError:
+            pass
+
+        new_msg.append(self._GetReference(sha1))
+        return "\n".join(new_msg)

subcmds/diff.py

@@ -15,58 +15,72 @@
 import functools
 import io
 
-from command import DEFAULT_LOCAL_JOBS, PagedCommand
+from command import DEFAULT_LOCAL_JOBS
+from command import PagedCommand
 
 
 class Diff(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Show changes between commit and working tree"
+    helpSummary = "Show changes between commit and working tree"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...]
 
 The -u option causes '%prog' to generate diff output with file paths
 relative to the repository root, so the output can be applied
 to the Unix 'patch' command.
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-u', '--absolute',
+        p.add_option(
-                 dest='absolute', action='store_true',
+            "-u",
-                 help='paths are relative to the repository root')
+            "--absolute",
+            action="store_true",
+            help="paths are relative to the repository root",
+        )
 
-  def _ExecuteOne(self, absolute, local, project):
+    @classmethod
-    """Obtains the diff for a specific project.
+    def _ExecuteOne(cls, absolute, local, project_idx):
+        """Obtains the diff for a specific project.
 
-    Args:
+        Args:
-      absolute: Paths are relative to the root.
+            absolute: Paths are relative to the root.
-      local: a boolean, if True, the path is relative to the local
+            local: a boolean, if True, the path is relative to the local
-             (sub)manifest.  If false, the path is relative to the
+                (sub)manifest.  If false, the path is relative to the outermost
-             outermost manifest.
+                manifest.
-      project: Project to get status of.
+            project_idx: Project index to get status of.
 
-    Returns:
+        Returns:
-      The status of the project.
+            The status of the project.
-    """
+        """
-    buf = io.StringIO()
+        buf = io.StringIO()
-    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
+        project = cls.get_parallel_context()["projects"][project_idx]
-    return (ret, buf.getvalue())
+        ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
+        return (ret, buf.getvalue())
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        all_projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
 
-    def _ProcessResults(_pool, _output, results):
+        def _ProcessResults(_pool, _output, results):
-      ret = 0
+            ret = 0
-      for (state, output) in results:
+            for state, output in results:
-        if output:
+                if output:
-          print(output, end='')
+                    print(output, end="")
-        if not state:
+                if not state:
-          ret = 1
+                    ret = 1
-      return ret
+            return ret
 
-    return self.ExecuteInParallel(
+        with self.ParallelContext():
-        opt.jobs,
+            self.get_parallel_context()["projects"] = all_projects
-        functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only),
+            return self.ExecuteInParallel(
-        all_projects,
+                opt.jobs,
-        callback=_ProcessResults,
+                functools.partial(
-        ordered=True)
+                    self._ExecuteOne, opt.absolute, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )

subcmds/diffmanifests.py

@@ -18,24 +18,24 @@ from manifest_xml import RepoClient
 
 
 class _Coloring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, "status")
+        Coloring.__init__(self, config, "status")
 
 
 class Diffmanifests(PagedCommand):
-  """ A command to see logs in projects represented by manifests
+    """A command to see logs in projects represented by manifests
 
-  This is used to see deeper differences between manifests. Where a simple
+    This is used to see deeper differences between manifests. Where a simple
-  diff would only show a diff of sha1s for example, this command will display
+    diff would only show a diff of sha1s for example, this command will display
-  the logs of the project between both sha1s, allowing user to see diff at a
+    the logs of the project between both sha1s, allowing user to see diff at a
-  deeper level.
+    deeper level.
-  """
+    """
 
-  COMMON = True
+    COMMON = True
-  helpSummary = "Manifest diff utility"
+    helpSummary = "Manifest diff utility"
-  helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
+    helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
 
-  helpDescription = """
+    helpDescription = """
 The %prog command shows differences between project revisions of manifest1 and
 manifest2. if manifest2 is not specified, current manifest.xml will be used
 instead. Both absolute and relative paths may be used for manifests. Relative
@@ -65,145 +65,198 @@ synced and their revisions won't be found.
 
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('--raw',
+        p.add_option(
-                 dest='raw', action='store_true',
+            "--raw",
-                 help='display raw diff')
+            action="store_true",
-    p.add_option('--no-color',
+            help="display raw diff",
-                 dest='color', action='store_false', default=True,
+        )
-                 help='does not display the diff in color')
+        p.add_option(
-    p.add_option('--pretty-format',
+            "--no-color",
-                 dest='pretty_format', action='store',
+            dest="color",
-                 metavar='<FORMAT>',
+            action="store_false",
-                 help='print the log using a custom git pretty format string')
+            default=True,
+            help="does not display the diff in color",
+        )
+        p.add_option(
+            "--pretty-format",
+            action="store",
+            metavar="<FORMAT>",
+            help="print the log using a custom git pretty format string",
+        )
 
-  def _printRawDiff(self, diff, pretty_format=None):
+    def _printRawDiff(self, diff, pretty_format=None, local=False):
-    for project in diff['added']:
+        _RelPath = lambda p: p.RelPath(local=local)
-      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+        for project in diff["added"]:
-      self.out.nl()
+            self.printText(f"A {_RelPath(project)} {project.revisionExpr}")
-
-    for project in diff['removed']:
-      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
-      self.out.nl()
-
-    for project, otherProject in diff['changed']:
-      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
-                                     otherProject.revisionExpr))
-      self.out.nl()
-      self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format)
-
-    for project, otherProject in diff['unreachable']:
-      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
-                                     otherProject.revisionExpr))
-      self.out.nl()
-
-  def _printDiff(self, diff, color=True, pretty_format=None):
-    if diff['added']:
-      self.out.nl()
-      self.printText('added projects : \n')
-      self.out.nl()
-      for project in diff['added']:
-        self.printProject('\t%s' % (project.relpath))
-        self.printText(' at revision ')
-        self.printRevision(project.revisionExpr)
-        self.out.nl()
-
-    if diff['removed']:
-      self.out.nl()
-      self.printText('removed projects : \n')
-      self.out.nl()
-      for project in diff['removed']:
-        self.printProject('\t%s' % (project.relpath))
-        self.printText(' at revision ')
-        self.printRevision(project.revisionExpr)
-        self.out.nl()
-
-    if diff['changed']:
-      self.out.nl()
-      self.printText('changed projects : \n')
-      self.out.nl()
-      for project, otherProject in diff['changed']:
-        self.printProject('\t%s' % (project.relpath))
-        self.printText(' changed from ')
-        self.printRevision(project.revisionExpr)
-        self.printText(' to ')
-        self.printRevision(otherProject.revisionExpr)
-        self.out.nl()
-        self._printLogs(project, otherProject, raw=False, color=color,
-                        pretty_format=pretty_format)
-        self.out.nl()
-
-    if diff['unreachable']:
-      self.out.nl()
-      self.printText('projects with unreachable revisions : \n')
-      self.out.nl()
-      for project, otherProject in diff['unreachable']:
-        self.printProject('\t%s ' % (project.relpath))
-        self.printRevision(project.revisionExpr)
-        self.printText(' or ')
-        self.printRevision(otherProject.revisionExpr)
-        self.printText(' not found')
-        self.out.nl()
-
-  def _printLogs(self, project, otherProject, raw=False, color=True,
-                 pretty_format=None):
-
-    logs = project.getAddedAndRemovedLogs(otherProject,
-                                          oneline=(pretty_format is None),
-                                          color=color,
-                                          pretty_format=pretty_format)
-    if logs['removed']:
-      removedLogs = logs['removed'].split('\n')
-      for log in removedLogs:
-        if log.strip():
-          if raw:
-            self.printText(' R ' + log)
-            self.out.nl()
-          else:
-            self.printRemoved('\t\t[-] ')
-            self.printText(log)
             self.out.nl()
 
-    if logs['added']:
+        for project in diff["removed"]:
-      addedLogs = logs['added'].split('\n')
+            self.printText(f"R {_RelPath(project)} {project.revisionExpr}")
-      for log in addedLogs:
-        if log.strip():
-          if raw:
-            self.printText(' A ' + log)
-            self.out.nl()
-          else:
-            self.printAdded('\t\t[+] ')
-            self.printText(log)
             self.out.nl()
 
-  def ValidateOptions(self, opt, args):
+        for project, otherProject in diff["changed"]:
-    if not args or len(args) > 2:
+            self.printText(
-      self.OptionParser.error('missing manifests to diff')
+                f"C {_RelPath(project)} {project.revisionExpr} "
-    if opt.this_manifest_only is False:
+                f"{otherProject.revisionExpr}"
-      raise self.OptionParser.error(
+            )
-          '`diffmanifest` only supports the current tree')
+            self.out.nl()
+            self._printLogs(
+                project,
+                otherProject,
+                raw=True,
+                color=False,
+                pretty_format=pretty_format,
+            )
 
-  def Execute(self, opt, args):
+        for project, otherProject in diff["unreachable"]:
-    self.out = _Coloring(self.client.globalConfig)
+            self.printText(
-    self.printText = self.out.nofmt_printer('text')
+                f"U {_RelPath(project)} {project.revisionExpr} "
-    if opt.color:
+                f"{otherProject.revisionExpr}"
-      self.printProject = self.out.nofmt_printer('project', attr='bold')
+            )
-      self.printAdded = self.out.nofmt_printer('green', fg='green', attr='bold')
+            self.out.nl()
-      self.printRemoved = self.out.nofmt_printer('red', fg='red', attr='bold')
-      self.printRevision = self.out.nofmt_printer('revision', fg='yellow')
-    else:
-      self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
 
-    manifest1 = RepoClient(self.repodir)
+    def _printDiff(self, diff, color=True, pretty_format=None, local=False):
-    manifest1.Override(args[0], load_local_manifests=False)
+        _RelPath = lambda p: p.RelPath(local=local)
-    if len(args) == 1:
+        if diff["added"]:
-      manifest2 = self.manifest
+            self.out.nl()
-    else:
+            self.printText("added projects : \n")
-      manifest2 = RepoClient(self.repodir)
+            self.out.nl()
-      manifest2.Override(args[1], load_local_manifests=False)
+            for project in diff["added"]:
+                self.printProject("\t%s" % (_RelPath(project)))
+                self.printText(" at revision ")
+                self.printRevision(project.revisionExpr)
+                self.out.nl()
 
-    diff = manifest1.projectsDiff(manifest2)
+        if diff["removed"]:
-    if opt.raw:
+            self.out.nl()
-      self._printRawDiff(diff, pretty_format=opt.pretty_format)
+            self.printText("removed projects : \n")
-    else:
+            self.out.nl()
-      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)
+            for project in diff["removed"]:
+                self.printProject("\t%s" % (_RelPath(project)))
+                self.printText(" at revision ")
+                self.printRevision(project.revisionExpr)
+                self.out.nl()
+
+        if diff["missing"]:
+            self.out.nl()
+            self.printText("missing projects : \n")
+            self.out.nl()
+            for project in diff["missing"]:
+                self.printProject("\t%s" % (_RelPath(project)))
+                self.printText(" at revision ")
+                self.printRevision(project.revisionExpr)
+                self.out.nl()
+
+        if diff["changed"]:
+            self.out.nl()
+            self.printText("changed projects : \n")
+            self.out.nl()
+            for project, otherProject in diff["changed"]:
+                self.printProject("\t%s" % (_RelPath(project)))
+                self.printText(" changed from ")
+                self.printRevision(project.revisionExpr)
+                self.printText(" to ")
+                self.printRevision(otherProject.revisionExpr)
+                self.out.nl()
+                self._printLogs(
+                    project,
+                    otherProject,
+                    raw=False,
+                    color=color,
+                    pretty_format=pretty_format,
+                )
+                self.out.nl()
+
+        if diff["unreachable"]:
+            self.out.nl()
+            self.printText("projects with unreachable revisions : \n")
+            self.out.nl()
+            for project, otherProject in diff["unreachable"]:
+                self.printProject("\t%s " % (_RelPath(project)))
+                self.printRevision(project.revisionExpr)
+                self.printText(" or ")
+                self.printRevision(otherProject.revisionExpr)
+                self.printText(" not found")
+                self.out.nl()
+
+    def _printLogs(
+        self, project, otherProject, raw=False, color=True, pretty_format=None
+    ):
+        logs = project.getAddedAndRemovedLogs(
+            otherProject,
+            oneline=(pretty_format is None),
+            color=color,
+            pretty_format=pretty_format,
+        )
+        if logs["removed"]:
+            removedLogs = logs["removed"].split("\n")
+            for log in removedLogs:
+                if log.strip():
+                    if raw:
+                        self.printText(" R " + log)
+                        self.out.nl()
+                    else:
+                        self.printRemoved("\t\t[-] ")
+                        self.printText(log)
+                        self.out.nl()
+
+        if logs["added"]:
+            addedLogs = logs["added"].split("\n")
+            for log in addedLogs:
+                if log.strip():
+                    if raw:
+                        self.printText(" A " + log)
+                        self.out.nl()
+                    else:
+                        self.printAdded("\t\t[+] ")
+                        self.printText(log)
+                        self.out.nl()
+
+    def ValidateOptions(self, opt, args):
+        if not args or len(args) > 2:
+            self.OptionParser.error("missing manifests to diff")
+        if opt.this_manifest_only is False:
+            raise self.OptionParser.error(
+                "`diffmanifest` only supports the current tree"
+            )
+
+    def Execute(self, opt, args):
+        self.out = _Coloring(self.client.globalConfig)
+        self.printText = self.out.nofmt_printer("text")
+        if opt.color:
+            self.printProject = self.out.nofmt_printer("project", attr="bold")
+            self.printAdded = self.out.nofmt_printer(
+                "green", fg="green", attr="bold"
+            )
+            self.printRemoved = self.out.nofmt_printer(
+                "red", fg="red", attr="bold"
+            )
+            self.printRevision = self.out.nofmt_printer("revision", fg="yellow")
+        else:
+            self.printProject = self.printAdded = self.printRemoved = (
+                self.printRevision
+            ) = self.printText
+
+        manifest1 = RepoClient(self.repodir)
+        manifest1.Override(args[0], load_local_manifests=False)
+        if len(args) == 1:
+            manifest2 = self.manifest
+        else:
+            manifest2 = RepoClient(self.repodir)
+            manifest2.Override(args[1], load_local_manifests=False)
+
+        diff = manifest1.projectsDiff(manifest2)
+        if opt.raw:
+            self._printRawDiff(
+                diff,
+                pretty_format=opt.pretty_format,
+                local=opt.this_manifest_only,
+            )
+        else:
+            self._printDiff(
+                diff,
+                color=opt.color,
+                pretty_format=opt.pretty_format,
+                local=opt.this_manifest_only,
+            )

subcmds/download.py

@@ -16,145 +16,197 @@ import re
 import sys
 
 from command import Command
-from error import GitError, NoSuchProjectError
+from error import GitError
+from error import NoSuchProjectError
+from error import RepoExitError
+from repo_logging import RepoLogger
 
-CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
+
+CHANGE_RE = re.compile(r"^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$")
+logger = RepoLogger(__file__)
+
+
+class DownloadCommandError(RepoExitError):
+    """Error raised when download command fails."""
 
 
 class Download(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Download and checkout a change"
+    helpSummary = "Download and checkout a change"
-  helpUsage = """
+    helpUsage = """
 %prog {[project] change[/patchset]}...
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command downloads a change from the review system and
 makes it available in your project's local working directory.
 If no project is specified try to use current directory as a project.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-b', '--branch',
+        p.add_option("-b", "--branch", help="create a new branch first")
-                 help='create a new branch first')
+        p.add_option(
-    p.add_option('-c', '--cherry-pick',
+            "-c",
-                 dest='cherrypick', action='store_true',
+            "--cherry-pick",
-                 help="cherry-pick instead of checkout")
+            dest="cherrypick",
-    p.add_option('-x', '--record-origin', action='store_true',
+            action="store_true",
-                 help='pass -x when cherry-picking')
+            help="cherry-pick instead of checkout",
-    p.add_option('-r', '--revert',
+        )
-                 dest='revert', action='store_true',
+        p.add_option(
-                 help="revert instead of checkout")
+            "-x",
-    p.add_option('-f', '--ff-only',
+            "--record-origin",
-                 dest='ffonly', action='store_true',
+            action="store_true",
-                 help="force fast-forward merge")
+            help="pass -x when cherry-picking",
+        )
+        p.add_option(
+            "-r",
+            "--revert",
+            action="store_true",
+            help="revert instead of checkout",
+        )
+        p.add_option(
+            "-f",
+            "--ff-only",
+            dest="ffonly",
+            action="store_true",
+            help="force fast-forward merge",
+        )
 
-  def _ParseChangeIds(self, opt, args):
+    def _ParseChangeIds(self, opt, args):
-    if not args:
+        if not args:
-      self.Usage()
+            self.Usage()
 
-    to_get = []
+        to_get = []
-    project = None
+        project = None
 
-    for a in args:
+        for a in args:
-      m = CHANGE_RE.match(a)
+            m = CHANGE_RE.match(a)
-      if m:
+            if m:
-        if not project:
+                if not project:
-          project = self.GetProjects(".")[0]
+                    project = self.GetProjects(".")[0]
-          print('Defaulting to cwd project', project.name)
+                    print("Defaulting to cwd project", project.name)
-        chg_id = int(m.group(1))
+                chg_id = int(m.group(1))
-        if m.group(2):
+                if m.group(2):
-          ps_id = int(m.group(2))
+                    ps_id = int(m.group(2))
-        else:
+                else:
-          ps_id = 1
+                    ps_id = 1
-          refs = 'refs/changes/%2.2d/%d/' % (chg_id % 100, chg_id)
+                    refs = "refs/changes/%2.2d/%d/" % (chg_id % 100, chg_id)
-          output = project._LsRemote(refs + '*')
+                    output = project._LsRemote(refs + "*")
-          if output:
+                    if output:
-            regex = refs + r'(\d+)'
+                        regex = refs + r"(\d+)"
-            rcomp = re.compile(regex, re.I)
+                        rcomp = re.compile(regex, re.I)
-            for line in output.splitlines():
+                        for line in output.splitlines():
-              match = rcomp.search(line)
+                            match = rcomp.search(line)
-              if match:
+                            if match:
-                ps_id = max(int(match.group(1)), ps_id)
+                                ps_id = max(int(match.group(1)), ps_id)
-        to_get.append((project, chg_id, ps_id))
+                to_get.append((project, chg_id, ps_id))
-      else:
+            else:
-        projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only)
+                projects = self.GetProjects(
-        if len(projects) > 1:
+                    [a], all_manifests=not opt.this_manifest_only
-          # If the cwd is one of the projects, assume they want that.
+                )
-          try:
+                if len(projects) > 1:
-            project = self.GetProjects('.')[0]
+                    # If the cwd is one of the projects, assume they want that.
-          except NoSuchProjectError:
+                    try:
-            project = None
+                        project = self.GetProjects(".")[0]
-          if project not in projects:
+                    except NoSuchProjectError:
-            print('error: %s matches too many projects; please re-run inside '
+                        project = None
-                  'the project checkout.' % (a,), file=sys.stderr)
+                    if project not in projects:
-            for project in projects:
+                        logger.error(
-              print('  %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only),
+                            "error: %s matches too many projects; please "
-                                    project.revisionExpr), file=sys.stderr)
+                            "re-run inside the project checkout.",
-            sys.exit(1)
+                            a,
-        else:
+                        )
-          project = projects[0]
+                        for project in projects:
-          print('Defaulting to cwd project', project.name)
+                            logger.error(
-    return to_get
+                                "  %s/ @ %s",
+                                project.RelPath(local=opt.this_manifest_only),
+                                project.revisionExpr,
+                            )
+                        raise NoSuchProjectError()
+                else:
+                    project = projects[0]
+                    print("Defaulting to cwd project", project.name)
+        return to_get
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if opt.record_origin:
+        if opt.record_origin:
-      if not opt.cherrypick:
+            if not opt.cherrypick:
-        self.OptionParser.error('-x only makes sense with --cherry-pick')
+                self.OptionParser.error(
+                    "-x only makes sense with --cherry-pick"
+                )
 
-      if opt.ffonly:
+            if opt.ffonly:
-        self.OptionParser.error('-x and --ff are mutually exclusive options')
+                self.OptionParser.error(
+                    "-x and --ff are mutually exclusive options"
+                )
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    for project, change_id, ps_id in self._ParseChangeIds(opt, args):
+        try:
-      dl = project.DownloadPatchSet(change_id, ps_id)
+            self._ExecuteHelper(opt, args)
-      if not dl:
+        except Exception as e:
-        print('[%s] change %d/%d not found'
+            if isinstance(e, RepoExitError):
-              % (project.name, change_id, ps_id),
+                raise e
-              file=sys.stderr)
+            raise DownloadCommandError(aggregate_errors=[e])
-        sys.exit(1)
 
-      if not opt.revert and not dl.commits:
+    def _ExecuteHelper(self, opt, args):
-        print('[%s] change %d/%d has already been merged'
+        for project, change_id, ps_id in self._ParseChangeIds(opt, args):
-              % (project.name, change_id, ps_id),
+            dl = project.DownloadPatchSet(change_id, ps_id)
-              file=sys.stderr)
-        continue
 
-      if len(dl.commits) > 1:
+            if not opt.revert and not dl.commits:
-        print('[%s] %d/%d depends on %d unmerged changes:'
+                logger.error(
-              % (project.name, change_id, ps_id, len(dl.commits)),
+                    "[%s] change %d/%d has already been merged",
-              file=sys.stderr)
+                    project.name,
-        for c in dl.commits:
+                    change_id,
-          print('  %s' % (c), file=sys.stderr)
+                    ps_id,
+                )
+                continue
 
-      if opt.cherrypick:
+            if len(dl.commits) > 1:
-        mode = 'cherry-pick'
+                logger.error(
-      elif opt.revert:
+                    "[%s] %d/%d depends on %d unmerged changes:",
-        mode = 'revert'
+                    project.name,
-      elif opt.ffonly:
+                    change_id,
-        mode = 'fast-forward merge'
+                    ps_id,
-      else:
+                    len(dl.commits),
-        mode = 'checkout'
+                )
+                for c in dl.commits:
+                    print("  %s" % (c), file=sys.stderr)
 
-      # We'll combine the branch+checkout operation, but all the rest need a
+            if opt.cherrypick:
-      # dedicated branch start.
+                mode = "cherry-pick"
-      if opt.branch and mode != 'checkout':
+            elif opt.revert:
-        project.StartBranch(opt.branch)
+                mode = "revert"
+            elif opt.ffonly:
+                mode = "fast-forward merge"
+            else:
+                mode = "checkout"
 
-      try:
+            # We'll combine the branch+checkout operation, but all the rest need
-        if opt.cherrypick:
+            # a dedicated branch start.
-          project._CherryPick(dl.commit, ffonly=opt.ffonly,
+            if opt.branch and mode != "checkout":
-                              record_origin=opt.record_origin)
+                project.StartBranch(opt.branch)
-        elif opt.revert:
-          project._Revert(dl.commit)
-        elif opt.ffonly:
-          project._FastForward(dl.commit, ffonly=True)
-        else:
-          if opt.branch:
-            project.StartBranch(opt.branch, revision=dl.commit)
-          else:
-            project._Checkout(dl.commit)
 
-      except GitError:
+            try:
-        print('[%s] Could not complete the %s of %s'
+                if opt.cherrypick:
-              % (project.name, mode, dl.commit), file=sys.stderr)
+                    project._CherryPick(
-        sys.exit(1)
+                        dl.commit,
+                        ffonly=opt.ffonly,
+                        record_origin=opt.record_origin,
+                    )
+                elif opt.revert:
+                    project._Revert(dl.commit)
+                elif opt.ffonly:
+                    project._FastForward(dl.commit, ffonly=True)
+                else:
+                    if opt.branch:
+                        project.StartBranch(opt.branch, revision=dl.commit)
+                    else:
+                        project._Checkout(dl.commit)
+
+            except GitError:
+                logger.error(
+                    "[%s] Could not complete the %s of %s",
+                    project.name,
+                    mode,
+                    dl.commit,
+                )
+                raise

subcmds/forall.py

@@ -15,39 +15,43 @@
 import errno
 import functools
 import io
-import multiprocessing
-import re
 import os
+import re
 import signal
-import sys
 import subprocess
+import sys
 
 from color import Coloring
-from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
+from command import MirrorSafeCommand
 from error import ManifestInvalidRevisionError
+from repo_logging import RepoLogger
 
+
+logger = RepoLogger(__file__)
 _CAN_COLOR = [
-    'branch',
+    "branch",
-    'diff',
+    "diff",
-    'grep',
+    "grep",
-    'log',
+    "log",
 ]
 
 
 class ForallColoring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, 'forall')
+        Coloring.__init__(self, config, "forall")
-    self.project = self.printer('project', attr='bold')
+        self.project = self.printer("project", attr="bold")
 
 
 class Forall(Command, MirrorSafeCommand):
-  COMMON = False
+    COMMON = False
-  helpSummary = "Run a shell command in each project"
+    helpSummary = "Run a shell command in each project"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
 %prog -r str1 [str2] ... -c <command> [<arg>...]
 """
-  helpDescription = """
+    helpDescription = """
 Executes the same shell command in each project.
 
 The -r option allows running the command only on projects matching
@@ -98,6 +102,12 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_UPSTREAM is the name of the upstream branch as specified in the
+manifest.
+
+REPO_DEST_BRANCH is the name of the destination branch for code review,
+as specified in the manifest.
+
 REPO_COUNT is the total number of projects being iterated.
 
 REPO_I is the current (1-based) iteration count. Can be used in
@@ -125,236 +135,283 @@ terminal and are not redirected.
 If -e is used, when a command exits unsuccessfully, '%prog' will abort
 without iterating through the remaining projects.
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  @staticmethod
+    @staticmethod
-  def _cmd_option(option, _opt_str, _value, parser):
+    def _cmd_option(option, _opt_str, _value, parser):
-    setattr(parser.values, option.dest, list(parser.rargs))
+        setattr(parser.values, option.dest, list(parser.rargs))
-    while parser.rargs:
+        while parser.rargs:
-      del parser.rargs[0]
+            del parser.rargs[0]
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-r', '--regex',
+        p.add_option(
-                 dest='regex', action='store_true',
+            "-r",
-                 help='execute the command only on projects matching regex or wildcard expression')
+            "--regex",
-    p.add_option('-i', '--inverse-regex',
+            action="store_true",
-                 dest='inverse_regex', action='store_true',
+            help="execute the command only on projects matching regex or "
-                 help='execute the command only on projects not matching regex or '
+            "wildcard expression",
-                      'wildcard expression')
+        )
-    p.add_option('-g', '--groups',
+        p.add_option(
-                 dest='groups',
+            "-i",
-                 help='execute the command only on projects matching the specified groups')
+            "--inverse-regex",
-    p.add_option('-c', '--command',
+            action="store_true",
-                 help='command (and arguments) to execute',
+            help="execute the command only on projects not matching regex or "
-                 dest='command',
+            "wildcard expression",
-                 action='callback',
+        )
-                 callback=self._cmd_option)
+        p.add_option(
-    p.add_option('-e', '--abort-on-errors',
+            "-g",
-                 dest='abort_on_errors', action='store_true',
+            "--groups",
-                 help='abort if a command exits unsuccessfully')
+            help="execute the command only on projects matching the specified "
-    p.add_option('--ignore-missing', action='store_true',
+            "groups",
-                 help='silently skip & do not exit non-zero due missing '
+        )
-                      'checkouts')
+        p.add_option(
+            "-c",
+            "--command",
+            dest="command",
+            help="command (and arguments) to execute",
+            action="callback",
+            callback=self._cmd_option,
+        )
+        p.add_option(
+            "-e",
+            "--abort-on-errors",
+            action="store_true",
+            help="abort if a command exits unsuccessfully",
+        )
+        p.add_option(
+            "--ignore-missing",
+            action="store_true",
+            help="silently skip & do not exit non-zero due missing "
+            "checkouts",
+        )
 
-    g = p.get_option_group('--quiet')
+        g = p.get_option_group("--quiet")
-    g.add_option('-p',
+        g.add_option(
-                 dest='project_header', action='store_true',
+            "-p",
-                 help='show project headers before output')
+            dest="project_header",
-    p.add_option('--interactive',
+            action="store_true",
-                 action='store_true',
+            help="show project headers before output",
-                 help='force interactive usage')
+        )
+        p.add_option(
+            "--interactive", action="store_true", help="force interactive usage"
+        )
 
-  def WantPager(self, opt):
+    def WantPager(self, opt):
-    return opt.project_header and opt.jobs == 1
+        return opt.project_header and opt.jobs == 1
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if not opt.command:
+        if not opt.command:
-      self.Usage()
+            self.Usage()
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    cmd = [opt.command[0]]
+        cmd = [opt.command[0]]
-    all_trees = not opt.this_manifest_only
+        all_trees = not opt.this_manifest_only
 
-    shell = True
+        shell = True
-    if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
+        if re.compile(r"^[a-z0-9A-Z_/\.-]+$").match(cmd[0]):
-      shell = False
+            shell = False
 
-    if shell:
+        if shell:
-      cmd.append(cmd[0])
+            cmd.append(cmd[0])
-    cmd.extend(opt.command[1:])
+        cmd.extend(opt.command[1:])
 
-    # Historically, forall operated interactively, and in serial.  If the user
+        # Historically, forall operated interactively, and in serial.  If the
-    # has selected 1 job, then default to interacive mode.
+        # user has selected 1 job, then default to interacive mode.
-    if opt.jobs == 1:
+        if opt.jobs == 1:
-      opt.interactive = True
+            opt.interactive = True
 
-    if opt.project_header \
+        if opt.project_header and not shell and cmd[0] == "git":
-            and not shell \
+            # If this is a direct git command that can enable colorized
-            and cmd[0] == 'git':
+            # output and the user prefers coloring, add --color into the
-      # If this is a direct git command that can enable colorized
+            # command line because we are going to wrap the command into
-      # output and the user prefers coloring, add --color into the
+            # a pipe and git won't know coloring should activate.
-      # command line because we are going to wrap the command into
+            #
-      # a pipe and git won't know coloring should activate.
+            for cn in cmd[1:]:
-      #
+                if not cn.startswith("-"):
-      for cn in cmd[1:]:
+                    break
-        if not cn.startswith('-'):
+            else:
-          break
+                cn = None
-      else:
+            if cn and cn in _CAN_COLOR:
-        cn = None
-      if cn and cn in _CAN_COLOR:
-        class ColorCmd(Coloring):
-          def __init__(self, config, cmd):
-            Coloring.__init__(self, config, cmd)
-        if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
-          cmd.insert(cmd.index(cn) + 1, '--color')
 
-    mirror = self.manifest.IsMirror
+                class ColorCmd(Coloring):
-    rc = 0
+                    def __init__(self, config, cmd):
+                        Coloring.__init__(self, config, cmd)
 
-    smart_sync_manifest_name = "smart_sync_override.xml"
+                if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
-    smart_sync_manifest_path = os.path.join(
+                    cmd.insert(cmd.index(cn) + 1, "--color")
-        self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
-    if os.path.isfile(smart_sync_manifest_path):
+        mirror = self.manifest.IsMirror
-      self.manifest.Override(smart_sync_manifest_path)
 
-    if opt.regex:
+        self.TryOverrideManifestWithSmartSync()
-      projects = self.FindProjects(args, all_manifests=all_trees)
-    elif opt.inverse_regex:
-      projects = self.FindProjects(args, inverse=True, all_manifests=all_trees)
-    else:
-      projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
 
-    os.environ['REPO_COUNT'] = str(len(projects))
+        if opt.regex:
+            projects = self.FindProjects(args, all_manifests=all_trees)
+        elif opt.inverse_regex:
+            projects = self.FindProjects(
+                args, inverse=True, all_manifests=all_trees
+            )
+        else:
+            projects = self.GetProjects(
+                args, groups=opt.groups, all_manifests=all_trees
+            )
 
-    try:
+        os.environ["REPO_COUNT"] = str(len(projects))
-      config = self.manifest.manifestProject.config
+
-      with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
+        def _ProcessResults(_pool, _output, results):
-        results_it = pool.imap(
+            rc = 0
-            functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config),
+            first = True
-            enumerate(projects),
+            for r, output in results:
-            chunksize=WORKER_BATCH_SIZE)
+                if output:
-        first = True
+                    if first:
-        for (r, output) in results_it:
+                        first = False
-          if output:
+                    elif opt.project_header:
-            if first:
+                        print()
-              first = False
+                    # To simplify the DoWorkWrapper, take care of automatic
-            elif opt.project_header:
+                    # newlines.
-              print()
+                    end = "\n"
-            # To simplify the DoWorkWrapper, take care of automatic newlines.
+                    if output[-1] == "\n":
-            end = '\n'
+                        end = ""
-            if output[-1] == '\n':
+                    print(output, end=end)
-              end = ''
+                rc = rc or r
-            print(output, end=end)
+                if r != 0 and opt.abort_on_errors:
-          rc = rc or r
+                    raise Exception("Aborting due to previous error")
-          if r != 0 and opt.abort_on_errors:
+            return rc
-            raise Exception('Aborting due to previous error')
+
-    except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+        try:
-      # Catch KeyboardInterrupt raised inside and outside of workers
+            config = self.manifest.manifestProject.config
-      rc = rc or errno.EINTR
+            with self.ParallelContext():
-    except Exception as e:
+                self.get_parallel_context()["projects"] = projects
-      # Catch any other exceptions raised
+                rc = self.ExecuteInParallel(
-      print('forall: unhandled error, terminating the pool: %s: %s' %
+                    opt.jobs,
-            (type(e).__name__, e),
+                    functools.partial(
-            file=sys.stderr)
+                        self.DoWorkWrapper, mirror, opt, cmd, shell, config
-      rc = rc or getattr(e, 'errno', 1)
+                    ),
-    if rc != 0:
+                    range(len(projects)),
-      sys.exit(rc)
+                    callback=_ProcessResults,
+                    ordered=True,
+                    initializer=self.InitWorker,
+                    chunksize=1,
+                )
+        except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+            # Catch KeyboardInterrupt raised inside and outside of workers
+            rc = errno.EINTR
+        except Exception as e:
+            # Catch any other exceptions raised
+            logger.error(
+                "forall: unhandled error, terminating the pool: %s: %s",
+                type(e).__name__,
+                e,
+            )
+            rc = getattr(e, "errno", 1)
+        if rc != 0:
+            sys.exit(rc)
+
+    @classmethod
+    def InitWorker(cls):
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    @classmethod
+    def DoWorkWrapper(cls, mirror, opt, cmd, shell, config, project_idx):
+        """A wrapper around the DoWork() method.
+
+        Catch the KeyboardInterrupt exceptions here and re-raise them as a
+        different, ``Exception``-based exception to stop it flooding the console
+        with stacktraces and making the parent hang indefinitely.
+
+        """
+        project = cls.get_parallel_context()["projects"][project_idx]
+        try:
+            return DoWork(project, mirror, opt, cmd, shell, project_idx, config)
+        except KeyboardInterrupt:
+            print("%s: Worker interrupted" % project.name)
+            raise WorkerKeyboardInterrupt()
 
 
 class WorkerKeyboardInterrupt(Exception):
-  """ Keyboard interrupt exception for worker processes. """
+    """Keyboard interrupt exception for worker processes."""
-
-
-def InitWorker():
-  signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-
-def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
-  """ A wrapper around the DoWork() method.
-
-  Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
-  ``Exception``-based exception to stop it flooding the console with stacktraces
-  and making the parent hang indefinitely.
-
-  """
-  cnt, project = args
-  try:
-    return DoWork(project, mirror, opt, cmd, shell, cnt, config)
-  except KeyboardInterrupt:
-    print('%s: Worker interrupted' % project.name)
-    raise WorkerKeyboardInterrupt()
 
 
 def DoWork(project, mirror, opt, cmd, shell, cnt, config):
-  env = os.environ.copy()
+    env = os.environ.copy()
 
-  def setenv(name, val):
+    def setenv(name, val):
-    if val is None:
+        if val is None:
-      val = ''
+            val = ""
-    env[name] = val
+        env[name] = val
 
-  setenv('REPO_PROJECT', project.name)
+    setenv("REPO_PROJECT", project.name)
-  setenv('REPO_OUTERPATH', project.manifest.path_prefix)
+    setenv("REPO_OUTERPATH", project.manifest.path_prefix)
-  setenv('REPO_INNERPATH', project.relpath)
+    setenv("REPO_INNERPATH", project.relpath)
-  setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only))
+    setenv("REPO_PATH", project.RelPath(local=opt.this_manifest_only))
-  setenv('REPO_REMOTE', project.remote.name)
+    setenv("REPO_REMOTE", project.remote.name)
-  try:
+    try:
-    # If we aren't in a fully synced state and we don't have the ref the manifest
+        # If we aren't in a fully synced state and we don't have the ref the
-    # wants, then this will fail.  Ignore it for the purposes of this code.
+        # manifest wants, then this will fail.  Ignore it for the purposes of
-    lrev = '' if mirror else project.GetRevisionId()
+        # this code.
-  except ManifestInvalidRevisionError:
+        lrev = "" if mirror else project.GetRevisionId()
-    lrev = ''
+    except ManifestInvalidRevisionError:
-  setenv('REPO_LREV', lrev)
+        lrev = ""
-  setenv('REPO_RREV', project.revisionExpr)
+    setenv("REPO_LREV", lrev)
-  setenv('REPO_UPSTREAM', project.upstream)
+    setenv("REPO_RREV", project.revisionExpr)
-  setenv('REPO_DEST_BRANCH', project.dest_branch)
+    setenv("REPO_UPSTREAM", project.upstream)
-  setenv('REPO_I', str(cnt + 1))
+    setenv("REPO_DEST_BRANCH", project.dest_branch)
-  for annotation in project.annotations:
+    setenv("REPO_I", str(cnt + 1))
-    setenv("REPO__%s" % (annotation.name), annotation.value)
+    for annotation in project.annotations:
+        setenv("REPO__%s" % (annotation.name), annotation.value)
 
-  if mirror:
+    if mirror:
-    setenv('GIT_DIR', project.gitdir)
+        setenv("GIT_DIR", project.gitdir)
-    cwd = project.gitdir
+        cwd = project.gitdir
-  else:
+    else:
-    cwd = project.worktree
+        cwd = project.worktree
 
-  if not os.path.exists(cwd):
+    if not os.path.exists(cwd):
-    # Allow the user to silently ignore missing checkouts so they can run on
+        # Allow the user to silently ignore missing checkouts so they can run on
-    # partial checkouts (good for infra recovery tools).
+        # partial checkouts (good for infra recovery tools).
-    if opt.ignore_missing:
+        if opt.ignore_missing:
-      return (0, '')
+            return (0, "")
 
-    output = ''
+        output = ""
-    if ((opt.project_header and opt.verbose)
+        if (opt.project_header and opt.verbose) or not opt.project_header:
-            or not opt.project_header):
+            output = "skipping %s/" % project.RelPath(
-      output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only)
+                local=opt.this_manifest_only
-    return (1, output)
+            )
+        return (1, output)
 
-  if opt.verbose:
+    if opt.verbose:
-    stderr = subprocess.STDOUT
+        stderr = subprocess.STDOUT
-  else:
+    else:
-    stderr = subprocess.DEVNULL
+        stderr = subprocess.DEVNULL
 
-  stdin = None if opt.interactive else subprocess.DEVNULL
+    stdin = None if opt.interactive else subprocess.DEVNULL
 
-  result = subprocess.run(
+    result = subprocess.run(
-      cmd, cwd=cwd, shell=shell, env=env, check=False,
+        cmd,
-      encoding='utf-8', errors='replace',
+        cwd=cwd,
-      stdin=stdin, stdout=subprocess.PIPE, stderr=stderr)
+        shell=shell,
+        env=env,
+        check=False,
+        encoding="utf-8",
+        errors="replace",
+        stdin=stdin,
+        stdout=subprocess.PIPE,
+        stderr=stderr,
+    )
 
-  output = result.stdout
+    output = result.stdout
-  if opt.project_header:
+    if opt.project_header:
-    if output:
+        if output:
-      buf = io.StringIO()
+            buf = io.StringIO()
-      out = ForallColoring(config)
+            out = ForallColoring(config)
-      out.redirect(buf)
+            out.redirect(buf)
-      if mirror:
+            if mirror:
-        project_header_path = project.name
+                project_header_path = project.name
-      else:
+            else:
-        project_header_path = project.RelPath(local=opt.this_manifest_only)
+                project_header_path = project.RelPath(
-      out.project('project %s/' % project_header_path)
+                    local=opt.this_manifest_only
-      out.nl()
+                )
-      buf.write(output)
+            out.project("project %s/" % project_header_path)
-      output = buf.getvalue()
+            out.nl()
-  return (result.returncode, output)
+            buf.write(output)
+            output = buf.getvalue()
+    return (result.returncode, output)

subcmds/gc.py

@@ -0,0 +1,312 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import List, Set
+
+from command import Command
+from git_command import git_require
+from git_command import GitCommand
+import platform_utils
+from progress import Progress
+from project import Project
+
+
+class Gc(Command):
+    COMMON = True
+    helpSummary = "Cleaning up internal repo and Git state."
+    helpUsage = """
+%prog
+"""
+
+    def _Options(self, p):
+        p.add_option(
+            "-n",
+            "--dry-run",
+            dest="dryrun",
+            default=False,
+            action="store_true",
+            help="do everything except actually delete",
+        )
+        p.add_option(
+            "-y",
+            "--yes",
+            default=False,
+            action="store_true",
+            help="answer yes to all safe prompts",
+        )
+        p.add_option(
+            "--repack",
+            default=False,
+            action="store_true",
+            help="repack all projects that use partial clone with "
+            "filter=blob:none",
+        )
+
+    def _find_git_to_delete(
+        self, to_keep: Set[str], start_dir: str
+    ) -> Set[str]:
+        """Searches no longer needed ".git" directories.
+
+        Scans the file system starting from `start_dir` and removes all
+        directories that end with ".git" that are not in the `to_keep` set.
+        """
+        to_delete = set()
+        for root, dirs, _ in platform_utils.walk(start_dir):
+            for directory in dirs:
+                if not directory.endswith(".git"):
+                    continue
+
+                path = os.path.join(root, directory)
+                if path not in to_keep:
+                    to_delete.add(path)
+
+        return to_delete
+
+    def delete_unused_projects(self, projects: List[Project], opt):
+        print(f"Scanning filesystem under {self.repodir}...")
+
+        project_paths = set()
+        project_object_paths = set()
+
+        for project in projects:
+            project_paths.add(project.gitdir)
+            project_object_paths.add(project.objdir)
+
+        to_delete = self._find_git_to_delete(
+            project_paths, os.path.join(self.repodir, "projects")
+        )
+
+        to_delete.update(
+            self._find_git_to_delete(
+                project_object_paths,
+                os.path.join(self.repodir, "project-objects"),
+            )
+        )
+
+        if not to_delete:
+            print("Nothing to clean up.")
+            return 0
+
+        print("Identified the following projects are no longer used:")
+        print("\n".join(to_delete))
+        print("")
+        if not opt.yes:
+            print(
+                "If you proceed, any local commits in those projects will be "
+                "destroyed!"
+            )
+            ask = input("Proceed? [y/N] ")
+            if ask.lower() != "y":
+                return 1
+
+        pm = Progress(
+            "Deleting",
+            len(to_delete),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for path in to_delete:
+            if opt.dryrun:
+                print(f"\nWould have deleted ${path}")
+            else:
+                tmp_path = os.path.join(
+                    os.path.dirname(path),
+                    f"to_be_deleted_{os.path.basename(path)}",
+                )
+                platform_utils.rename(path, tmp_path)
+                platform_utils.rmtree(tmp_path)
+            pm.update(msg=path)
+        pm.end()
+
+        return 0
+
+    def _generate_promisor_files(self, pack_dir: str):
+        """Generates promisor files for all pack files in the given directory.
+
+        Promisor files are empty files with the same name as the corresponding
+        pack file but with the ".promisor" extension. They are used by Git.
+        """
+        for root, _, files in platform_utils.walk(pack_dir):
+            for file in files:
+                if not file.endswith(".pack"):
+                    continue
+                with open(os.path.join(root, f"{file[:-4]}promisor"), "w"):
+                    pass
+
+    def repack_projects(self, projects: List[Project], opt):
+        repack_projects = []
+        # Find all projects eligible for repacking:
+        # - can't be shared
+        # - have a specific fetch filter
+        for project in projects:
+            if project.config.GetBoolean("extensions.preciousObjects"):
+                continue
+            if not project.clone_depth:
+                continue
+            if project.manifest.CloneFilterForDepth != "blob:none":
+                continue
+
+            repack_projects.append(project)
+
+        if opt.dryrun:
+            print(f"Would have repacked {len(repack_projects)} projects.")
+            return 0
+
+        pm = Progress(
+            "Repacking (this will take a while)",
+            len(repack_projects),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for project in repack_projects:
+            pm.update(msg=f"{project.name}")
+
+            pack_dir = os.path.join(project.gitdir, "tmp_repo_repack")
+            if os.path.isdir(pack_dir):
+                platform_utils.rmtree(pack_dir)
+            os.mkdir(pack_dir)
+
+            # Prepare workspace for repacking - remove all unreachable refs and
+            # their objects.
+            GitCommand(
+                project,
+                ["reflog", "expire", "--expire-unreachable=all"],
+                verify_command=True,
+            ).Wait()
+            pm.update(msg=f"{project.name} | gc", inc=0)
+            GitCommand(
+                project,
+                ["gc"],
+                verify_command=True,
+            ).Wait()
+
+            # Get all objects that are reachable from the remote, and pack them.
+            pm.update(msg=f"{project.name} | generating list of objects", inc=0)
+            remote_objects_cmd = GitCommand(
+                project,
+                [
+                    "rev-list",
+                    "--objects",
+                    "--missing=allow-promisor",
+                    f"--remotes={project.remote.name}",
+                    "--filter=blob:none",
+                    "--tags",
+                ],
+                capture_stdout=True,
+                verify_command=True,
+            )
+
+            # Get all local objects and pack them.
+            local_head_objects_cmd = GitCommand(
+                project,
+                [
+                    "rev-list",
+                    "--objects",
+                    "--missing=allow-promisor",
+                    "HEAD^{tree}",
+                ],
+                capture_stdout=True,
+                verify_command=True,
+            )
+            local_objects_cmd = GitCommand(
+                project,
+                [
+                    "rev-list",
+                    "--objects",
+                    "--missing=allow-promisor",
+                    "--all",
+                    "--reflog",
+                    "--indexed-objects",
+                    "--not",
+                    f"--remotes={project.remote.name}",
+                    "--tags",
+                ],
+                capture_stdout=True,
+                verify_command=True,
+            )
+
+            remote_objects_cmd.Wait()
+
+            pm.update(msg=f"{project.name} | remote repack", inc=0)
+            GitCommand(
+                project,
+                ["pack-objects", os.path.join(pack_dir, "pack")],
+                input=remote_objects_cmd.stdout,
+                capture_stderr=True,
+                capture_stdout=True,
+                verify_command=True,
+            ).Wait()
+
+            # create promisor file for each pack file
+            self._generate_promisor_files(pack_dir)
+
+            local_head_objects_cmd.Wait()
+            local_objects_cmd.Wait()
+
+            pm.update(msg=f"{project.name} | local repack", inc=0)
+            GitCommand(
+                project,
+                ["pack-objects", os.path.join(pack_dir, "pack")],
+                input=local_head_objects_cmd.stdout + local_objects_cmd.stdout,
+                capture_stderr=True,
+                capture_stdout=True,
+                verify_command=True,
+            ).Wait()
+
+            # Swap the old pack directory with the new one.
+            platform_utils.rename(
+                os.path.join(project.objdir, "objects", "pack"),
+                os.path.join(project.objdir, "objects", "pack_old"),
+            )
+            platform_utils.rename(
+                pack_dir,
+                os.path.join(project.objdir, "objects", "pack"),
+            )
+            platform_utils.rmtree(
+                os.path.join(project.objdir, "objects", "pack_old")
+            )
+
+        pm.end()
+        return 0
+
+    def Execute(self, opt, args):
+        projects: List[Project] = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+
+        # If the user specified projects, fetch the global list separately
+        # to avoid deleting untargeted projects.
+        if args:
+            all_projects = self.GetProjects(
+                [], all_manifests=not opt.this_manifest_only
+            )
+        else:
+            all_projects = projects
+
+        ret = self.delete_unused_projects(all_projects, opt)
+        if ret != 0:
+            return ret
+
+        if opt.repack:
+            git_require((2, 17, 0), fail=True, msg="--repack")
+            ret = self.repack_projects(projects, opt)
+
+        return ret

subcmds/gitc_delete.py

@@ -1,46 +0,0 @@
-# Copyright (C) 2015 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-from command import Command, GitcClientCommand
-import platform_utils
-
-
-class GitcDelete(Command, GitcClientCommand):
-  COMMON = True
-  visible_everywhere = False
-  helpSummary = "Delete a GITC Client."
-  helpUsage = """
-%prog
-"""
-  helpDescription = """
-This subcommand deletes the current GITC client, deleting the GITC manifest
-and all locally downloaded sources.
-"""
-
-  def _Options(self, p):
-    p.add_option('-f', '--force',
-                 dest='force', action='store_true',
-                 help='force the deletion (no prompt)')
-
-  def Execute(self, opt, args):
-    if not opt.force:
-      prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' %
-                self.gitc_manifest.gitc_client_name)
-      response = input(prompt).lower()
-      if not response == 'yes':
-        print('Response was not "yes"\n Exiting...')
-        sys.exit(1)
-    platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)

subcmds/gitc_init.py

@@ -1,75 +0,0 @@
-# Copyright (C) 2015 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-
-import gitc_utils
-from command import GitcAvailableCommand
-from manifest_xml import GitcManifest
-from subcmds import init
-import wrapper
-
-
-class GitcInit(init.Init, GitcAvailableCommand):
-  COMMON = True
-  MULTI_MANIFEST_SUPPORT = False
-  helpSummary = "Initialize a GITC Client."
-  helpUsage = """
-%prog [options] [client name]
-"""
-  helpDescription = """
-The '%prog' command is ran to initialize a new GITC client for use
-with the GITC file system.
-
-This command will setup the client directory, initialize repo, just
-like repo init does, and then downloads the manifest collection
-and installs it in the .repo/directory of the GITC client.
-
-Once this is done, a GITC manifest is generated by pulling the HEAD
-SHA for each project and generates the properly formatted XML file
-and installs it as .manifest in the GITC client directory.
-
-The -c argument is required to specify the GITC client name.
-
-The optional -f argument can be used to specify the manifest file to
-use for this GITC client.
-"""
-
-  def _Options(self, p):
-    super()._Options(p, gitc_init=True)
-
-  def Execute(self, opt, args):
-    gitc_client = gitc_utils.parse_clientdir(os.getcwd())
-    if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client):
-      print('fatal: Please update your repo command. See go/gitc for instructions.',
-            file=sys.stderr)
-      sys.exit(1)
-    self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
-                                   gitc_client)
-    super().Execute(opt, args)
-
-    manifest_file = self.manifest.manifestFile
-    if opt.manifest_file:
-      if not os.path.exists(opt.manifest_file):
-        print('fatal: Specified manifest file %s does not exist.' %
-              opt.manifest_file)
-        sys.exit(1)
-      manifest_file = opt.manifest_file
-
-    manifest = GitcManifest(self.repodir, gitc_client)
-    manifest.Override(manifest_file)
-    gitc_utils.generate_gitc_manifest(None, manifest)
-    print('Please run `cd %s` to view your GITC client.' %
-          os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client))

subcmds/grep.py

@@ -14,27 +14,51 @@
 
 import functools
 import sys
+from typing import NamedTuple
 
 from color import Coloring
-from command import DEFAULT_LOCAL_JOBS, PagedCommand
+from command import DEFAULT_LOCAL_JOBS
+from command import PagedCommand
 from error import GitError
+from error import InvalidArgumentsError
+from error import SilentRepoExitError
 from git_command import GitCommand
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
 
 
 class GrepColoring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, 'grep')
+        Coloring.__init__(self, config, "grep")
-    self.project = self.printer('project', attr='bold')
+        self.project = self.printer("project", attr="bold")
-    self.fail = self.printer('fail', fg='red')
+        self.fail = self.printer("fail", fg="red")
+
+
+class ExecuteOneResult(NamedTuple):
+    """Result from an execute instance."""
+
+    project_idx: int
+    rc: int
+    stdout: str
+    stderr: str
+    error: GitError
+
+
+class GrepCommandError(SilentRepoExitError):
+    """Grep command failure. Since Grep command
+    output already outputs errors ensure that
+    aggregate errors exit silently."""
 
 
 class Grep(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Print lines matching a pattern"
+    helpSummary = "Print lines matching a pattern"
-  helpUsage = """
+    helpUsage = """
 %prog {pattern | -e pattern} [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 Search for the specified patterns in all project files.
 
 # Boolean Options
@@ -62,215 +86,323 @@ contain a line that matches both expressions:
   repo grep --all-match -e NODE -e Unexpected
 
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  @staticmethod
+    @staticmethod
-  def _carry_option(_option, opt_str, value, parser):
+    def _carry_option(_option, opt_str, value, parser):
-    pt = getattr(parser.values, 'cmd_argv', None)
+        pt = getattr(parser.values, "cmd_argv", None)
-    if pt is None:
+        if pt is None:
-      pt = []
+            pt = []
-      setattr(parser.values, 'cmd_argv', pt)
+            parser.values.cmd_argv = pt
 
-    if opt_str == '-(':
+        if opt_str == "-(":
-      pt.append('(')
+            pt.append("(")
-    elif opt_str == '-)':
+        elif opt_str == "-)":
-      pt.append(')')
+            pt.append(")")
-    else:
+        else:
-      pt.append(opt_str)
+            pt.append(opt_str)
 
-    if value is not None:
+        if value is not None:
-      pt.append(value)
+            pt.append(value)
 
-  def _CommonOptions(self, p):
+    def _CommonOptions(self, p):
-    """Override common options slightly."""
+        """Override common options slightly."""
-    super()._CommonOptions(p, opt_v=False)
+        super()._CommonOptions(p, opt_v=False)
 
-  def _Options(self, p):
+    def _Options(self, p):
-    g = p.add_option_group('Sources')
+        g = p.add_option_group("Sources")
-    g.add_option('--cached',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "--cached",
-                 help='Search the index, instead of the work tree')
+            action="callback",
-    g.add_option('-r', '--revision',
+            callback=self._carry_option,
-                 dest='revision', action='append', metavar='TREEish',
+            help="Search the index, instead of the work tree",
-                 help='Search TREEish, instead of the work tree')
+        )
+        g.add_option(
+            "-r",
+            "--revision",
+            action="append",
+            metavar="TREEish",
+            help="Search TREEish, instead of the work tree",
+        )
 
-    g = p.add_option_group('Pattern')
+        g = p.add_option_group("Pattern")
-    g.add_option('-e',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "-e",
-                 metavar='PATTERN', type='str',
+            action="callback",
-                 help='Pattern to search for')
+            callback=self._carry_option,
-    g.add_option('-i', '--ignore-case',
+            metavar="PATTERN",
-                 action='callback', callback=self._carry_option,
+            type="str",
-                 help='Ignore case differences')
+            help="Pattern to search for",
-    g.add_option('-a', '--text',
+        )
-                 action='callback', callback=self._carry_option,
+        g.add_option(
-                 help="Process binary files as if they were text")
+            "-i",
-    g.add_option('-I',
+            "--ignore-case",
-                 action='callback', callback=self._carry_option,
+            action="callback",
-                 help="Don't match the pattern in binary files")
+            callback=self._carry_option,
-    g.add_option('-w', '--word-regexp',
+            help="Ignore case differences",
-                 action='callback', callback=self._carry_option,
+        )
-                 help='Match the pattern only at word boundaries')
+        g.add_option(
-    g.add_option('-v', '--invert-match',
+            "-a",
-                 action='callback', callback=self._carry_option,
+            "--text",
-                 help='Select non-matching lines')
+            action="callback",
-    g.add_option('-G', '--basic-regexp',
+            callback=self._carry_option,
-                 action='callback', callback=self._carry_option,
+            help="Process binary files as if they were text",
-                 help='Use POSIX basic regexp for patterns (default)')
+        )
-    g.add_option('-E', '--extended-regexp',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "-I",
-                 help='Use POSIX extended regexp for patterns')
+            action="callback",
-    g.add_option('-F', '--fixed-strings',
+            callback=self._carry_option,
-                 action='callback', callback=self._carry_option,
+            help="Don't match the pattern in binary files",
-                 help='Use fixed strings (not regexp) for pattern')
+        )
+        g.add_option(
+            "-w",
+            "--word-regexp",
+            action="callback",
+            callback=self._carry_option,
+            help="Match the pattern only at word boundaries",
+        )
+        g.add_option(
+            "-v",
+            "--invert-match",
+            action="callback",
+            callback=self._carry_option,
+            help="Select non-matching lines",
+        )
+        g.add_option(
+            "-G",
+            "--basic-regexp",
+            action="callback",
+            callback=self._carry_option,
+            help="Use POSIX basic regexp for patterns (default)",
+        )
+        g.add_option(
+            "-E",
+            "--extended-regexp",
+            action="callback",
+            callback=self._carry_option,
+            help="Use POSIX extended regexp for patterns",
+        )
+        g.add_option(
+            "-F",
+            "--fixed-strings",
+            action="callback",
+            callback=self._carry_option,
+            help="Use fixed strings (not regexp) for pattern",
+        )
 
-    g = p.add_option_group('Pattern Grouping')
+        g = p.add_option_group("Pattern Grouping")
-    g.add_option('--all-match',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "--all-match",
-                 help='Limit match to lines that have all patterns')
+            action="callback",
-    g.add_option('--and', '--or', '--not',
+            callback=self._carry_option,
-                 action='callback', callback=self._carry_option,
+            help="Limit match to lines that have all patterns",
-                 help='Boolean operators to combine patterns')
+        )
-    g.add_option('-(', '-)',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "--and",
-                 help='Boolean operator grouping')
+            "--or",
+            "--not",
+            action="callback",
+            callback=self._carry_option,
+            help="Boolean operators to combine patterns",
+        )
+        g.add_option(
+            "-(",
+            "-)",
+            action="callback",
+            callback=self._carry_option,
+            help="Boolean operator grouping",
+        )
 
-    g = p.add_option_group('Output')
+        g = p.add_option_group("Output")
-    g.add_option('-n',
+        g.add_option(
-                 action='callback', callback=self._carry_option,
+            "-n",
-                 help='Prefix the line number to matching lines')
+            action="callback",
-    g.add_option('-C',
+            callback=self._carry_option,
-                 action='callback', callback=self._carry_option,
+            help="Prefix the line number to matching lines",
-                 metavar='CONTEXT', type='str',
+        )
-                 help='Show CONTEXT lines around match')
+        g.add_option(
-    g.add_option('-B',
+            "-C",
-                 action='callback', callback=self._carry_option,
+            action="callback",
-                 metavar='CONTEXT', type='str',
+            callback=self._carry_option,
-                 help='Show CONTEXT lines before match')
+            metavar="CONTEXT",
-    g.add_option('-A',
+            type="str",
-                 action='callback', callback=self._carry_option,
+            help="Show CONTEXT lines around match",
-                 metavar='CONTEXT', type='str',
+        )
-                 help='Show CONTEXT lines after match')
+        g.add_option(
-    g.add_option('-l', '--name-only', '--files-with-matches',
+            "-B",
-                 action='callback', callback=self._carry_option,
+            action="callback",
-                 help='Show only file names containing matching lines')
+            callback=self._carry_option,
-    g.add_option('-L', '--files-without-match',
+            metavar="CONTEXT",
-                 action='callback', callback=self._carry_option,
+            type="str",
-                 help='Show only file names not containing matching lines')
+            help="Show CONTEXT lines before match",
+        )
+        g.add_option(
+            "-A",
+            action="callback",
+            callback=self._carry_option,
+            metavar="CONTEXT",
+            type="str",
+            help="Show CONTEXT lines after match",
+        )
+        g.add_option(
+            "-l",
+            "--name-only",
+            "--files-with-matches",
+            action="callback",
+            callback=self._carry_option,
+            help="Show only file names containing matching lines",
+        )
+        g.add_option(
+            "-L",
+            "--files-without-match",
+            action="callback",
+            callback=self._carry_option,
+            help="Show only file names not containing matching lines",
+        )
 
-  def _ExecuteOne(self, cmd_argv, project):
+    @classmethod
-    """Process one project."""
+    def _ExecuteOne(cls, cmd_argv, project_idx):
-    try:
+        """Process one project."""
-      p = GitCommand(project,
+        project = cls.get_parallel_context()["projects"][project_idx]
-                     cmd_argv,
+        try:
-                     bare=False,
+            p = GitCommand(
-                     capture_stdout=True,
+                project,
-                     capture_stderr=True)
+                cmd_argv,
-    except GitError as e:
+                bare=False,
-      return (project, -1, None, str(e))
+                capture_stdout=True,
+                capture_stderr=True,
+                verify_command=True,
+            )
+        except GitError as e:
+            return ExecuteOneResult(project_idx, -1, None, str(e), e)
 
-    return (project, p.Wait(), p.stdout, p.stderr)
+        try:
+            error = None
+            rc = p.Wait()
+        except GitError as e:
+            rc = 1
+            error = e
+        return ExecuteOneResult(project_idx, rc, p.stdout, p.stderr, error)
 
-  @staticmethod
+    @staticmethod
-  def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
+    def _ProcessResults(
-    git_failed = False
+        full_name, have_rev, opt, projects, _pool, out, results
-    bad_rev = False
+    ):
-    have_match = False
+        git_failed = False
-    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+        bad_rev = False
+        have_match = False
+        _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+        errors = []
 
-    for project, rc, stdout, stderr in results:
+        for result in results:
-      if rc < 0:
+            project = projects[result.project_idx]
-        git_failed = True
+            if result.rc < 0:
-        out.project('--- project %s ---' % _RelPath(project))
+                git_failed = True
-        out.nl()
+                out.project("--- project %s ---" % _RelPath(project))
-        out.fail('%s', stderr)
+                out.nl()
-        out.nl()
+                out.fail("%s", result.stderr)
-        continue
+                out.nl()
+                errors.append(result.error)
+                continue
 
-      if rc:
+            if result.rc:
-        # no results
+                # no results
-        if stderr:
+                if result.stderr:
-          if have_rev and 'fatal: ambiguous argument' in stderr:
+                    if (
-            bad_rev = True
+                        have_rev
-          else:
+                        and "fatal: ambiguous argument" in result.stderr
-            out.project('--- project %s ---' % _RelPath(project))
+                    ):
-            out.nl()
+                        bad_rev = True
-            out.fail('%s', stderr.strip())
+                    else:
-            out.nl()
+                        out.project("--- project %s ---" % _RelPath(project))
-        continue
+                        out.nl()
-      have_match = True
+                        out.fail("%s", result.stderr.strip())
+                        out.nl()
+                    if result.error is not None:
+                        errors.append(result.error)
+                continue
+            have_match = True
 
-      # We cut the last element, to avoid a blank line.
+            # We cut the last element, to avoid a blank line.
-      r = stdout.split('\n')
+            r = result.stdout.split("\n")
-      r = r[0:-1]
+            r = r[0:-1]
 
-      if have_rev and full_name:
+            if have_rev and full_name:
-        for line in r:
+                for line in r:
-          rev, line = line.split(':', 1)
+                    rev, line = line.split(":", 1)
-          out.write("%s", rev)
+                    out.write("%s", rev)
-          out.write(':')
+                    out.write(":")
-          out.project(_RelPath(project))
+                    out.project(_RelPath(project))
-          out.write('/')
+                    out.write("/")
-          out.write("%s", line)
+                    out.write("%s", line)
-          out.nl()
+                    out.nl()
-      elif full_name:
+            elif full_name:
-        for line in r:
+                for line in r:
-          out.project(_RelPath(project))
+                    out.project(_RelPath(project))
-          out.write('/')
+                    out.write("/")
-          out.write("%s", line)
+                    out.write("%s", line)
-          out.nl()
+                    out.nl()
-      else:
+            else:
-        for line in r:
+                for line in r:
-          print(line)
+                    print(line)
 
-    return (git_failed, bad_rev, have_match)
+        return (git_failed, bad_rev, have_match, errors)
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    out = GrepColoring(self.manifest.manifestProject.config)
+        out = GrepColoring(self.manifest.manifestProject.config)
 
-    cmd_argv = ['grep']
+        cmd_argv = ["grep"]
-    if out.is_on:
+        if out.is_on:
-      cmd_argv.append('--color')
+            cmd_argv.append("--color")
-    cmd_argv.extend(getattr(opt, 'cmd_argv', []))
+        cmd_argv.extend(getattr(opt, "cmd_argv", []))
 
-    if '-e' not in cmd_argv:
+        if "-e" not in cmd_argv:
-      if not args:
+            if not args:
-        self.Usage()
+                self.Usage()
-      cmd_argv.append('-e')
+            cmd_argv.append("-e")
-      cmd_argv.append(args[0])
+            cmd_argv.append(args[0])
-      args = args[1:]
+            args = args[1:]
 
-    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
 
-    full_name = False
+        full_name = False
-    if len(projects) > 1:
+        if len(projects) > 1:
-      cmd_argv.append('--full-name')
+            cmd_argv.append("--full-name")
-      full_name = True
+            full_name = True
 
-    have_rev = False
+        have_rev = False
-    if opt.revision:
+        if opt.revision:
-      if '--cached' in cmd_argv:
+            if "--cached" in cmd_argv:
-        print('fatal: cannot combine --cached and --revision', file=sys.stderr)
+                msg = "fatal: cannot combine --cached and --revision"
-        sys.exit(1)
+                logger.error(msg)
-      have_rev = True
+                raise InvalidArgumentsError(msg)
-      cmd_argv.extend(opt.revision)
+            have_rev = True
-    cmd_argv.append('--')
+            cmd_argv.extend(opt.revision)
+        cmd_argv.append("--")
 
-    git_failed, bad_rev, have_match = self.ExecuteInParallel(
+        with self.ParallelContext():
-        opt.jobs,
+            self.get_parallel_context()["projects"] = projects
-        functools.partial(self._ExecuteOne, cmd_argv),
+            git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
-        projects,
+                opt.jobs,
-        callback=functools.partial(self._ProcessResults, full_name, have_rev, opt),
+                functools.partial(self._ExecuteOne, cmd_argv),
-        output=out,
+                range(len(projects)),
-        ordered=True)
+                callback=functools.partial(
+                    self._ProcessResults, full_name, have_rev, opt, projects
+                ),
+                output=out,
+                ordered=True,
+                chunksize=1,
+            )
 
-    if git_failed:
+        if git_failed:
-      sys.exit(1)
+            raise GrepCommandError(
-    elif have_match:
+                "error: git failures", aggregate_errors=errors
-      sys.exit(0)
+            )
-    elif have_rev and bad_rev:
+        elif have_match:
-      for r in opt.revision:
+            sys.exit(0)
-        print("error: can't search revision %s" % r, file=sys.stderr)
+        elif have_rev and bad_rev:
-      sys.exit(1)
+            for r in opt.revision:
-    else:
+                logger.error("error: can't search revision %s", r)
-      sys.exit(1)
+        raise GrepCommandError(aggregate_errors=errors)

subcmds/help.py

@@ -16,165 +16,176 @@ import re
 import sys
 import textwrap
 
-from subcmds import all_commands
 from color import Coloring
-from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
+from command import MirrorSafeCommand
-import gitc_utils
+from command import PagedCommand
+from error import RepoExitError
+from subcmds import all_commands
 from wrapper import Wrapper
 
 
+class InvalidHelpCommand(RepoExitError):
+    """Invalid command passed into help."""
+
+
 class Help(PagedCommand, MirrorSafeCommand):
-  COMMON = False
+    COMMON = False
-  helpSummary = "Display detailed help on a command"
+    helpSummary = "Display detailed help on a command"
-  helpUsage = """
+    helpUsage = """
 %prog [--all|command]
 """
-  helpDescription = """
+    helpDescription = """
 Displays detailed usage information about a command.
 """
 
-  def _PrintCommands(self, commandNames):
+    def _PrintCommands(self, commandNames):
-    """Helper to display |commandNames| summaries."""
+        """Helper to display |commandNames| summaries."""
-    maxlen = 0
+        maxlen = 0
-    for name in commandNames:
+        for name in commandNames:
-      maxlen = max(maxlen, len(name))
+            maxlen = max(maxlen, len(name))
-    fmt = '  %%-%ds  %%s' % maxlen
+        fmt = "  %%-%ds  %%s" % maxlen
 
-    for name in commandNames:
+        for name in commandNames:
-      command = all_commands[name]()
+            command = all_commands[name]()
-      try:
+            try:
-        summary = command.helpSummary.strip()
+                summary = command.helpSummary.strip()
-      except AttributeError:
+            except AttributeError:
-        summary = ''
+                summary = ""
-      print(fmt % (name, summary))
+            print(fmt % (name, summary))
 
-  def _PrintAllCommands(self):
+    def _PrintAllCommands(self):
-    print('usage: repo COMMAND [ARGS]')
+        print("usage: repo COMMAND [ARGS]")
-    self.PrintAllCommandsBody()
+        self.PrintAllCommandsBody()
 
-  def PrintAllCommandsBody(self):
+    def PrintAllCommandsBody(self):
-    print('The complete list of recognized repo commands is:')
+        print("The complete list of recognized repo commands is:")
-    commandNames = list(sorted(all_commands))
+        commandNames = sorted(all_commands)
-    self._PrintCommands(commandNames)
+        self._PrintCommands(commandNames)
-    print("See 'repo help <command>' for more information on a "
+        print(
-          'specific command.')
+            "See 'repo help <command>' for more information on a "
-    print('Bug reports:', Wrapper().BUG_URL)
+            "specific command."
+        )
+        print("Bug reports:", Wrapper().BUG_URL)
 
-  def _PrintCommonCommands(self):
+    def _PrintCommonCommands(self):
-    print('usage: repo COMMAND [ARGS]')
+        print("usage: repo COMMAND [ARGS]")
-    self.PrintCommonCommandsBody()
+        self.PrintCommonCommandsBody()
 
-  def PrintCommonCommandsBody(self):
+    def PrintCommonCommandsBody(self):
-    print('The most commonly used repo commands are:')
+        print("The most commonly used repo commands are:")
 
-    def gitc_supported(cmd):
+        commandNames = sorted(
-      if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand):
+            name for name, command in all_commands.items() if command.COMMON
-        return True
+        )
-      if self.client.isGitcClient:
+        self._PrintCommands(commandNames)
-        return True
-      if isinstance(cmd, GitcClientCommand):
-        return False
-      if gitc_utils.get_gitc_manifest_dir():
-        return True
-      return False
 
-    commandNames = list(sorted([name
+        print(
-                                for name, command in all_commands.items()
+            "See 'repo help <command>' for more information on a specific "
-                                if command.COMMON and gitc_supported(command)]))
+            "command.\nSee 'repo help --all' for a complete list of recognized "
-    self._PrintCommands(commandNames)
+            "commands."
+        )
+        print("Bug reports:", Wrapper().BUG_URL)
 
-    print(
+    def _PrintCommandHelp(self, cmd, header_prefix=""):
-        "See 'repo help <command>' for more information on a specific command.\n"
+        class _Out(Coloring):
-        "See 'repo help --all' for a complete list of recognized commands.")
+            def __init__(self, gc):
-    print('Bug reports:', Wrapper().BUG_URL)
+                Coloring.__init__(self, gc, "help")
+                self.heading = self.printer("heading", attr="bold")
+                self._first = True
 
-  def _PrintCommandHelp(self, cmd, header_prefix=''):
+            def _PrintSection(self, heading, bodyAttr):
-    class _Out(Coloring):
+                try:
-      def __init__(self, gc):
+                    body = getattr(cmd, bodyAttr)
-        Coloring.__init__(self, gc, 'help')
+                except AttributeError:
-        self.heading = self.printer('heading', attr='bold')
+                    return
-        self._first = True
+                if body == "" or body is None:
+                    return
 
-      def _PrintSection(self, heading, bodyAttr):
+                if not self._first:
-        try:
+                    self.nl()
-          body = getattr(cmd, bodyAttr)
+                self._first = False
-        except AttributeError:
-          return
-        if body == '' or body is None:
-          return
 
-        if not self._first:
+                self.heading("%s%s", header_prefix, heading)
-          self.nl()
+                self.nl()
-        self._first = False
+                self.nl()
 
-        self.heading('%s%s', header_prefix, heading)
+                me = "repo %s" % cmd.NAME
-        self.nl()
+                body = body.strip()
-        self.nl()
+                body = body.replace("%prog", me)
 
-        me = 'repo %s' % cmd.NAME
+                # Extract the title, but skip any trailing {#anchors}.
-        body = body.strip()
+                asciidoc_hdr = re.compile(r"^\n?#+ ([^{]+)(\{#.+\})?$")
-        body = body.replace('%prog', me)
+                for para in body.split("\n\n"):
+                    if para.startswith(" "):
+                        self.write("%s", para)
+                        self.nl()
+                        self.nl()
+                        continue
 
-        # Extract the title, but skip any trailing {#anchors}.
+                    m = asciidoc_hdr.match(para)
-        asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$')
+                    if m:
-        for para in body.split("\n\n"):
+                        self.heading("%s%s", header_prefix, m.group(1))
-          if para.startswith(' '):
+                        self.nl()
-            self.write('%s', para)
+                        self.nl()
-            self.nl()
+                        continue
-            self.nl()
-            continue
 
-          m = asciidoc_hdr.match(para)
+                    lines = textwrap.wrap(
-          if m:
+                        para.replace("  ", " "),
-            self.heading('%s%s', header_prefix, m.group(1))
+                        width=80,
-            self.nl()
+                        break_long_words=False,
-            self.nl()
+                        break_on_hyphens=False,
-            continue
+                    )
+                    for line in lines:
+                        self.write("%s", line)
+                        self.nl()
+                    self.nl()
 
-          lines = textwrap.wrap(para.replace('  ', ' '), width=80,
+        out = _Out(self.client.globalConfig)
-                                break_long_words=False, break_on_hyphens=False)
+        out._PrintSection("Summary", "helpSummary")
-          for line in lines:
+        cmd.OptionParser.print_help()
-            self.write('%s', line)
+        out._PrintSection("Description", "helpDescription")
-            self.nl()
-          self.nl()
 
-    out = _Out(self.client.globalConfig)
+    def _PrintAllCommandHelp(self):
-    out._PrintSection('Summary', 'helpSummary')
+        for name in sorted(all_commands):
-    cmd.OptionParser.print_help()
+            cmd = all_commands[name](manifest=self.manifest)
-    out._PrintSection('Description', 'helpDescription')
+            self._PrintCommandHelp(cmd, header_prefix=f"[{name}] ")
 
-  def _PrintAllCommandHelp(self):
+    def _Options(self, p):
-    for name in sorted(all_commands):
+        p.add_option(
-      cmd = all_commands[name](manifest=self.manifest)
+            "-a",
-      self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
+            "--all",
+            dest="show_all",
+            action="store_true",
+            help="show the complete list of commands",
+        )
+        p.add_option(
+            "--help-all",
+            dest="show_all_help",
+            action="store_true",
+            help="show the --help of all commands",
+        )
 
-  def _Options(self, p):
+    def Execute(self, opt, args):
-    p.add_option('-a', '--all',
+        if len(args) == 0:
-                 dest='show_all', action='store_true',
+            if opt.show_all_help:
-                 help='show the complete list of commands')
+                self._PrintAllCommandHelp()
-    p.add_option('--help-all',
+            elif opt.show_all:
-                 dest='show_all_help', action='store_true',
+                self._PrintAllCommands()
-                 help='show the --help of all commands')
+            else:
+                self._PrintCommonCommands()
 
-  def Execute(self, opt, args):
+        elif len(args) == 1:
-    if len(args) == 0:
+            name = args[0]
-      if opt.show_all_help:
-        self._PrintAllCommandHelp()
-      elif opt.show_all:
-        self._PrintAllCommands()
-      else:
-        self._PrintCommonCommands()
 
-    elif len(args) == 1:
+            try:
-      name = args[0]
+                cmd = all_commands[name](manifest=self.manifest)
+            except KeyError:
+                print(
+                    "repo: '%s' is not a repo command." % name, file=sys.stderr
+                )
+                raise InvalidHelpCommand(name)
 
-      try:
+            self._PrintCommandHelp(cmd)
-        cmd = all_commands[name](manifest=self.manifest)
-      except KeyError:
-        print("repo: '%s' is not a repo command." % name, file=sys.stderr)
-        sys.exit(1)
 
-      self._PrintCommandHelp(cmd)
+        else:
-
+            self._PrintCommandHelp(self)
-    else:
-      self._PrintCommandHelp(self)

subcmds/info.py

@@ -12,211 +12,446 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import enum
+import functools
+import io
+import json
 import optparse
+import sys
+from typing import Any, Dict, List, NamedTuple
 
-from command import PagedCommand
 from color import Coloring
-from git_refs import R_M, R_HEADS
+from command import DEFAULT_LOCAL_JOBS
+from command import PagedCommand
+from git_refs import R_HEADS
+from git_refs import R_M
+
+
+class BranchInfo(NamedTuple):
+    """Holds information about a branch in a project."""
+
+    relpath: str
+    name: str
+    commits: Any
+    date: str
+    is_current: bool
+
+
+class OutputFormat(enum.Enum):
+    """Type for the requested output format."""
+
+    # Human-readable text output.
+    TEXT = enum.auto()
+
+    # Machine-readable JSON output.
+    JSON = enum.auto()
 
 
 class _Coloring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, "status")
+        Coloring.__init__(self, config, "status")
 
 
 class Info(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
-  helpUsage = "%prog [-dl] [-o [-c]] [<project>...]"
+    helpSummary = (
+        "Get info on the manifest branch, current branch or unmerged branches"
+    )
+    helpUsage = "%prog [-dl] [-o [-c]] [--format=<format>] [<project>...]"
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-d', '--diff',
+        p.add_option(
-                 dest='all', action='store_true',
+            "-d",
-                 help="show full info and commit diff including remote branches")
+            "--diff",
-    p.add_option('-o', '--overview',
+            dest="all",
-                 dest='overview', action='store_true',
+            action="store_true",
-                 help='show overview of all local commits')
+            help="show full info and commit diff including remote branches",
-    p.add_option('-c', '--current-branch',
+        )
-                 dest="current_branch", action="store_true",
+        p.add_option(
-                 help="consider only checked out branches")
+            "-o",
-    p.add_option('--no-current-branch',
+            "--overview",
-                 dest='current_branch', action='store_false',
+            action="store_true",
-                 help='consider all local branches')
+            help="show overview of all local commits",
-    # Turn this into a warning & remove this someday.
+        )
-    p.add_option('-b',
+        p.add_option(
-                 dest='current_branch', action='store_true',
+            "--include-summary",
-                 help=optparse.SUPPRESS_HELP)
+            action="store_true",
-    p.add_option('-l', '--local-only',
+            default=True,
-                 dest="local", action="store_true",
+            help="include manifest summary (default: true)",
-                 help="disable all remote operations")
+        )
+        p.add_option(
+            "--no-include-summary",
+            dest="include_summary",
+            action="store_false",
+            help="exclude manifest summary",
+        )
+        p.add_option(
+            "--include-projects",
+            action="store_true",
+            default=True,
+            help="include project details (default: true)",
+        )
+        p.add_option(
+            "--no-include-projects",
+            dest="include_projects",
+            action="store_false",
+            help="exclude project details",
+        )
+        p.add_option(
+            "-c",
+            "--current-branch",
+            action="store_true",
+            help="consider only checked out branches",
+        )
+        p.add_option(
+            "--no-current-branch",
+            dest="current_branch",
+            action="store_false",
+            help="consider all local branches",
+        )
+        # Turn this into a warning & remove this someday.
+        p.add_option(
+            "-b",
+            dest="current_branch",
+            action="store_true",
+            help=optparse.SUPPRESS_HELP,
+        )
+        p.add_option(
+            "-l",
+            "--local-only",
+            dest="local",
+            action="store_true",
+            help="disable all remote operations",
+        )
+        formats = tuple(x.lower() for x in OutputFormat.__members__.keys())
+        p.add_option(
+            "--format",
+            default=OutputFormat.TEXT.name.lower(),
+            choices=formats,
+            help=f"output format: {', '.join(formats)} (default: %default)",
+        )
 
-  def Execute(self, opt, args):
+    def WantPager(self, opt):
-    self.out = _Coloring(self.client.globalConfig)
+        return OutputFormat[opt.format.upper()] == OutputFormat.TEXT
-    self.heading = self.out.printer('heading', attr='bold')
-    self.headtext = self.out.nofmt_printer('headtext', fg='yellow')
-    self.redtext = self.out.printer('redtext', fg='red')
-    self.sha = self.out.printer("sha", fg='yellow')
-    self.text = self.out.nofmt_printer('text')
-    self.dimtext = self.out.printer('dimtext', attr='dim')
 
-    self.opt = opt
+    def ValidateOptions(self, opt, args):
+        output_format = OutputFormat[opt.format.upper()]
+        if output_format == OutputFormat.JSON:
+            if opt.all:
+                self.OptionParser.error("--diff is not supported with JSON")
+            if opt.overview:
+                self.OptionParser.error("--overview is not supported with JSON")
 
-    if not opt.this_manifest_only:
+    def Execute(self, opt, args):
-      self.manifest = self.manifest.outer_client
+        if not opt.this_manifest_only:
-    manifestConfig = self.manifest.manifestProject.config
+            self.manifest = self.manifest.outer_client
-    mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = self.manifest.GetGroupsStr()
 
-    self.heading("Manifest branch: ")
+        self.TryOverrideManifestWithSmartSync()
-    if self.manifest.default.revisionExpr:
-        self.headtext(self.manifest.default.revisionExpr)
-    self.out.nl()
-    self.heading("Manifest merge branch: ")
-    self.headtext(mergeBranch)
-    self.out.nl()
-    self.heading("Manifest groups: ")
-    self.headtext(manifestGroups)
-    self.out.nl()
 
-    self.printSeparator()
+        output_format = OutputFormat[opt.format.upper()]
+        if output_format == OutputFormat.JSON:
+            self._ExecuteJson(opt, args)
+        else:
+            self._ExecuteText(opt, args)
 
-    if not opt.overview:
+    def _ExecuteText(self, opt, args) -> None:
-      self._printDiffInfo(opt, args)
+        """Output info as human-readable text."""
-    else:
+        self.out = _Coloring(self.client.globalConfig)
-      self._printCommitOverview(opt, args)
+        self.heading = self.out.printer("heading", attr="bold")
+        self.headtext = self.out.nofmt_printer("headtext", fg="yellow")
+        self.redtext = self.out.printer("redtext", fg="red")
+        self.sha = self.out.printer("sha", fg="yellow")
+        self.text = self.out.nofmt_printer("text")
+        self.dimtext = self.out.printer("dimtext", attr="dim")
 
-  def printSeparator(self):
+        self.opt = opt
-    self.text("----------------------------")
-    self.out.nl()
 
-  def _printDiffInfo(self, opt, args):
+        if opt.include_summary:
-    # We let exceptions bubble up to main as they'll be well structured.
+            self._printSummary()
-    projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
-    for p in projs:
+        if not opt.include_projects:
-      self.heading("Project: ")
+            return
-      self.headtext(p.name)
+        elif not opt.overview:
-      self.out.nl()
+            self._printDiffInfo(opt, args)
+        else:
+            self._printCommitOverview(opt, args)
 
-      self.heading("Mount path: ")
+    def _getSummaryData(self) -> Dict[str, Any]:
-      self.headtext(p.worktree)
+        """Gather manifest summary data as a dict."""
-      self.out.nl()
+        manifestConfig = self.manifest.manifestProject.config
+        mergeBranch = manifestConfig.GetBranch("default").merge
+        manifestGroups = self.manifest.GetManifestGroupsStr()
+        sp = self.manifest.superproject
+        srev = sp.commit_id if sp and sp.commit_id else None
+        return {
+            "manifest_branch": self.manifest.default.revisionExpr or "",
+            "manifest_merge_branch": mergeBranch or "",
+            "manifest_groups": manifestGroups,
+            "superproject_revision": srev,
+        }
 
-      self.heading("Current revision: ")
+    def _getProjectData(self, project) -> Dict[str, Any]:
-      self.headtext(p.GetRevisionId())
+        """Gather project data as a dict."""
-      self.out.nl()
+        data = {
+            "name": project.name,
+            "mount_path": project.worktree,
+            "current_revision": project.GetRevisionId(),
+            "manifest_revision": project.revisionExpr,
+            "local_branches": list(project.GetBranches()),
+        }
+        currentBranch = project.CurrentBranch
+        if currentBranch:
+            data["current_branch"] = currentBranch
+        return data
 
-      currentBranch = p.CurrentBranch
+    def _ExecuteJson(self, opt, args) -> None:
-      if currentBranch:
+        """Output info as JSON."""
-        self.heading('Current branch: ')
+        result = {}
-        self.headtext(currentBranch)
+        if opt.include_summary:
+            result["summary"] = self._getSummaryData()
+        if opt.include_projects:
+            projs = self.GetProjects(
+                args, all_manifests=not opt.this_manifest_only
+            )
+            result["projects"] = [self._getProjectData(p) for p in projs]
+
+        json_settings = {
+            # JSON style guide says Unicode characters are fully allowed.
+            "ensure_ascii": False,
+            # We use 2 space indent to match JSON style guide.
+            "indent": 2,
+            "separators": (",", ": "),
+            "sort_keys": True,
+        }
+        sys.stdout.write(json.dumps(result, **json_settings) + "\n")
+
+    def _printSummary(self) -> None:
+        """Print manifest summary in text format."""
+        data = self._getSummaryData()
+        self.heading("Manifest branch: ")
+        self.headtext(data["manifest_branch"])
+        self.out.nl()
+        self.heading("Manifest merge branch: ")
+        self.headtext(data["manifest_merge_branch"])
+        self.out.nl()
+        self.heading("Manifest groups: ")
+        self.headtext(data["manifest_groups"])
+        self.out.nl()
+        self.heading("Superproject revision: ")
+        self.headtext(data["superproject_revision"] or "None")
+        self.out.nl()
+        self.printSeparator()
+
+    def printSeparator(self):
+        self.text("----------------------------")
         self.out.nl()
 
-      self.heading("Manifest revision: ")
+    @classmethod
-      self.headtext(p.revisionExpr)
+    def _DiffHelper(cls, project_idx: int, opt: Any) -> str:
-      self.out.nl()
+        """Helper for ParallelContext to get diff info for a project."""
+        buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
+        config = cls.get_parallel_context()["config"]
 
-      localBranches = list(p.GetBranches().keys())
+        out = _Coloring(config)
-      self.heading("Local Branches: ")
+        out.redirect(buf)
-      self.redtext(str(len(localBranches)))
-      if localBranches:
-        self.text(" [")
-        self.text(", ".join(localBranches))
-        self.text("]")
-      self.out.nl()
 
-      if self.opt.all:
+        heading = out.printer("heading", attr="bold")
-        self.findRemoteLocalDiff(p)
+        headtext = out.nofmt_printer("headtext", fg="yellow")
+        redtext = out.printer("redtext", fg="red")
+        sha = out.printer("sha", fg="yellow")
+        text = out.nofmt_printer("text")
+        dimtext = out.printer("dimtext", attr="dim")
 
-      self.printSeparator()
+        heading("Project: ")
+        headtext(project.name)
+        out.nl()
 
-  def findRemoteLocalDiff(self, project):
+        heading("Mount path: ")
-    # Fetch all the latest commits.
+        headtext(project.worktree)
-    if not self.opt.local:
+        out.nl()
-      project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
 
-    branch = self.manifest.manifestProject.config.GetBranch('default').merge
+        heading("Current revision: ")
-    if branch.startswith(R_HEADS):
+        headtext(project.GetRevisionId())
-      branch = branch[len(R_HEADS):]
+        out.nl()
-    logTarget = R_M + branch
 
-    bareTmp = project.bare_git._bare
+        currentBranch = project.CurrentBranch
-    project.bare_git._bare = False
+        if currentBranch:
-    localCommits = project.bare_git.rev_list(
+            heading("Current branch: ")
-        '--abbrev=8',
+            headtext(currentBranch)
-        '--abbrev-commit',
+            out.nl()
-        '--pretty=oneline',
-        logTarget + "..",
-        '--')
 
-    originCommits = project.bare_git.rev_list(
+        heading("Manifest revision: ")
-        '--abbrev=8',
+        headtext(project.revisionExpr)
-        '--abbrev-commit',
+        out.nl()
-        '--pretty=oneline',
-        ".." + logTarget,
-        '--')
-    project.bare_git._bare = bareTmp
 
-    self.heading("Local Commits: ")
+        localBranches = list(project.GetBranches().keys())
-    self.redtext(str(len(localCommits)))
+        heading("Local Branches: ")
-    self.dimtext(" (on current branch)")
+        redtext(str(len(localBranches)))
-    self.out.nl()
+        if localBranches:
+            text(" [")
+            text(", ".join(localBranches))
+            text("]")
+        out.nl()
 
-    for c in localCommits:
+        if opt.all:
-      split = c.split()
+            if not opt.local:
-      self.sha(split[0] + " ")
+                project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
-      self.text(" ".join(split[1:]))
-      self.out.nl()
 
-    self.printSeparator()
+            branch = project.manifest.manifestProject.config.GetBranch(
+                "default"
+            ).merge
+            if branch.startswith(R_HEADS):
+                branch = branch[len(R_HEADS) :]
+            logTarget = R_M + branch
 
-    self.heading("Remote Commits: ")
+            bareTmp = project.bare_git._bare
-    self.redtext(str(len(originCommits)))
+            project.bare_git._bare = False
-    self.out.nl()
+            localCommits = project.bare_git.rev_list(
+                "--abbrev=8",
+                "--abbrev-commit",
+                "--pretty=oneline",
+                logTarget + "..",
+                "--",
+            )
 
-    for c in originCommits:
+            originCommits = project.bare_git.rev_list(
-      split = c.split()
+                "--abbrev=8",
-      self.sha(split[0] + " ")
+                "--abbrev-commit",
-      self.text(" ".join(split[1:]))
+                "--pretty=oneline",
-      self.out.nl()
+                ".." + logTarget,
+                "--",
+            )
+            project.bare_git._bare = bareTmp
 
-  def _printCommitOverview(self, opt, args):
+            heading("Local Commits: ")
-    all_branches = []
+            redtext(str(len(localCommits)))
-    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
+            dimtext(" (on current branch)")
-      br = [project.GetUploadableBranch(x)
+            out.nl()
-            for x in project.GetBranches()]
-      br = [x for x in br if x]
-      if self.opt.current_branch:
-        br = [x for x in br if x.name == project.CurrentBranch]
-      all_branches.extend(br)
 
-    if not all_branches:
+            for c in localCommits:
-      return
+                split = c.split()
+                sha(split[0] + " ")
+                text(" ".join(split[1:]))
+                out.nl()
 
-    self.out.nl()
+            text("----------------------------")
-    self.heading('Projects Overview')
+            out.nl()
-    project = None
+
+            heading("Remote Commits: ")
+            redtext(str(len(originCommits)))
+            out.nl()
+
+            for c in originCommits:
+                split = c.split()
+                sha(split[0] + " ")
+                text(" ".join(split[1:]))
+                out.nl()
+
+        text("----------------------------")
+        out.nl()
+
+        return buf.getvalue()
+
+    def _printDiffInfo(self, opt, args):
+        projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+
+        def _ProcessResults(_pool, _output, results):
+            for output in results:
+                if output:
+                    print(output, end="")
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projs
+            self.get_parallel_context()[
+                "config"
+            ] = self.manifest.manifestProject.config
+
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._DiffHelper, opt=opt),
+                range(len(projs)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )
+
+    @classmethod
+    def _OverviewHelper(cls, project_idx: int, opt: Any) -> List[BranchInfo]:
+        """Helper to get overview of uploadable branches."""
+        project = cls.get_parallel_context()["projects"][project_idx]
+
+        branches = []
+        br = [project.GetUploadableBranch(x) for x in project.GetBranches()]
+        br = [x for x in br if x]
+        if opt.current_branch:
+            br = [x for x in br if x.name == project.CurrentBranch]
+
+        for b in br:
+            branches.append(
+                BranchInfo(
+                    relpath=project.RelPath(local=opt.this_manifest_only),
+                    name=b.name,
+                    commits=b.commits,
+                    date=b.date,
+                    is_current=b.name == project.CurrentBranch,
+                )
+            )
+        return branches
+
+    def _printCommitOverview(self, opt, args):
+        projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+
+        all_branches = []
+
+        def _ProcessResults(_pool, _output, results):
+            for branches in results:
+                all_branches.extend(branches)
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projs
+
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._OverviewHelper, opt=opt),
+                range(len(projs)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )
+
+        if not all_branches:
+            return
 
-    for branch in all_branches:
-      if project != branch.project:
-        project = branch.project
-        self.out.nl()
-        self.headtext(project.RelPath(local=opt.this_manifest_only))
         self.out.nl()
+        self.heading("Projects Overview")
+        current_relpath = None
 
-      commits = branch.commits
+        for branch in all_branches:
-      date = branch.date
+            if current_relpath != branch.relpath:
-      self.text('%s %-33s (%2d commit%s, %s)' % (
+                current_relpath = branch.relpath
-          branch.name == project.CurrentBranch and '*' or ' ',
+                self.out.nl()
-          branch.name,
+                self.headtext(current_relpath)
-          len(commits),
+                self.out.nl()
-          len(commits) != 1 and 's' or '',
-          date))
-      self.out.nl()
 
-      for commit in commits:
+            commits = branch.commits
-        split = commit.split()
+            date = branch.date
-        self.text('{0:38}{1} '.format('', '-'))
+            self.text(
-        self.sha(split[0] + " ")
+                "%s %-33s (%2d commit%s, %s)"
-        self.text(" ".join(split[1:]))
+                % (
-        self.out.nl()
+                    branch.is_current and "*" or " ",
+                    branch.name,
+                    len(commits),
+                    len(commits) != 1 and "s" or "",
+                    date,
+                )
+            )
+            self.out.nl()
+
+            for commit in commits:
+                split = commit.split()
+                self.text(f"{'':38}{'-'} ")
+                self.sha(split[0] + " ")
+                self.text(" ".join(split[1:]))
+                self.out.nl()

subcmds/init.py

@@ -13,28 +13,32 @@
 # limitations under the License.
 
 import os
-import platform
-import re
 import sys
-import urllib.parse
 
 from color import Coloring
-from command import InteractiveCommand, MirrorSafeCommand
+from command import InteractiveCommand
-from error import ManifestParseError
+from command import MirrorSafeCommand
-from project import SyncBuffer
+from error import RepoUnhandledExceptionError
-from git_config import GitConfig
+from error import UpdateManifestError
-from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
+from git_command import git_require
+from repo_logging import RepoLogger
 from wrapper import Wrapper
+from wrapper import WrapperDir
+
+
+logger = RepoLogger(__file__)
+
+_REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
 
 
 class Init(InteractiveCommand, MirrorSafeCommand):
-  COMMON = True
+    COMMON = True
-  MULTI_MANIFEST_SUPPORT = True
+    MULTI_MANIFEST_SUPPORT = True
-  helpSummary = "Initialize a repo client checkout in the current directory"
+    helpSummary = "Initialize a repo client checkout in the current directory"
-  helpUsage = """
+    helpUsage = """
 %prog [options] [manifest url]
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command is run once to install and initialize repo.
 The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
@@ -48,6 +52,10 @@ The optional -b argument can be used to select the manifest branch
 to checkout and use.  If no branch is specified, the remote's default
 branch is used.  This is equivalent to using -b HEAD.
 
+The optional --manifest-upstream-branch argument can be used when a commit is
+provided to --manifest-branch (or -b), to specify the name of the git ref in
+which the commit can be found.
+
 The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
 will be used.
@@ -83,239 +91,321 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
 to update the working directory files.
 """
 
-  def _CommonOptions(self, p):
+    def _CommonOptions(self, p):
-    """Disable due to re-use of Wrapper()."""
+        """Disable due to re-use of Wrapper()."""
 
-  def _Options(self, p, gitc_init=False):
+    def _Options(self, p):
-    Wrapper().InitParser(p, gitc_init=gitc_init)
+        Wrapper().InitParser(p)
-    m = p.add_option_group('Multi-manifest')
+        m = p.add_option_group("Multi-manifest")
-    m.add_option('--outer-manifest', action='store_true', default=True,
+        m.add_option(
-                 help='operate starting at the outermost manifest')
+            "--outer-manifest",
-    m.add_option('--no-outer-manifest', dest='outer_manifest',
+            action="store_true",
-                 action='store_false', help='do not operate on outer manifests')
+            default=True,
-    m.add_option('--this-manifest-only', action='store_true', default=None,
+            help="operate starting at the outermost manifest",
-                 help='only operate on this (sub)manifest')
+        )
-    m.add_option('--no-this-manifest-only', '--all-manifests',
+        m.add_option(
-                 dest='this_manifest_only', action='store_false',
+            "--no-outer-manifest",
-                 help='operate on this manifest and its submanifests')
+            dest="outer_manifest",
+            action="store_false",
+            help="do not operate on outer manifests",
+        )
+        m.add_option(
+            "--this-manifest-only",
+            action="store_true",
+            default=None,
+            help="only operate on this (sub)manifest",
+        )
+        m.add_option(
+            "--no-this-manifest-only",
+            "--all-manifests",
+            dest="this_manifest_only",
+            action="store_false",
+            help="operate on this manifest and its submanifests",
+        )
 
-  def _RegisteredEnvironmentOptions(self):
+    def _RegisteredEnvironmentOptions(self):
-    return {'REPO_MANIFEST_URL': 'manifest_url',
+        return {
-            'REPO_MIRROR_LOCATION': 'reference'}
+            "REPO_MANIFEST_URL": "manifest_url",
+            "REPO_MIRROR_LOCATION": "reference",
+            "REPO_GIT_LFS": "git_lfs",
+        }
 
-  def _SyncManifest(self, opt):
+    def _SyncManifest(self, opt):
-    """Call manifestProject.Sync with arguments from opt.
+        """Call manifestProject.Sync with arguments from opt.
 
-    Args:
+        Args:
-      opt: options from optparse.
+            opt: options from optparse.
-    """
+        """
-    if not self.manifest.manifestProject.Sync(
+        # Normally this value is set when instantiating the project, but the
-        manifest_url=opt.manifest_url,
+        # manifest project is special and is created when instantiating the
-        manifest_branch=opt.manifest_branch,
+        # manifest which happens before we parse options.
-        standalone_manifest=opt.standalone_manifest,
+        self.manifest.manifestProject.clone_depth = opt.manifest_depth
-        groups=opt.groups,
+        self.manifest.manifestProject.upstream = opt.manifest_upstream_branch
-        platform=opt.platform,
+        clone_filter_for_depth = (
-        mirror=opt.mirror,
+            "blob:none" if (_REPO_ALLOW_SHALLOW == "0") else None
-        dissociate=opt.dissociate,
+        )
-        reference=opt.reference,
+        if not self.manifest.manifestProject.Sync(
-        worktree=opt.worktree,
+            manifest_url=opt.manifest_url,
-        submodules=opt.submodules,
+            manifest_branch=opt.manifest_branch,
-        archive=opt.archive,
+            standalone_manifest=opt.standalone_manifest,
-        partial_clone=opt.partial_clone,
+            groups=opt.groups,
-        clone_filter=opt.clone_filter,
+            platform=opt.platform,
-        partial_clone_exclude=opt.partial_clone_exclude,
+            mirror=opt.mirror,
-        clone_bundle=opt.clone_bundle,
+            dissociate=opt.dissociate,
-        git_lfs=opt.git_lfs,
+            reference=opt.reference,
-        use_superproject=opt.use_superproject,
+            worktree=opt.worktree,
-        verbose=opt.verbose,
+            submodules=opt.submodules,
-        current_branch_only=opt.current_branch_only,
+            archive=opt.archive,
-        tags=opt.tags,
+            partial_clone=opt.partial_clone,
-        depth=opt.depth,
+            clone_filter=opt.clone_filter,
-        git_event_log=self.git_event_log,
+            partial_clone_exclude=opt.partial_clone_exclude,
-        manifest_name=opt.manifest_name):
+            clone_filter_for_depth=clone_filter_for_depth,
-      sys.exit(1)
+            clone_bundle=opt.clone_bundle,
+            git_lfs=opt.git_lfs,
+            use_superproject=opt.use_superproject,
+            verbose=opt.verbose,
+            current_branch_only=opt.current_branch_only,
+            tags=opt.tags,
+            depth=opt.depth,
+            git_event_log=self.git_event_log,
+            manifest_name=opt.manifest_name,
+        ):
+            manifest_name = opt.manifest_name
+            raise UpdateManifestError(
+                f"Unable to sync manifest {manifest_name}"
+            )
 
-  def _Prompt(self, prompt, value):
+    def _Prompt(self, prompt, value):
-    print('%-10s [%s]: ' % (prompt, value), end='')
+        print("%-10s [%s]: " % (prompt, value), end="", flush=True)
-    # TODO: When we require Python 3, use flush=True w/print above.
+        a = sys.stdin.readline().strip()
-    sys.stdout.flush()
+        if a == "":
-    a = sys.stdin.readline().strip()
+            return value
-    if a == '':
+        return a
-      return value
-    return a
 
-  def _ShouldConfigureUser(self, opt):
+    def _ShouldConfigureUser(self, opt, existing_checkout):
-    gc = self.client.globalConfig
+        gc = self.client.globalConfig
-    mp = self.manifest.manifestProject
+        mp = self.manifest.manifestProject
 
-    # If we don't have local settings, get from global.
+        # If we don't have local settings, get from global.
-    if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+        if not mp.config.Has("user.name") or not mp.config.Has("user.email"):
-      if not gc.Has('user.name') or not gc.Has('user.email'):
+            if not gc.Has("user.name") or not gc.Has("user.email"):
-        return True
+                return True
 
-      mp.config.SetString('user.name', gc.GetString('user.name'))
+            mp.config.SetString("user.name", gc.GetString("user.name"))
-      mp.config.SetString('user.email', gc.GetString('user.email'))
+            mp.config.SetString("user.email", gc.GetString("user.email"))
 
-    if not opt.quiet:
+        if not opt.quiet and not existing_checkout or opt.verbose:
-      print()
+            print()
-      print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+            print(
-                                           mp.config.GetString('user.email')))
+                "Your identity is: %s <%s>"
-      print("If you want to change this, please re-run 'repo init' with --config-name")
+                % (
-    return False
+                    mp.config.GetString("user.name"),
+                    mp.config.GetString("user.email"),
+                )
+            )
+            print(
+                "If you want to change this, please re-run 'repo init' with "
+                "--config-name"
+            )
+        return False
 
-  def _ConfigureUser(self, opt):
+    def _ConfigureUser(self, opt):
-    mp = self.manifest.manifestProject
+        mp = self.manifest.manifestProject
+
+        while True:
+            if not opt.quiet:
+                print()
+            name = self._Prompt("Your Name", mp.UserName)
+            email = self._Prompt("Your Email", mp.UserEmail)
+
+            if not opt.quiet:
+                print()
+            print(f"Your identity is: {name} <{email}>")
+            print("is this correct [y/N]? ", end="", flush=True)
+            a = sys.stdin.readline().strip().lower()
+            if a in ("yes", "y", "t", "true"):
+                break
+
+        if name != mp.UserName:
+            mp.config.SetString("user.name", name)
+        if email != mp.UserEmail:
+            mp.config.SetString("user.email", email)
+
+    def _HasColorSet(self, gc):
+        for n in ["ui", "diff", "status"]:
+            if gc.Has("color.%s" % n):
+                return True
+        return False
+
+    def _ConfigureColor(self):
+        gc = self.client.globalConfig
+        if self._HasColorSet(gc):
+            return
+
+        class _Test(Coloring):
+            def __init__(self):
+                Coloring.__init__(self, gc, "test color display")
+                self._on = True
+
+        out = _Test()
 
-    while True:
-      if not opt.quiet:
         print()
-      name = self._Prompt('Your Name', mp.UserName)
+        print("Testing colorized output (for 'repo diff', 'repo status'):")
-      email = self._Prompt('Your Email', mp.UserEmail)
+
+        for c in ["black", "red", "green", "yellow", "blue", "magenta", "cyan"]:
+            out.write(" ")
+            out.printer(fg=c)(" %-6s ", c)
+        out.write(" ")
+        out.printer(fg="white", bg="black")(" %s " % "white")
+        out.nl()
+
+        for c in ["bold", "dim", "ul", "reverse"]:
+            out.write(" ")
+            out.printer(fg="black", attr=c)(" %-6s ", c)
+        out.nl()
+
+        print(
+            "Enable color display in this user account (y/N)? ",
+            end="",
+            flush=True,
+        )
+        a = sys.stdin.readline().strip().lower()
+        if a in ("y", "yes", "t", "true", "on"):
+            gc.SetString("color.ui", "auto")
+
+    def _DisplayResult(self):
+        if self.manifest.IsMirror:
+            init_type = "mirror "
+        else:
+            init_type = ""
 
-      if not opt.quiet:
         print()
-      print('Your identity is: %s <%s>' % (name, email))
+        print(
-      print('is this correct [y/N]? ', end='')
+            "repo %shas been initialized in %s"
-      # TODO: When we require Python 3, use flush=True w/print above.
+            % (init_type, self.manifest.topdir)
-      sys.stdout.flush()
+        )
-      a = sys.stdin.readline().strip().lower()
-      if a in ('yes', 'y', 't', 'true'):
-        break
 
-    if name != mp.UserName:
+        current_dir = os.getcwd()
-      mp.config.SetString('user.name', name)
+        if current_dir != self.manifest.topdir:
-    if email != mp.UserEmail:
+            print(
-      mp.config.SetString('user.email', email)
+                "If this is not the directory in which you want to initialize "
+                "repo, please run:"
+            )
+            print("   rm -r %s" % os.path.join(self.manifest.topdir, ".repo"))
+            print("and try again.")
 
-  def _HasColorSet(self, gc):
+    def ValidateOptions(self, opt, args):
-    for n in ['ui', 'diff', 'status']:
+        if opt.reference:
-      if gc.Has('color.%s' % n):
+            opt.reference = os.path.expanduser(opt.reference)
-        return True
-    return False
 
-  def _ConfigureColor(self):
+        # Check this here, else manifest will be tagged "not new" and init won't
-    gc = self.client.globalConfig
+        # be possible anymore without removing the .repo/manifests directory.
-    if self._HasColorSet(gc):
+        if opt.mirror:
-      return
+            if opt.archive:
+                self.OptionParser.error(
+                    "--mirror and --archive cannot be used " "together."
+                )
+            if opt.use_superproject is not None:
+                self.OptionParser.error(
+                    "--mirror and --use-superproject cannot be "
+                    "used together."
+                )
+        if opt.archive and opt.use_superproject is not None:
+            self.OptionParser.error(
+                "--archive and --use-superproject cannot be used " "together."
+            )
 
-    class _Test(Coloring):
+        if opt.standalone_manifest and (
-      def __init__(self):
+            opt.manifest_branch or opt.manifest_name != "default.xml"
-        Coloring.__init__(self, gc, 'test color display')
+        ):
-        self._on = True
+            self.OptionParser.error(
-    out = _Test()
+                "--manifest-branch and --manifest-name cannot"
+                " be used with --standalone-manifest."
+            )
 
-    print()
+        if opt.manifest_upstream_branch and opt.manifest_branch is None:
-    print("Testing colorized output (for 'repo diff', 'repo status'):")
+            self.OptionParser.error(
+                "--manifest-upstream-branch cannot be used without "
+                "--manifest-branch."
+            )
 
-    for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']:
+        if args:
-      out.write(' ')
+            if opt.manifest_url:
-      out.printer(fg=c)(' %-6s ', c)
+                self.OptionParser.error(
-    out.write(' ')
+                    "--manifest-url option and URL argument both specified: "
-    out.printer(fg='white', bg='black')(' %s ' % 'white')
+                    "only use one to select the manifest URL."
-    out.nl()
+                )
 
-    for c in ['bold', 'dim', 'ul', 'reverse']:
+            opt.manifest_url = args.pop(0)
-      out.write(' ')
-      out.printer(fg='black', attr=c)(' %-6s ', c)
-    out.nl()
 
-    print('Enable color display in this user account (y/N)? ', end='')
+            if args:
-    # TODO: When we require Python 3, use flush=True w/print above.
+                self.OptionParser.error("too many arguments to init")
-    sys.stdout.flush()
-    a = sys.stdin.readline().strip().lower()
-    if a in ('y', 'yes', 't', 'true', 'on'):
-      gc.SetString('color.ui', 'auto')
 
-  def _DisplayResult(self, opt):
+    def Execute(self, opt, args):
-    if self.manifest.IsMirror:
+        wrapper = Wrapper()
-      init_type = 'mirror '
-    else:
-      init_type = ''
 
-    if not opt.quiet:
+        reqs = wrapper.Requirements.from_dir(WrapperDir())
-      print()
+        git_require(reqs.get_hard_ver("git"), fail=True)
-      print('repo %shas been initialized in %s' %
+        min_git_version_soft = reqs.get_soft_ver("git")
-            (init_type, self.manifest.topdir))
+        if not git_require(min_git_version_soft):
+            logger.warning(
+                "repo: warning: git-%s+ will soon be required; "
+                "please upgrade your version of git to maintain "
+                "support.",
+                ".".join(str(x) for x in min_git_version_soft),
+            )
 
-    current_dir = os.getcwd()
+        rp = self.manifest.repoProject
-    if current_dir != self.manifest.topdir:
-      print('If this is not the directory in which you want to initialize '
-            'repo, please run:')
-      print('   rm -r %s/.repo' % self.manifest.topdir)
-      print('and try again.')
 
-  def ValidateOptions(self, opt, args):
+        # Handle new --repo-url requests.
-    if opt.reference:
+        if opt.repo_url:
-      opt.reference = os.path.expanduser(opt.reference)
+            remote = rp.GetRemote("origin")
+            remote.url = opt.repo_url
+            remote.Save()
 
-    # Check this here, else manifest will be tagged "not new" and init won't be
+        # Handle new --repo-rev requests.
-    # possible anymore without removing the .repo/manifests directory.
+        if opt.repo_rev:
-    if opt.mirror:
+            try:
-      if opt.archive:
+                remote_ref, rev = wrapper.check_repo_rev(
-        self.OptionParser.error('--mirror and --archive cannot be used '
+                    rp.worktree,
-                                'together.')
+                    opt.repo_rev,
-      if opt.use_superproject is not None:
+                    repo_verify=opt.repo_verify,
-        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                    quiet=opt.quiet,
-                                'used together.')
+                )
-    if opt.archive and opt.use_superproject is not None:
+            except wrapper.CloneFailure as e:
-      self.OptionParser.error('--archive and --use-superproject cannot be used '
+                err_msg = "fatal: double check your --repo-rev setting."
-                              'together.')
+                logger.error(err_msg)
+                self.git_event_log.ErrorEvent(err_msg)
+                raise RepoUnhandledExceptionError(e)
 
-    if opt.standalone_manifest and (opt.manifest_branch or
+            branch = rp.GetBranch("default")
-                                    opt.manifest_name != 'default.xml'):
+            branch.merge = remote_ref
-      self.OptionParser.error('--manifest-branch and --manifest-name cannot'
+            rp.work_git.reset("--hard", rev)
-                              ' be used with --standalone-manifest.')
+            branch.Save()
 
-    if args:
+        if opt.worktree:
-      if opt.manifest_url:
+            # Older versions of git supported worktree, but had dangerous gc
-        self.OptionParser.error(
+            # bugs.
-            '--manifest-url option and URL argument both specified: only use '
+            git_require((2, 15, 0), fail=True, msg="git gc worktree corruption")
-            'one to select the manifest URL.')
 
-      opt.manifest_url = args.pop(0)
+        # Provide a short notice that we're reinitializing an existing checkout.
+        # Sometimes developers might not realize that they're in one, or that
+        # repo doesn't do nested checkouts.
+        existing_checkout = self.manifest.manifestProject.Exists
+        if not opt.quiet and existing_checkout:
+            print(
+                "repo: reusing existing repo client checkout in",
+                self.manifest.topdir,
+            )
 
-      if args:
+        self._SyncManifest(opt)
-        self.OptionParser.error('too many arguments to init')
 
-  def Execute(self, opt, args):
+        if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-    git_require(MIN_GIT_VERSION_HARD, fail=True)
+            if opt.config_name or self._ShouldConfigureUser(
-    if not git_require(MIN_GIT_VERSION_SOFT):
+                opt, existing_checkout
-      print('repo: warning: git-%s+ will soon be required; please upgrade your '
+            ):
-            'version of git to maintain support.'
+                self._ConfigureUser(opt)
-            % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
+            self._ConfigureColor()
-            file=sys.stderr)
 
-    rp = self.manifest.repoProject
+        if not opt.quiet:
-
+            self._DisplayResult()
-    # Handle new --repo-url requests.
-    if opt.repo_url:
-      remote = rp.GetRemote('origin')
-      remote.url = opt.repo_url
-      remote.Save()
-
-    # Handle new --repo-rev requests.
-    if opt.repo_rev:
-      wrapper = Wrapper()
-      try:
-        remote_ref, rev = wrapper.check_repo_rev(
-            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
-      except wrapper.CloneFailure:
-        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
-        sys.exit(1)
-      branch = rp.GetBranch('default')
-      branch.merge = remote_ref
-      rp.work_git.reset('--hard', rev)
-      branch.Save()
-
-    if opt.worktree:
-      # Older versions of git supported worktree, but had dangerous gc bugs.
-      git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
-
-    self._SyncManifest(opt)
-
-    if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      if opt.config_name or self._ShouldConfigureUser(opt):
-        self._ConfigureUser(opt)
-      self._ConfigureColor()
-
-    self._DisplayResult(opt)

subcmds/list.py

@@ -14,17 +14,18 @@
 
 import os
 
-from command import Command, MirrorSafeCommand
+from command import Command
+from command import MirrorSafeCommand
 
 
 class List(Command, MirrorSafeCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "List projects and their associated directories"
+    helpSummary = "List projects and their associated directories"
-  helpUsage = """
+    helpUsage = """
 %prog [-f] [<project>...]
 %prog [-f] -r str1 [str2]...
 """
-  helpDescription = """
+    helpDescription = """
 List all projects; pass '.' to list the project for the cwd.
 
 By default, only projects that currently exist in the checkout are shown.  If
@@ -35,69 +36,98 @@ groups, then also pass --groups all.
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-r', '--regex',
+        p.add_option(
-                 dest='regex', action='store_true',
+            "-r",
-                 help='filter the project list based on regex or wildcard matching of strings')
+            "--regex",
-    p.add_option('-g', '--groups',
+            action="store_true",
-                 dest='groups',
+            help="filter the project list based on regex or wildcard matching "
-                 help='filter the project list based on the groups the project is in')
+            "of strings",
-    p.add_option('-a', '--all',
+        )
-                 action='store_true',
+        p.add_option(
-                 help='show projects regardless of checkout state')
+            "-g",
-    p.add_option('-n', '--name-only',
+            "--groups",
-                 dest='name_only', action='store_true',
+            help="filter the project list based on the groups the project is "
-                 help='display only the name of the repository')
+            "in",
-    p.add_option('-p', '--path-only',
+        )
-                 dest='path_only', action='store_true',
+        p.add_option(
-                 help='display only the path of the repository')
+            "-a",
-    p.add_option('-f', '--fullpath',
+            "--all",
-                 dest='fullpath', action='store_true',
+            action="store_true",
-                 help='display the full work tree path instead of the relative path')
+            help="show projects regardless of checkout state",
-    p.add_option('--relative-to', metavar='PATH',
+        )
-                 help='display paths relative to this one (default: top of repo client checkout)')
+        p.add_option(
+            "-n",
+            "--name-only",
+            action="store_true",
+            help="display only the name of the repository",
+        )
+        p.add_option(
+            "-p",
+            "--path-only",
+            action="store_true",
+            help="display only the path of the repository",
+        )
+        p.add_option(
+            "-f",
+            "--fullpath",
+            action="store_true",
+            help="display the full work tree path instead of the relative path",
+        )
+        p.add_option(
+            "--relative-to",
+            metavar="PATH",
+            help="display paths relative to this one (default: top of repo "
+            "client checkout)",
+        )
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if opt.fullpath and opt.name_only:
+        if opt.fullpath and opt.name_only:
-      self.OptionParser.error('cannot combine -f and -n')
+            self.OptionParser.error("cannot combine -f and -n")
 
-    # Resolve any symlinks so the output is stable.
+        # Resolve any symlinks so the output is stable.
-    if opt.relative_to:
+        if opt.relative_to:
-      opt.relative_to = os.path.realpath(opt.relative_to)
+            opt.relative_to = os.path.realpath(opt.relative_to)
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    """List all projects and the associated directories.
+        """List all projects and the associated directories.
 
-    This may be possible to do with 'repo forall', but repo newbies have
+        This may be possible to do with 'repo forall', but repo newbies have
-    trouble figuring that out.  The idea here is that it should be more
+        trouble figuring that out.  The idea here is that it should be more
-    discoverable.
+        discoverable.
 
-    Args:
+        Args:
-      opt: The options.
+            opt: The options.
-      args: Positional args.  Can be a list of projects to list, or empty.
+            args: Positional args.  Can be a list of projects to list, or empty.
-    """
+        """
-    if not opt.regex:
+        if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all,
+            projects = self.GetProjects(
-                                  all_manifests=not opt.this_manifest_only)
+                args,
-    else:
+                groups=opt.groups,
-      projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only)
+                missing_ok=opt.all,
+                all_manifests=not opt.this_manifest_only,
+            )
+        else:
+            projects = self.FindProjects(
+                args, all_manifests=not opt.this_manifest_only
+            )
 
-    def _getpath(x):
+        def _getpath(x):
-      if opt.fullpath:
+            if opt.fullpath:
-        return x.worktree
+                return x.worktree
-      if opt.relative_to:
+            if opt.relative_to:
-        return os.path.relpath(x.worktree, opt.relative_to)
+                return os.path.relpath(x.worktree, opt.relative_to)
-      return x.RelPath(local=opt.this_manifest_only)
+            return x.RelPath(local=opt.this_manifest_only)
 
-    lines = []
+        lines = []
-    for project in projects:
+        for project in projects:
-      if opt.name_only and not opt.path_only:
+            if opt.name_only and not opt.path_only:
-        lines.append("%s" % (project.name))
+                lines.append("%s" % (project.name))
-      elif opt.path_only and not opt.name_only:
+            elif opt.path_only and not opt.name_only:
-        lines.append("%s" % (_getpath(project)))
+                lines.append("%s" % (_getpath(project)))
-      else:
+            else:
-        lines.append("%s : %s" % (_getpath(project), project.name))
+                lines.append(f"{_getpath(project)} : {project.name}")
 
-    if lines:
+        if lines:
-      lines.sort()
+            lines.sort()
-      print('\n'.join(lines))
+            print("\n".join(lines))

subcmds/manifest.py

@@ -12,21 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import enum
 import json
+import optparse
 import os
 import sys
-import optparse
 
 from command import PagedCommand
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class OutputFormat(enum.Enum):
+    """Type for the requested output format."""
+
+    # Canonicalized manifest in XML format.
+    XML = enum.auto()
+
+    # Canonicalized manifest in JSON format.
+    JSON = enum.auto()
 
 
 class Manifest(PagedCommand):
-  COMMON = False
+    COMMON = False
-  helpSummary = "Manifest inspection utility"
+    helpSummary = "Manifest inspection utility"
-  helpUsage = """
+    helpUsage = """
 %prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r]
 """
-  _helpDescription = """
+    _helpDescription = """
 
 With the -o option, exports the current manifest for inspection.
 The manifest and (if present) local_manifests/ are combined
@@ -39,94 +54,152 @@ revisions set to the current commit hash.  These are known as
 In this case, the 'upstream' attribute is set to the ref we were on
 when the manifest was generated.  The 'dest-branch' attribute is set
 to indicate the remote ref to push changes to via 'repo upload'.
+
+Multiple output formats are supported via --format.  The default output
+is XML, and formats are generally "condensed".  Use --pretty for more
+human-readable variations.
 """
 
-  @property
+    @property
-  def helpDescription(self):
+    def helpDescription(self):
-    helptext = self._helpDescription + '\n'
+        helptext = self._helpDescription + "\n"
-    r = os.path.dirname(__file__)
+        r = os.path.dirname(__file__)
-    r = os.path.dirname(r)
+        r = os.path.dirname(r)
-    with open(os.path.join(r, 'docs', 'manifest-format.md')) as fd:
+        with open(os.path.join(r, "docs", "manifest-format.md")) as fd:
-      for line in fd:
+            for line in fd:
-        helptext += line
+                helptext += line
-    return helptext
+        return helptext
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-r', '--revision-as-HEAD',
+        p.add_option(
-                 dest='peg_rev', action='store_true',
+            "-r",
-                 help='save revisions as current HEAD')
+            "--revision-as-HEAD",
-    p.add_option('-m', '--manifest-name',
+            dest="peg_rev",
-                 help='temporary manifest to use for this sync', metavar='NAME.xml')
+            action="store_true",
-    p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream',
+            help="save revisions as current HEAD",
-                 default=True, action='store_false',
+        )
-                 help='if in -r mode, do not write the upstream field '
+        p.add_option(
-                 '(only of use if the branch names for a sha1 manifest are '
+            "-m",
-                 'sensitive)')
+            "--manifest-name",
-    p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch',
+            help="temporary manifest to use for this sync",
-                 default=True, action='store_false',
+            metavar="NAME.xml",
-                 help='if in -r mode, do not write the dest-branch field '
+        )
-                 '(only of use if the branch names for a sha1 manifest are '
+        p.add_option(
-                 'sensitive)')
+            "--suppress-upstream-revision",
-    p.add_option('--json', default=False, action='store_true',
+            dest="peg_rev_upstream",
-                 help='output manifest in JSON format (experimental)')
+            default=True,
-    p.add_option('--pretty', default=False, action='store_true',
+            action="store_false",
-                 help='format output for humans to read')
+            help="if in -r mode, do not write the upstream field "
-    p.add_option('--no-local-manifests', default=False, action='store_true',
+            "(only of use if the branch names for a sha1 manifest are "
-                 dest='ignore_local_manifests', help='ignore local manifests')
+            "sensitive)",
-    p.add_option('-o', '--output-file',
+        )
-                 dest='output_file',
+        p.add_option(
-                 default='-',
+            "--suppress-dest-branch",
-                 help='file to save the manifest to. (Filename prefix for multi-tree.)',
+            dest="peg_rev_dest_branch",
-                 metavar='-|NAME.xml')
+            default=True,
+            action="store_false",
+            help="if in -r mode, do not write the dest-branch field "
+            "(only of use if the branch names for a sha1 manifest are "
+            "sensitive)",
+        )
+        # Replaced with --format=json.  Kept for backwards compatibility.
+        # Can delete in Jun 2026 or later.
+        p.add_option(
+            "--json",
+            action="store_const",
+            dest="format",
+            const=OutputFormat.JSON.name.lower(),
+            help=optparse.SUPPRESS_HELP,
+        )
+        formats = tuple(x.lower() for x in OutputFormat.__members__.keys())
+        p.add_option(
+            "--format",
+            default=OutputFormat.XML.name.lower(),
+            choices=formats,
+            help=f"output format: {', '.join(formats)} (default: %default)",
+        )
+        p.add_option(
+            "--pretty",
+            default=False,
+            action="store_true",
+            help="format output for humans to read",
+        )
+        p.add_option(
+            "--no-local-manifests",
+            default=False,
+            action="store_true",
+            dest="ignore_local_manifests",
+            help="ignore local manifests",
+        )
+        p.add_option(
+            "-o",
+            "--output-file",
+            default="-",
+            help="file to save the manifest to. (Filename prefix for "
+            "multi-tree.)",
+            metavar="-|NAME.xml",
+        )
 
-  def _Output(self, opt):
+    def _Output(self, opt):
-    # If alternate manifest is specified, override the manifest file that we're using.
+        # If alternate manifest is specified, override the manifest file that
-    if opt.manifest_name:
+        # we're using.
-      self.manifest.Override(opt.manifest_name, False)
+        if opt.manifest_name:
+            self.manifest.Override(opt.manifest_name, False)
 
-    for manifest in self.ManifestList(opt):
+        output_format = OutputFormat[opt.format.upper()]
-      output_file = opt.output_file
-      if output_file == '-':
-        fd = sys.stdout
-      else:
-        if manifest.path_prefix:
-          output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}'
-        fd = open(output_file, 'w')
 
-      manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
+        for manifest in self.ManifestList(opt):
+            output_file = opt.output_file
+            if output_file == "-":
+                fd = sys.stdout
+            else:
+                if manifest.path_prefix:
+                    output_file = (
+                        f"{opt.output_file}:"
+                        f'{manifest.path_prefix.replace("/", "%2f")}'
+                    )
+                fd = open(output_file, "w")
 
-      if opt.json:
+            manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
-        print('warning: --json is experimental!', file=sys.stderr)
-        doc = manifest.ToDict(peg_rev=opt.peg_rev,
-                                   peg_rev_upstream=opt.peg_rev_upstream,
-                                   peg_rev_dest_branch=opt.peg_rev_dest_branch)
 
-        json_settings = {
+            if output_format == OutputFormat.JSON:
-            # JSON style guide says Uunicode characters are fully allowed.
+                doc = manifest.ToDict(
-            'ensure_ascii': False,
+                    peg_rev=opt.peg_rev,
-            # We use 2 space indent to match JSON style guide.
+                    peg_rev_upstream=opt.peg_rev_upstream,
-            'indent': 2 if opt.pretty else None,
+                    peg_rev_dest_branch=opt.peg_rev_dest_branch,
-            'separators': (',', ': ') if opt.pretty else (',', ':'),
+                )
-            'sort_keys': True,
-        }
-        fd.write(json.dumps(doc, **json_settings))
-      else:
-        manifest.Save(fd,
-                      peg_rev=opt.peg_rev,
-                      peg_rev_upstream=opt.peg_rev_upstream,
-                      peg_rev_dest_branch=opt.peg_rev_dest_branch)
-      if output_file != '-':
-        fd.close()
-        if manifest.path_prefix:
-          print(f'Saved {manifest.path_prefix} submanifest to {output_file}',
-                file=sys.stderr)
-        else:
-          print(f'Saved manifest to {output_file}', file=sys.stderr)
 
+                json_settings = {
+                    # JSON style guide says Unicode characters are fully
+                    # allowed.
+                    "ensure_ascii": False,
+                    # We use 2 space indent to match JSON style guide.
+                    "indent": 2 if opt.pretty else None,
+                    "separators": (",", ": ") if opt.pretty else (",", ":"),
+                    "sort_keys": True,
+                }
+                fd.write(json.dumps(doc, **json_settings) + "\n")
+            else:
+                manifest.Save(
+                    fd,
+                    peg_rev=opt.peg_rev,
+                    peg_rev_upstream=opt.peg_rev_upstream,
+                    peg_rev_dest_branch=opt.peg_rev_dest_branch,
+                )
+            if output_file != "-":
+                fd.close()
+                if manifest.path_prefix:
+                    logger.warning(
+                        "Saved %s submanifest to %s",
+                        manifest.path_prefix,
+                        output_file,
+                    )
+                else:
+                    logger.warning("Saved manifest to %s", output_file)
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if args:
+        if args:
-      self.Usage()
+            self.Usage()
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    self._Output(opt)
+        self._Output(opt)

subcmds/overview.py

@@ -19,12 +19,12 @@ from command import PagedCommand
 
 
 class Overview(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Display overview of unmerged project branches"
+    helpSummary = "Display overview of unmerged project branches"
-  helpUsage = """
+    helpUsage = """
 %prog [--current-branch] [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command is used to display an overview of the projects branches,
 and list any local commits that have not yet been merged into the project.
 
@@ -33,59 +33,76 @@ branches currently checked out in each project.  By default, all branches
 are displayed.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-c', '--current-branch',
+        p.add_option(
-                 dest="current_branch", action="store_true",
+            "-c",
-                 help="consider only checked out branches")
+            "--current-branch",
-    p.add_option('--no-current-branch',
+            action="store_true",
-                 dest='current_branch', action='store_false',
+            help="consider only checked out branches",
-                 help='consider all local branches')
+        )
-    # Turn this into a warning & remove this someday.
+        p.add_option(
-    p.add_option('-b',
+            "--no-current-branch",
-                 dest='current_branch', action='store_true',
+            dest="current_branch",
-                 help=optparse.SUPPRESS_HELP)
+            action="store_false",
+            help="consider all local branches",
+        )
+        # Turn this into a warning & remove this someday.
+        p.add_option(
+            "-b",
+            dest="current_branch",
+            action="store_true",
+            help=optparse.SUPPRESS_HELP,
+        )
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    all_branches = []
+        all_branches = []
-    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
+        for project in self.GetProjects(
-      br = [project.GetUploadableBranch(x)
+            args, all_manifests=not opt.this_manifest_only
-            for x in project.GetBranches()]
+        ):
-      br = [x for x in br if x]
+            br = [project.GetUploadableBranch(x) for x in project.GetBranches()]
-      if opt.current_branch:
+            br = [x for x in br if x]
-        br = [x for x in br if x.name == project.CurrentBranch]
+            if opt.current_branch:
-      all_branches.extend(br)
+                br = [x for x in br if x.name == project.CurrentBranch]
+            all_branches.extend(br)
 
-    if not all_branches:
+        if not all_branches:
-      return
+            return
 
-    class Report(Coloring):
+        class Report(Coloring):
-      def __init__(self, config):
+            def __init__(self, config):
-        Coloring.__init__(self, config, 'status')
+                Coloring.__init__(self, config, "status")
-        self.project = self.printer('header', attr='bold')
+                self.project = self.printer("header", attr="bold")
-        self.text = self.printer('text')
+                self.text = self.printer("text")
 
-    out = Report(all_branches[0].project.config)
+        out = Report(all_branches[0].project.config)
-    out.text("Deprecated. See repo info -o.")
+        out.text("Deprecated. See repo info -o.")
-    out.nl()
-    out.project('Projects Overview')
-    out.nl()
-
-    project = None
-
-    for branch in all_branches:
-      if project != branch.project:
-        project = branch.project
         out.nl()
-        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
+        out.project("Projects Overview")
         out.nl()
 
-      commits = branch.commits
+        project = None
-      date = branch.date
+
-      print('%s %-33s (%2d commit%s, %s)' % (
+        for branch in all_branches:
-            branch.name == project.CurrentBranch and '*' or ' ',
+            if project != branch.project:
-            branch.name,
+                project = branch.project
-            len(commits),
+                out.nl()
-            len(commits) != 1 and 's' or ' ',
+                out.project(
-            date))
+                    "project %s/"
-      for commit in commits:
+                    % project.RelPath(local=opt.this_manifest_only)
-        print('%-35s   - %s' % ('', commit))
+                )
+                out.nl()
+
+            commits = branch.commits
+            date = branch.date
+            print(
+                "%s %-33s (%2d commit%s, %s)"
+                % (
+                    branch.name == project.CurrentBranch and "*" or " ",
+                    branch.name,
+                    len(commits),
+                    len(commits) != 1 and "s" or " ",
+                    date,
+                )
+            )
+            for commit in commits:
+                print("%-35s   - %s" % ("", commit))

subcmds/prune.py

@@ -15,67 +15,83 @@
 import itertools
 
 from color import Coloring
-from command import DEFAULT_LOCAL_JOBS, PagedCommand
+from command import DEFAULT_LOCAL_JOBS
+from command import PagedCommand
 
 
 class Prune(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Prune (delete) already merged topics"
+    helpSummary = "Prune (delete) already merged topics"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...]
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def _ExecuteOne(self, project):
+    @classmethod
-    """Process one project."""
+    def _ExecuteOne(cls, project_idx):
-    return project.PruneHeads()
+        """Process one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
+        return project.PruneHeads()
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
 
-    # NB: Should be able to refactor this module to display summary as results
+        # NB: Should be able to refactor this module to display summary as
-    # come back from children.
+        # results come back from children.
-    def _ProcessResults(_pool, _output, results):
+        def _ProcessResults(_pool, _output, results):
-      return list(itertools.chain.from_iterable(results))
+            return list(itertools.chain.from_iterable(results))
 
-    all_branches = self.ExecuteInParallel(
+        with self.ParallelContext():
-        opt.jobs,
+            self.get_parallel_context()["projects"] = projects
-        self._ExecuteOne,
+            all_branches = self.ExecuteInParallel(
-        projects,
+                opt.jobs,
-        callback=_ProcessResults,
+                self._ExecuteOne,
-        ordered=True)
+                range(len(projects)),
+                callback=_ProcessResults,
+                ordered=True,
+            )
 
-    if not all_branches:
+        if not all_branches:
-      return
+            return
 
-    class Report(Coloring):
+        class Report(Coloring):
-      def __init__(self, config):
+            def __init__(self, config):
-        Coloring.__init__(self, config, 'status')
+                Coloring.__init__(self, config, "status")
-        self.project = self.printer('header', attr='bold')
+                self.project = self.printer("header", attr="bold")
 
-    out = Report(all_branches[0].project.config)
+        out = Report(all_branches[0].project.config)
-    out.project('Pending Branches')
+        out.project("Pending Branches")
-    out.nl()
-
-    project = None
-
-    for branch in all_branches:
-      if project != branch.project:
-        project = branch.project
-        out.nl()
-        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
-      print('%s %-33s ' % (
+        project = None
-            branch.name == project.CurrentBranch and '*' or ' ',
-            branch.name), end='')
 
-      if not branch.base_exists:
+        for branch in all_branches:
-        print('(ignoring: tracking branch is gone: %s)' % (branch.base,))
+            if project != branch.project:
-      else:
+                project = branch.project
-        commits = branch.commits
+                out.nl()
-        date = branch.date
+                out.project(
-        print('(%2d commit%s, %s)' % (
+                    "project %s/"
-            len(commits),
+                    % project.RelPath(local=opt.this_manifest_only)
-            len(commits) != 1 and 's' or ' ',
+                )
-            date))
+                out.nl()
+
+            print(
+                "%s %-33s "
+                % (
+                    branch.name == project.CurrentBranch and "*" or " ",
+                    branch.name,
+                ),
+                end="",
+            )
+
+            if not branch.base_exists:
+                print(f"(ignoring: tracking branch is gone: {branch.base})")
+            else:
+                commits = branch.commits
+                date = branch.date
+                print(
+                    "(%2d commit%s, %s)"
+                    % (len(commits), len(commits) != 1 and "s" or " ", date)
+                )

subcmds/rebase.py

@@ -17,149 +17,191 @@ import sys
 from color import Coloring
 from command import Command
 from git_command import GitCommand
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
 
 
 class RebaseColoring(Coloring):
-  def __init__(self, config):
+    def __init__(self, config):
-    Coloring.__init__(self, config, 'rebase')
+        Coloring.__init__(self, config, "rebase")
-    self.project = self.printer('project', attr='bold')
+        self.project = self.printer("project", attr="bold")
-    self.fail = self.printer('fail', fg='red')
+        self.fail = self.printer("fail", fg="red")
 
 
 class Rebase(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Rebase local branches on upstream branch"
+    helpSummary = "Rebase local branches on upstream branch"
-  helpUsage = """
+    helpUsage = """
 %prog {[<project>...] | -i <project>...}
 """
-  helpDescription = """
+    helpDescription = """
 '%prog' uses git rebase to move local changes in the current topic branch to
 the HEAD of the upstream history, useful when you have made commits in a topic
 branch but need to incorporate new upstream changes "underneath" them.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    g = p.get_option_group('--quiet')
+        g = p.get_option_group("--quiet")
-    g.add_option('-i', '--interactive',
+        g.add_option(
-                 dest="interactive", action="store_true",
+            "-i",
-                 help="interactive rebase (single project only)")
+            "--interactive",
+            action="store_true",
+            help="interactive rebase (single project only)",
+        )
 
-    p.add_option('--fail-fast',
+        p.add_option(
-                 dest='fail_fast', action='store_true',
+            "--fail-fast",
-                 help='stop rebasing after first error is hit')
+            action="store_true",
-    p.add_option('-f', '--force-rebase',
+            help="stop rebasing after first error is hit",
-                 dest='force_rebase', action='store_true',
+        )
-                 help='pass --force-rebase to git rebase')
+        p.add_option(
-    p.add_option('--no-ff',
+            "-f",
-                 dest='ff', default=True, action='store_false',
+            "--force-rebase",
-                 help='pass --no-ff to git rebase')
+            action="store_true",
-    p.add_option('--autosquash',
+            help="pass --force-rebase to git rebase",
-                 dest='autosquash', action='store_true',
+        )
-                 help='pass --autosquash to git rebase')
+        p.add_option(
-    p.add_option('--whitespace',
+            "--no-ff",
-                 dest='whitespace', action='store', metavar='WS',
+            dest="ff",
-                 help='pass --whitespace to git rebase')
+            default=True,
-    p.add_option('--auto-stash',
+            action="store_false",
-                 dest='auto_stash', action='store_true',
+            help="pass --no-ff to git rebase",
-                 help='stash local modifications before starting')
+        )
-    p.add_option('-m', '--onto-manifest',
+        p.add_option(
-                 dest='onto_manifest', action='store_true',
+            "--autosquash",
-                 help='rebase onto the manifest version instead of upstream '
+            action="store_true",
-                      'HEAD (this helps to make sure the local tree stays '
+            help="pass --autosquash to git rebase",
-                      'consistent if you previously synced to a manifest)')
+        )
+        p.add_option(
+            "--whitespace",
+            action="store",
+            metavar="WS",
+            help="pass --whitespace to git rebase",
+        )
+        p.add_option(
+            "--auto-stash",
+            action="store_true",
+            help="stash local modifications before starting",
+        )
+        p.add_option(
+            "-m",
+            "--onto-manifest",
+            action="store_true",
+            help="rebase onto the manifest version instead of upstream "
+            "HEAD (this helps to make sure the local tree stays "
+            "consistent if you previously synced to a manifest)",
+        )
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        all_projects = self.GetProjects(
-    one_project = len(all_projects) == 1
+            args, all_manifests=not opt.this_manifest_only
+        )
+        one_project = len(all_projects) == 1
 
-    if opt.interactive and not one_project:
+        if opt.interactive and not one_project:
-      print('error: interactive rebase not supported with multiple projects',
+            logger.error(
-            file=sys.stderr)
+                "error: interactive rebase not supported with multiple projects"
-      if len(args) == 1:
+            )
-        print('note: project %s is mapped to more than one path' % (args[0],),
-              file=sys.stderr)
-      return 1
 
-    # Setup the common git rebase args that we use for all projects.
+            if len(args) == 1:
-    common_args = ['rebase']
+                logger.warning(
-    if opt.whitespace:
+                    "note: project %s is mapped to more than one path", args[0]
-      common_args.append('--whitespace=%s' % opt.whitespace)
+                )
-    if opt.quiet:
-      common_args.append('--quiet')
-    if opt.force_rebase:
-      common_args.append('--force-rebase')
-    if not opt.ff:
-      common_args.append('--no-ff')
-    if opt.autosquash:
-      common_args.append('--autosquash')
-    if opt.interactive:
-      common_args.append('-i')
 
-    config = self.manifest.manifestProject.config
+            return 1
-    out = RebaseColoring(config)
-    out.redirect(sys.stdout)
-    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
-    ret = 0
+        # Setup the common git rebase args that we use for all projects.
-    for project in all_projects:
+        common_args = ["rebase"]
-      if ret and opt.fail_fast:
+        if opt.whitespace:
-        break
+            common_args.append("--whitespace=%s" % opt.whitespace)
+        if opt.quiet:
+            common_args.append("--quiet")
+        if opt.force_rebase:
+            common_args.append("--force-rebase")
+        if not opt.ff:
+            common_args.append("--no-ff")
+        if opt.autosquash:
+            common_args.append("--autosquash")
+        if opt.interactive:
+            common_args.append("-i")
 
-      cb = project.CurrentBranch
+        config = self.manifest.manifestProject.config
-      if not cb:
+        out = RebaseColoring(config)
-        if one_project:
+        out.redirect(sys.stdout)
-          print("error: project %s has a detached HEAD" % _RelPath(project),
+        _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
-                file=sys.stderr)
-          return 1
-        # ignore branches with detatched HEADs
-        continue
 
-      upbranch = project.GetBranch(cb)
+        ret = 0
-      if not upbranch.LocalMerge:
+        for project in all_projects:
-        if one_project:
+            if ret and opt.fail_fast:
-          print("error: project %s does not track any remote branches"
+                break
-                % _RelPath(project), file=sys.stderr)
-          return 1
-        # ignore branches without remotes
-        continue
 
-      args = common_args[:]
+            cb = project.CurrentBranch
-      if opt.onto_manifest:
+            if not cb:
-        args.append('--onto')
+                if one_project:
-        args.append(project.revisionExpr)
+                    logger.error(
+                        "error: project %s has a detached HEAD",
+                        _RelPath(project),
+                    )
+                    return 1
+                # Ignore branches with detached HEADs.
+                continue
 
-      args.append(upbranch.LocalMerge)
+            upbranch = project.GetBranch(cb)
+            if not upbranch.LocalMerge:
+                if one_project:
+                    logger.error(
+                        "error: project %s does not track any remote branches",
+                        _RelPath(project),
+                    )
+                    return 1
+                # Ignore branches without remotes.
+                continue
 
-      out.project('project %s: rebasing %s -> %s',
+            args = common_args[:]
-                  _RelPath(project), cb, upbranch.LocalMerge)
+            if opt.onto_manifest:
-      out.nl()
+                args.append("--onto")
-      out.flush()
+                args.append(project.revisionExpr)
 
-      needs_stash = False
+            args.append(upbranch.LocalMerge)
-      if opt.auto_stash:
-        stash_args = ["update-index", "--refresh", "-q"]
 
-        if GitCommand(project, stash_args).Wait() != 0:
+            out.project(
-          needs_stash = True
+                "project %s: rebasing %s -> %s",
-          # Dirty index, requires stash...
+                _RelPath(project),
-          stash_args = ["stash"]
+                cb,
+                upbranch.LocalMerge,
+            )
+            out.nl()
+            out.flush()
 
-          if GitCommand(project, stash_args).Wait() != 0:
+            needs_stash = False
-            ret += 1
+            if opt.auto_stash:
-            continue
+                stash_args = ["update-index", "--refresh", "-q"]
 
-      if GitCommand(project, args).Wait() != 0:
+                if GitCommand(project, stash_args).Wait() != 0:
-        ret += 1
+                    needs_stash = True
-        continue
+                    # Dirty index, requires stash...
+                    stash_args = ["stash"]
 
-      if needs_stash:
+                    if GitCommand(project, stash_args).Wait() != 0:
-        stash_args.append('pop')
+                        ret += 1
-        stash_args.append('--quiet')
+                        continue
-        if GitCommand(project, stash_args).Wait() != 0:
-          ret += 1
 
-    if ret:
+            if GitCommand(project, args).Wait() != 0:
-      out.fail('%i projects had errors', ret)
+                ret += 1
-      out.nl()
+                continue
 
-    return ret
+            if needs_stash:
+                stash_args.append("pop")
+                stash_args.append("--quiet")
+                if GitCommand(project, stash_args).Wait() != 0:
+                    ret += 1
+
+        if ret:
+            msg_fmt = "%d projects had errors"
+            self.git_event_log.ErrorEvent(msg_fmt % (ret), msg_fmt)
+            out.fail(msg_fmt, ret)
+            out.nl()
+
+        return ret

subcmds/selfupdate.py

@@ -12,21 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from optparse import SUPPRESS_HELP
+import optparse
-import sys
 
-from command import Command, MirrorSafeCommand
+from command import Command
-from subcmds.sync import _PostRepoUpgrade
+from command import MirrorSafeCommand
+from error import RepoExitError
+from repo_logging import RepoLogger
 from subcmds.sync import _PostRepoFetch
+from subcmds.sync import _PostRepoUpgrade
+
+
+logger = RepoLogger(__file__)
+
+
+class SelfupdateError(RepoExitError):
+    """Exit error for failed selfupdate command."""
 
 
 class Selfupdate(Command, MirrorSafeCommand):
-  COMMON = False
+    COMMON = False
-  helpSummary = "Update repo to the latest version"
+    helpSummary = "Update repo to the latest version"
-  helpUsage = """
+    helpUsage = """
 %prog
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command upgrades repo to the latest version, if a
 newer version is available.
 
@@ -34,28 +43,33 @@ Normally this is done automatically by 'repo sync' and does not
 need to be performed by an end-user.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    g = p.add_option_group('repo Version options')
+        g = p.add_option_group("repo Version options")
-    g.add_option('--no-repo-verify',
+        g.add_option(
-                 dest='repo_verify', default=True, action='store_false',
+            "--no-repo-verify",
-                 help='do not verify repo source code')
+            dest="repo_verify",
-    g.add_option('--repo-upgraded',
+            default=True,
-                 dest='repo_upgraded', action='store_true',
+            action="store_false",
-                 help=SUPPRESS_HELP)
+            help="do not verify repo source code",
+        )
+        g.add_option(
+            "--repo-upgraded",
+            action="store_true",
+            help=optparse.SUPPRESS_HELP,
+        )
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    rp = self.manifest.repoProject
+        rp = self.manifest.repoProject
-    rp.PreSync()
+        rp.PreSync()
 
-    if opt.repo_upgraded:
+        if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest)
+            _PostRepoUpgrade(self.manifest)
 
-    else:
+        else:
-      if not rp.Sync_NetworkHalf():
+            result = rp.Sync_NetworkHalf()
-        print("error: can't update repo", file=sys.stderr)
+            if result.error:
-        sys.exit(1)
+                logger.error("error: can't update repo")
+                raise SelfupdateError(aggregate_errors=[result.error])
 
-      rp.bare_git.gc('--auto')
+            rp.bare_git.gc("--auto")
-      _PostRepoFetch(rp,
+            _PostRepoFetch(rp, repo_verify=opt.repo_verify, verbose=True)
-                     repo_verify=opt.repo_verify,
-                     verbose=True)

subcmds/smartsync.py

@@ -16,18 +16,18 @@ from subcmds.sync import Sync
 
 
 class Smartsync(Sync):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Update working tree to the latest known good revision"
+    helpSummary = "Update working tree to the latest known good revision"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command is a shortcut for sync -s.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    Sync._Options(self, p, show_smart=False)
+        Sync._Options(self, p, show_smart=False)
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    opt.smart_sync = True
+        opt.smart_sync = True
-    Sync.Execute(self, opt, args)
+        Sync.Execute(self, opt, args)

subcmds/stage.py

@@ -17,100 +17,117 @@ import sys
 from color import Coloring
 from command import InteractiveCommand
 from git_command import GitCommand
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
 
 
 class _ProjectList(Coloring):
-  def __init__(self, gc):
+    def __init__(self, gc):
-    Coloring.__init__(self, gc, 'interactive')
+        Coloring.__init__(self, gc, "interactive")
-    self.prompt = self.printer('prompt', fg='blue', attr='bold')
+        self.prompt = self.printer("prompt", fg="blue", attr="bold")
-    self.header = self.printer('header', attr='bold')
+        self.header = self.printer("header", attr="bold")
-    self.help = self.printer('help', fg='red', attr='bold')
+        self.help = self.printer("help", fg="red", attr="bold")
 
 
 class Stage(InteractiveCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Stage file(s) for commit"
+    helpSummary = "Stage file(s) for commit"
-  helpUsage = """
+    helpUsage = """
 %prog -i [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 The '%prog' command stages files to prepare the next commit.
 """
 
-  def _Options(self, p):
+    def _Options(self, p):
-    g = p.get_option_group('--quiet')
+        g = p.get_option_group("--quiet")
-    g.add_option('-i', '--interactive',
+        g.add_option(
-                 dest='interactive', action='store_true',
+            "-i",
-                 help='use interactive staging')
+            "--interactive",
+            action="store_true",
+            help="use interactive staging",
+        )
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    if opt.interactive:
+        if opt.interactive:
-      self._Interactive(opt, args)
+            self._Interactive(opt, args)
-    else:
+        else:
-      self.Usage()
+            self.Usage()
 
-  def _Interactive(self, opt, args):
+    def _Interactive(self, opt, args):
-    all_projects = [
+        all_projects = [
-        p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+            p
-        if p.IsDirty()]
+            for p in self.GetProjects(
-    if not all_projects:
+                args, all_manifests=not opt.this_manifest_only
-      print('no projects have uncommitted modifications', file=sys.stderr)
+            )
-      return
+            if p.IsDirty()
+        ]
+        if not all_projects:
+            logger.error("no projects have uncommitted modifications")
+            return
 
-    out = _ProjectList(self.manifest.manifestProject.config)
+        out = _ProjectList(self.manifest.manifestProject.config)
-    while True:
+        while True:
-      out.header('        %s', 'project')
+            out.header("        %s", "project")
-      out.nl()
+            out.nl()
 
-      for i in range(len(all_projects)):
+            for i in range(len(all_projects)):
-        project = all_projects[i]
+                project = all_projects[i]
-        out.write('%3d:    %s', i + 1,
+                out.write(
-                  project.RelPath(local=opt.this_manifest_only) + '/')
+                    "%3d:    %s",
-        out.nl()
+                    i + 1,
-      out.nl()
+                    project.RelPath(local=opt.this_manifest_only) + "/",
+                )
+                out.nl()
+            out.nl()
 
-      out.write('%3d: (', 0)
+            out.write("%3d: (", 0)
-      out.prompt('q')
+            out.prompt("q")
-      out.write('uit)')
+            out.write("uit)")
-      out.nl()
+            out.nl()
 
-      out.prompt('project> ')
+            out.prompt("project> ")
-      try:
+            out.flush()
-        a = sys.stdin.readline()
+            try:
-      except KeyboardInterrupt:
+                a = sys.stdin.readline()
-        out.nl()
+            except KeyboardInterrupt:
-        break
+                out.nl()
-      if a == '':
+                break
-        out.nl()
+            if a == "":
-        break
+                out.nl()
+                break
 
-      a = a.strip()
+            a = a.strip()
-      if a.lower() in ('q', 'quit', 'exit'):
+            if a.lower() in ("q", "quit", "exit"):
-        break
+                break
-      if not a:
+            if not a:
-        continue
+                continue
 
-      try:
+            try:
-        a_index = int(a)
+                a_index = int(a)
-      except ValueError:
+            except ValueError:
-        a_index = None
+                a_index = None
 
-      if a_index is not None:
+            if a_index is not None:
-        if a_index == 0:
+                if a_index == 0:
-          break
+                    break
-        if 0 < a_index and a_index <= len(all_projects):
+                if 0 < a_index and a_index <= len(all_projects):
-          _AddI(all_projects[a_index - 1])
+                    _AddI(all_projects[a_index - 1])
-          continue
+                    continue
 
-      projects = [
+            projects = [
-          p for p in all_projects
+                p
-          if a in [p.name, p.RelPath(local=opt.this_manifest_only)]]
+                for p in all_projects
-      if len(projects) == 1:
+                if a in [p.name, p.RelPath(local=opt.this_manifest_only)]
-        _AddI(projects[0])
+            ]
-        continue
+            if len(projects) == 1:
-    print('Bye.')
+                _AddI(projects[0])
+                continue
+        print("Bye.")
 
 
 def _AddI(project):
-  p = GitCommand(project, ['add', '--interactive'], bare=False)
+    p = GitCommand(project, ["add", "--interactive"], bare=False)
-  p.Wait()
+    p.Wait()

subcmds/start.py

@@ -13,131 +13,146 @@
 # limitations under the License.
 
 import functools
-import os
+from typing import NamedTuple
-import sys
 
-from command import Command, DEFAULT_LOCAL_JOBS
+from command import Command
-from git_config import IsImmutable
+from command import DEFAULT_LOCAL_JOBS
+from error import RepoExitError
 from git_command import git
-import gitc_utils
+from git_config import IsImmutable
 from progress import Progress
-from project import SyncBuffer
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class ExecuteOneResult(NamedTuple):
+    project_idx: int
+    error: Exception
+
+
+class StartError(RepoExitError):
+    """Exit error for failed start command."""
 
 
 class Start(Command):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Start a new branch for development"
+    helpSummary = "Start a new branch for development"
-  helpUsage = """
+    helpUsage = """
 %prog <newbranchname> [--all | <project>...]
 """
-  helpDescription = """
+    helpDescription = """
 '%prog' begins a new branch of development, starting from the
 revision specified in the manifest.
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('--all',
+        p.add_option(
-                 dest='all', action='store_true',
+            "--all",
-                 help='begin branch in all projects')
+            action="store_true",
-    p.add_option('-r', '--rev', '--revision', dest='revision',
+            help="begin branch in all projects",
-                 help='point branch at this revision instead of upstream')
+        )
-    p.add_option('--head', '--HEAD',
+        p.add_option(
-                 dest='revision', action='store_const', const='HEAD',
+            "-r",
-                 help='abbreviation for --rev HEAD')
+            "--rev",
+            "--revision",
+            dest="revision",
+            help="point branch at this revision instead of upstream",
+        )
+        p.add_option(
+            "--head",
+            "--HEAD",
+            dest="revision",
+            action="store_const",
+            const="HEAD",
+            help="abbreviation for --rev HEAD",
+        )
 
-  def ValidateOptions(self, opt, args):
+    def ValidateOptions(self, opt, args):
-    if not args:
+        if not args:
-      self.Usage()
+            self.Usage()
 
-    nb = args[0]
+        nb = args[0]
-    if not git.check_ref_format('heads/%s' % nb):
+        if not git.check_ref_format("heads/%s" % nb):
-      self.OptionParser.error("'%s' is not a valid name" % nb)
+            self.OptionParser.error("'%s' is not a valid name" % nb)
 
-  def _ExecuteOne(self, revision, nb, project):
+    @classmethod
-    """Start one project."""
+    def _ExecuteOne(cls, revision, nb, default_revisionExpr, project_idx):
-    # If the current revision is immutable, such as a SHA1, a tag or
+        """Start one project."""
-    # a change, then we can't push back to it. Substitute with
+        # If the current revision is immutable, such as a SHA1, a tag or
-    # dest_branch, if defined; or with manifest default revision instead.
+        # a change, then we can't push back to it. Substitute with
-    branch_merge = ''
+        # dest_branch, if defined; or with manifest default revision instead.
-    if IsImmutable(project.revisionExpr):
+        branch_merge = ""
-      if project.dest_branch:
+        error = None
-        branch_merge = project.dest_branch
+        project = cls.get_parallel_context()["projects"][project_idx]
-      else:
+        if IsImmutable(project.revisionExpr):
-        branch_merge = self.manifest.default.revisionExpr
+            if project.dest_branch:
+                branch_merge = project.dest_branch
+            else:
+                branch_merge = default_revisionExpr
 
-    try:
+        try:
-      ret = project.StartBranch(
+            project.StartBranch(
-          nb, branch_merge=branch_merge, revision=revision)
+                nb, branch_merge=branch_merge, revision=revision
-    except Exception as e:
+            )
-      print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr)
+        except Exception as e:
-      ret = False
+            logger.error("error: unable to checkout %s: %s", project.name, e)
-    return (ret, project)
+            error = e
+        return ExecuteOneResult(project_idx, error)
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    nb = args[0]
+        nb = args[0]
-    err = []
+        self.TryOverrideManifestWithSmartSync()
-    projects = []
+        err_projects = []
-    if not opt.all:
+        err = []
-      projects = args[1:]
+        projects = []
-      if len(projects) < 1:
+        if not opt.all:
-        projects = ['.']  # start it in the local project by default
+            projects = args[1:]
+            if len(projects) < 1:
+                projects = ["."]  # start it in the local project by default
 
-    all_projects = self.GetProjects(projects,
+        all_projects = self.GetProjects(
-                                    missing_ok=bool(self.gitc_manifest),
+            projects,
-                                    all_manifests=not opt.this_manifest_only)
+            all_manifests=not opt.this_manifest_only,
+        )
 
-    # This must happen after we find all_projects, since GetProjects may need
+        def _ProcessResults(_pool, pm, results):
-    # the local directory, which will disappear once we save the GITC manifest.
+            for result in results:
-    if self.gitc_manifest:
+                if result.error:
-      gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                    project = all_projects[result.project_idx]
-                                       missing_ok=True)
+                    err_projects.append(project)
-      for project in gitc_projects:
+                    err.append(result.error)
-        if project.old_revision:
+                pm.update(msg="")
-          project.already_synced = True
-        else:
-          project.already_synced = False
-          project.old_revision = project.revisionExpr
-        project.revisionExpr = None
-      # Save the GITC manifest.
-      gitc_utils.save_manifest(self.gitc_manifest)
 
-      # Make sure we have a valid CWD
+        with self.ParallelContext():
-      if not os.path.exists(os.getcwd()):
+            self.get_parallel_context()["projects"] = all_projects
-        os.chdir(self.manifest.topdir)
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne,
+                    opt.revision,
+                    nb,
+                    self.manifest.default.revisionExpr,
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Starting {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
 
-      pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet)
+        if err_projects:
-      for project in all_projects:
+            for p in err_projects:
-        gitc_project = self.gitc_manifest.paths[project.relpath]
+                logger.error(
-        # Sync projects that have not been opened.
+                    "error: %s/: cannot start %s",
-        if not gitc_project.already_synced:
+                    p.RelPath(local=opt.this_manifest_only),
-          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                    nb,
-                                       project.relpath)
+                )
-          project.worktree = proj_localdir
+            msg_fmt = "cannot start %d project(s)"
-          if not os.path.exists(proj_localdir):
+            self.git_event_log.ErrorEvent(
-            os.makedirs(proj_localdir)
+                msg_fmt % (len(err_projects)), msg_fmt
-          project.Sync_NetworkHalf()
+            )
-          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+            raise StartError(aggregate_errors=err)
-          project.Sync_LocalHalf(sync_buf)
-          project.revisionId = gitc_project.old_revision
-        pm.update()
-      pm.end()
-
-    def _ProcessResults(_pool, pm, results):
-      for (result, project) in results:
-        if not result:
-          err.append(project)
-        pm.update()
-
-    self.ExecuteInParallel(
-        opt.jobs,
-        functools.partial(self._ExecuteOne, opt.revision, nb),
-        all_projects,
-        callback=_ProcessResults,
-        output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet))
-
-    if err:
-      for p in err:
-        print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb),
-              file=sys.stderr)
-      sys.exit(1)

subcmds/status.py

@@ -17,19 +17,19 @@ import glob
 import io
 import os
 
-from command import DEFAULT_LOCAL_JOBS, PagedCommand
-
 from color import Coloring
+from command import DEFAULT_LOCAL_JOBS
+from command import PagedCommand
 import platform_utils
 
 
 class Status(PagedCommand):
-  COMMON = True
+    COMMON = True
-  helpSummary = "Show the working tree status"
+    helpSummary = "Show the working tree status"
-  helpUsage = """
+    helpUsage = """
 %prog [<project>...]
 """
-  helpDescription = """
+    helpDescription = """
 '%prog' compares the working tree to the staging area (aka index),
 and the most recent commit on this branch (HEAD), in each project
 specified.  A summary is displayed, one line per file where there
@@ -76,109 +76,132 @@ the following meanings:
  d:  deleted       (    in index, not in work tree                )
 
 """
-  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def _Options(self, p):
+    def _Options(self, p):
-    p.add_option('-o', '--orphans',
+        p.add_option(
-                 dest='orphans', action='store_true',
+            "-o",
-                 help="include objects in working directory outside of repo projects")
+            "--orphans",
+            action="store_true",
+            help="include objects in working directory outside of repo "
+            "projects",
+        )
 
-  def _StatusHelper(self, quiet, local, project):
+    @classmethod
-    """Obtains the status for a specific project.
+    def _StatusHelper(cls, quiet, local, project_idx):
+        """Obtains the status for a specific project.
 
-    Obtains the status for a project, redirecting the output to
+        Obtains the status for a project, redirecting the output to
-    the specified object.
+        the specified object.
 
-    Args:
+        Args:
-      quiet: Where to output the status.
+            quiet: Where to output the status.
-      local: a boolean, if True, the path is relative to the local
+            local: a boolean, if True, the path is relative to the local
-             (sub)manifest.  If false, the path is relative to the
+                (sub)manifest.  If false, the path is relative to the outermost
-             outermost manifest.
+                manifest.
-      project: Project to get status of.
+            project_idx: Project index to get status of.
 
-    Returns:
+        Returns:
-      The status of the project.
+            The status of the project.
-    """
+        """
-    buf = io.StringIO()
+        buf = io.StringIO()
-    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf,
+        project = cls.get_parallel_context()["projects"][project_idx]
-                                      local=local)
+        ret = project.PrintWorkTreeStatus(
-    return (ret, buf.getvalue())
+            quiet=quiet, output_redir=buf, local=local
+        )
+        return (ret, buf.getvalue())
 
-  def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
+    def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
-    """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
+        """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""  # noqa: E501
-    status_header = ' --\t'
+        status_header = " --\t"
-    for item in dirs:
+        for item in dirs:
-      if not platform_utils.isdir(item):
+            if not platform_utils.isdir(item):
-        outstring.append(''.join([status_header, item]))
+                outstring.append("".join([status_header, item]))
-        continue
+                continue
-      if item in proj_dirs:
+            if item in proj_dirs:
-        continue
+                continue
-      if item in proj_dirs_parents:
+            if item in proj_dirs_parents:
-        self._FindOrphans(glob.glob('%s/.*' % item) +
+                self._FindOrphans(
-                          glob.glob('%s/*' % item),
+                    glob.glob("%s/.*" % item) + glob.glob("%s/*" % item),
-                          proj_dirs, proj_dirs_parents, outstring)
+                    proj_dirs,
-        continue
+                    proj_dirs_parents,
-      outstring.append(''.join([status_header, item, '/']))
+                    outstring,
+                )
+                continue
+            outstring.append("".join([status_header, item, "/"]))
 
-  def Execute(self, opt, args):
+    def Execute(self, opt, args):
-    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        all_projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
 
-    def _ProcessResults(_pool, _output, results):
+        def _ProcessResults(_pool, _output, results):
-      ret = 0
+            ret = 0
-      for (state, output) in results:
+            for state, output in results:
-        if output:
+                if output:
-          print(output, end='')
+                    print(output, end="")
-        if state == 'CLEAN':
+                if state == "CLEAN":
-          ret += 1
+                    ret += 1
-      return ret
+            return ret
 
-    counter = self.ExecuteInParallel(
+        with self.ParallelContext():
-        opt.jobs,
+            self.get_parallel_context()["projects"] = all_projects
-        functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only),
+            counter = self.ExecuteInParallel(
-        all_projects,
+                opt.jobs,
-        callback=_ProcessResults,
+                functools.partial(
-        ordered=True)
+                    self._StatusHelper, opt.quiet, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )
 
-    if not opt.quiet and len(all_projects) == counter:
+        if not opt.quiet and len(all_projects) == counter:
-      print('nothing to commit (working directory clean)')
+            print("nothing to commit (working directory clean)")
 
-    if opt.orphans:
+        if opt.orphans:
-      proj_dirs = set()
+            proj_dirs = set()
-      proj_dirs_parents = set()
+            proj_dirs_parents = set()
-      for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only):
+            for project in self.GetProjects(
-        relpath = project.RelPath(local=opt.this_manifest_only)
+                None, missing_ok=True, all_manifests=not opt.this_manifest_only
-        proj_dirs.add(relpath)
+            ):
-        (head, _tail) = os.path.split(relpath)
+                relpath = project.RelPath(local=opt.this_manifest_only)
-        while head != "":
+                proj_dirs.add(relpath)
-          proj_dirs_parents.add(head)
+                (head, _tail) = os.path.split(relpath)
-          (head, _tail) = os.path.split(head)
+                while head != "":
-      proj_dirs.add('.repo')
+                    proj_dirs_parents.add(head)
+                    (head, _tail) = os.path.split(head)
+            proj_dirs.add(".repo")
 
-      class StatusColoring(Coloring):
+            class StatusColoring(Coloring):
-        def __init__(self, config):
+                def __init__(self, config):
-          Coloring.__init__(self, config, 'status')
+                    Coloring.__init__(self, config, "status")
-          self.project = self.printer('header', attr='bold')
+                    self.project = self.printer("header", attr="bold")
-          self.untracked = self.printer('untracked', fg='red')
+                    self.untracked = self.printer("untracked", fg="red")
 
-      orig_path = os.getcwd()
+            orig_path = os.getcwd()
-      try:
+            try:
-        os.chdir(self.manifest.topdir)
+                os.chdir(self.manifest.topdir)
 
-        outstring = []
+                outstring = []
-        self._FindOrphans(glob.glob('.*') +
+                self._FindOrphans(
-                          glob.glob('*'),
+                    glob.glob(".*") + glob.glob("*"),
-                          proj_dirs, proj_dirs_parents, outstring)
+                    proj_dirs,
+                    proj_dirs_parents,
+                    outstring,
+                )
 
-        if outstring:
+                if outstring:
-          output = StatusColoring(self.client.globalConfig)
+                    output = StatusColoring(self.client.globalConfig)
-          output.project('Objects not within a project (orphans)')
+                    output.project("Objects not within a project (orphans)")
-          output.nl()
+                    output.nl()
-          for entry in outstring:
+                    for entry in outstring:
-            output.untracked(entry)
+                        output.untracked(entry)
-            output.nl()
+                        output.nl()
-        else:
+                else:
-          print('No orphan files or directories')
+                    print("No orphan files or directories")
 
-      finally:
+            finally:
-        # Restore CWD.
+                # Restore CWD.
-        os.chdir(orig_path)
+                os.chdir(orig_path)