upload: Add rev to rootRepo push option

Bug: b/401147338
Change-Id: Iac19af5aadd250538702920d9beaeef9250c78fe
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/478801
Tested-by: Gavin Mak <gavinmak@google.com>
Reviewed-by: Scott Lee <ddoman@google.com>

.flake8

@@ -0,0 +1,17 @@
+[flake8]
+max-line-length = 80
+per-file-ignores =
+    # E501: line too long
+    tests/test_git_superproject.py: E501
+extend-ignore =
+    # E203: Whitespace before ':'
+    # See https://github.com/PyCQA/pycodestyle/issues/373
+    E203,
+    # E402: Module level import not at top of file
+    E402,
+    # E731: do not assign a lambda expression, use a def
+    E731,
+exclude =
+    .venv,
+    venv,
+    .tox,

.gitattributes

@@ -0,0 +1,4 @@
+# Prevent /bin/sh scripts from being clobbered by autocrlf=true
+git_ssh text eol=lf
+repo text eol=lf
+hooks/* text eol=lf

.github/workflows/close-pull-request.yml

@@ -0,0 +1,22 @@
+# GitHub actions workflow.
+# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+
+# https://github.com/superbrothers/close-pull-request
+name: Close Pull Request
+
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: superbrothers/close-pull-request@v3
+      with:
+        comment: >
+          Thanks for your contribution!
+          Unfortunately, we don't use GitHub pull requests to manage code
+          contributions to this repository.
+          Instead, please see [README.md](../blob/HEAD/SUBMITTING_PATCHES.md)
+          which provides full instructions on how to get involved.

.github/workflows/flake8-postsubmit.yml

@@ -0,0 +1,23 @@
+# GitHub actions workflow.
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
+# https://github.com/marketplace/actions/python-flake8
+
+name: Flake8
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  lint:
+    name: Python Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Run flake8
+        uses: julianwachholz/flake8-action@v2
+        with:
+          checkName: "Python Lint"

.github/workflows/test-ci.yml

@@ -0,0 +1,32 @@
+# GitHub actions workflow.
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
+
+name: Test CI
+
+on:
+  push:
+    branches: [main, repo-1, stable, maint]
+    tags: [v*]
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        # ubuntu-20.04 is the last version that supports python 3.6
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install tox tox-gh-actions
+    - name: Test with tox
+      run: tox

.gitignore

@@ -1 +1,13 @@
+*.asc
+*.egg-info/
+*.log
 *.pyc
+__pycache__
+/dist
+.repopickle_*
+/repoc
+/.tox
+/.venv
+
+# PyCharm related
+/.idea/

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

.isort.cfg

@@ -0,0 +1,41 @@
+# Copyright 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the isort python module.
+# This is used to enforce import sorting standards.
+#
+# https://pycqa.github.io/isort/docs/configuration/options.html
+
+[settings]
+# Be compatible with `black` since it also matches what we want.
+profile = black
+
+line_length = 80
+length_sort = false
+force_single_line = true
+lines_after_imports = 2
+from_first = false
+case_sensitive = false
+force_sort_within_sections = true
+order_by_type = false
+
+# Ignore generated files.
+extend_skip_glob = *_pb2.py
+
+# Allow importing multiple classes on a single line from these modules.
+# https://google.github.io/styleguide/pyguide#s2.2-imports
+single_line_exclusions =
+    abc,
+    collections.abc,
+    typing,

.mailmap

@@ -0,0 +1,13 @@
+Anthony Newnam <anthony.newnam@garmin.com>    Anthony <anthony@bnovc.com>
+He Ping <tdihp@hotmail.com>                   heping <tdihp@hotmail.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu xiuyun <xiuyun.hu@hisilicon.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu Xiuyun <clouds08@qq.com>
+Jelly Chen <chenguodong@huawei.com>           chenguodong <chenguodong@huawei.com>
+Jia Bi <bijia@xiaomi.com>                     bijia <bijia@xiaomi.com>
+Jiri Tyr <jiri.tyr@gmail.com>                 Jiri tyr <jiri.tyr@gmail.com>
+JoonCheol Park <jooncheol@gmail.com>          Jooncheol Park <jooncheol@gmail.com>
+Sergii Pylypenko <x.pelya.x@gmail.com>        pelya <x.pelya.x@gmail.com>
+Shawn Pearce <sop@google.com>                 Shawn O. Pearce <sop@google.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@gmail.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@sonyericsson.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>git-repo</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/git-repo</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

MANIFEST.in

@@ -0,0 +1,6 @@
+graft docs hooks tests
+include *.py
+include LICENSE
+include git_ssh
+include repo
+include run_tests

README.md

@@ -0,0 +1,55 @@
+# repo
+
+Repo is a tool built on top of Git.  Repo helps manage many Git repositories,
+does the uploads to revision control systems, and automates parts of the
+development workflow.  Repo is not meant to replace Git, only to make it
+easier to work with Git.  The repo command is an executable Python script
+that you can put anywhere in your path.
+
+* Homepage: <https://gerrit.googlesource.com/git-repo/>
+* Mailing list: [repo-discuss on Google Groups][repo-discuss]
+* Bug reports: <https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071>
+* Source: <https://gerrit.googlesource.com/git-repo/>
+* Overview: <https://source.android.com/source/developing.html>
+* Docs: <https://source.android.com/source/using-repo.html>
+* [repo Manifest Format](./docs/manifest-format.md)
+* [repo Hooks](./docs/repo-hooks.md)
+* [Submitting patches](./SUBMITTING_PATCHES.md)
+* Running Repo in [Microsoft Windows](./docs/windows.md)
+* GitHub mirror: <https://github.com/GerritCodeReview/git-repo>
+* Postsubmit tests: <https://github.com/GerritCodeReview/git-repo/actions>
+
+## Contact
+
+Please use the [repo-discuss] mailing list or [issue tracker] for questions.
+
+You can [file a new bug report][new-bug] under the "repo" component.
+
+Please do not e-mail individual developers for support.
+They do not have the bandwidth for it, and often times questions have already
+been asked on [repo-discuss] or bugs posted to the [issue tracker].
+So please search those sites first.
+
+## Install
+
+Many distros include repo, so you might be able to install from there.
+```sh
+# Debian/Ubuntu.
+$ sudo apt-get install repo
+
+# Gentoo.
+$ sudo emerge dev-vcs/repo
+```
+
+You can install it manually as well as it's a single script.
+```sh
+$ mkdir -p ~/.bin
+$ PATH="${HOME}/.bin:${PATH}"
+$ curl https://storage.googleapis.com/git-repo-downloads/repo > ~/.bin/repo
+$ chmod a+rx ~/.bin/repo
+```
+
+
+[new-bug]: https://issues.gerritcodereview.com/issues/new?component=1370071
+[issue tracker]: https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071
+[repo-discuss]: https://groups.google.com/forum/#!forum/repo-discuss

SUBMITTING_PATCHES.md

@@ -0,0 +1,190 @@
+# Submitting Changes
+
+Here's a short overview of the process.
+
+*   Make small logical changes.
+*   [Provide a meaningful commit message][commit-message-style].
+*   Make sure all code is under the Apache License, 2.0.
+*   Publish your changes for review.
+    *   `git push origin HEAD:refs/for/main`
+*   Make corrections if requested.
+*   [Verify your changes on Gerrit.](#verify)
+*   [Send to the commit queue for testing & merging.](#cq)
+
+[TOC]
+
+## Long Version
+
+I wanted a file describing how to submit patches for repo,
+so I started with the one found in the core Git distribution
+(Documentation/SubmittingPatches), which itself was based on the
+patch submission guidelines for the Linux kernel.
+
+However there are some differences, so please review and familiarize
+yourself with the following relevant bits.
+
+
+## Make separate commits for logically separate changes.
+
+Unless your patch is really trivial, you should not be sending out a patch that
+was generated between your working tree and your commit head.
+Instead, always make a commit with a complete
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
+It is a good discipline.
+
+Describe the technical detail of the change(s).
+
+If your description starts to get too long, that's a sign that you
+probably need to split up your commit to finer grained pieces.
+
+
+## Linting and formatting code
+
+Lint any changes by running:
+```sh
+$ tox -e lint -- file.py
+```
+
+And format with:
+```sh
+$ tox -e format -- file.py
+```
+
+Or format everything:
+```sh
+$ tox -e format
+```
+
+Repo uses [black](https://black.readthedocs.io/) with line length of 80 as its
+formatter and flake8 as its linter. Repo also follows
+[Google's Python Style Guide].
+
+There should be no new errors or warnings introduced.
+
+Warnings that cannot be avoided without going against the Google Style Guide
+may be suppressed inline individally using a `# noqa` comment as described
+in the [flake8 documentation].
+
+If there are many occurrences of the same warning, these may be suppressed for
+the entire project in the included `.flake8` file.
+
+[Google's Python Style Guide]: https://google.github.io/styleguide/pyguide.html
+[PEP 8]: https://www.python.org/dev/peps/pep-0008/
+[flake8 documentation]: https://flake8.pycqa.org/en/3.1.1/user/ignoring-errors.html#in-line-ignoring-errors
+
+## Running tests
+
+We use [pytest](https://pytest.org/) and [tox](https://tox.readthedocs.io/) for
+running tests.  You should make sure to install those first.
+
+To run the full suite against all supported Python versions, simply execute:
+```sh
+$ tox -p auto
+```
+
+We have [`./run_tests`](./run_tests) which is a simple wrapper around `pytest`:
+```sh
+# Run the full suite against the default Python version.
+$ ./run_tests
+# List each test as it runs.
+$ ./run_tests -v
+
+# Run a specific unittest module (and all tests in it).
+$ ./run_tests tests/test_git_command.py
+
+# Run a specific testsuite in a specific unittest module.
+$ ./run_tests tests/test_editor.py::EditString
+
+# Run a single test.
+$ ./run_tests tests/test_editor.py::EditString::test_cat_editor
+
+# List all available tests.
+$ ./run_tests --collect-only
+
+# Run a single test using substring match.
+$ ./run_tests -k test_cat_editor
+```
+
+The coverage isn't great currently, but it should still be run for all commits.
+Adding more unittests for changes you make would be greatly appreciated :).
+Check out the [tests/](./tests/) subdirectory for more details.
+
+
+## Check the license
+
+repo is licensed under the Apache License, 2.0.
+
+Because of this licensing model *every* file within the project
+*must* list the license that covers it in the header of the file.
+Any new contributions to an existing file *must* be submitted under
+the current license of that file.  Any new files *must* clearly
+indicate which license they are provided under in the file header.
+
+Please verify that you are legally allowed and willing to submit your
+changes under the license covering each file *prior* to submitting
+your patch.  It is virtually impossible to remove a patch once it
+has been applied and pushed out.
+
+
+## Sending your patches.
+
+Do not email your patches to anyone.
+
+Instead, login to the Gerrit Code Review tool at:
+
+  https://gerrit-review.googlesource.com/
+
+Ensure you have completed one of the necessary contributor
+agreements, providing documentation to the project maintainers that
+they have right to redistribute your work under the Apache License:
+
+  https://gerrit-review.googlesource.com/#/settings/agreements
+
+Ensure you have obtained an HTTP password to authenticate:
+
+  https://gerrit-review.googlesource.com/new-password
+
+Ensure that you have the local commit hook installed to automatically
+add a ChangeId to your commits:
+
+    curl -Lo `git rev-parse --git-dir`/hooks/commit-msg https://gerrit-review.googlesource.com/tools/hooks/commit-msg
+    chmod +x `git rev-parse --git-dir`/hooks/commit-msg
+
+If you have already committed your changes you will need to amend the commit
+to get the ChangeId added.
+
+    git commit --amend
+
+Push your patches over HTTPS to the review server, possibly through
+a remembered remote to make this easier in the future:
+
+    git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+    git config remote.review.push HEAD:refs/for/main
+
+    git push review
+
+You will be automatically emailed a copy of your commits, and any
+comments made by the project maintainers.
+
+
+## Make changes if requested
+
+The project maintainer who reviews your changes might request changes to your
+commit. If you make the requested changes you will need to amend your commit
+and push it to the review server again.
+
+
+## Verify your changes on Gerrit {#verify}
+
+After you receive a Code-Review+2 from the maintainer, select the Verified
+button on the Gerrit page for the change. This verifies that you have tested
+your changes and notifies the maintainer that they are ready to be submitted.
+
+## Merge your changes via the commit queue {#cq}
+
+Once a change is ready to be merged, select the Commit-Queue+2 setting on the
+Gerrit page for it. This tells the CI system to test the change, and if it
+passes all the checks, automatically merges it.
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

color.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,138 +16,203 @@ import os
 import sys
 
 import pager
-from git_config import GitConfig
 
-COLORS = {None     :-1,
-          'normal' :-1,
-          'black'  : 0,
-          'red'    : 1,
-          'green'  : 2,
-          'yellow' : 3,
-          'blue'   : 4,
-          'magenta': 5,
-          'cyan'   : 6,
-          'white'  : 7}
 
-ATTRS = {None     :-1,
-         'bold'   : 1,
-         'dim'    : 2,
-         'ul'     : 4,
-         'blink'  : 5,
-         'reverse': 7}
+COLORS = {
+    None: -1,
+    "normal": -1,
+    "black": 0,
+    "red": 1,
+    "green": 2,
+    "yellow": 3,
+    "blue": 4,
+    "magenta": 5,
+    "cyan": 6,
+    "white": 7,
+}
+
+ATTRS = {None: -1, "bold": 1, "dim": 2, "ul": 4, "blink": 5, "reverse": 7}
 
 RESET = "\033[m"
 
-def is_color(s): return s in COLORS
-def is_attr(s):  return s in ATTRS
 
-def _Color(fg = None, bg = None, attr = None):
+def is_color(s):
+    return s in COLORS
+
+
+def is_attr(s):
+    return s in ATTRS
+
+
+def _Color(fg=None, bg=None, attr=None):
     fg = COLORS[fg]
     bg = COLORS[bg]
     attr = ATTRS[attr]
 
     if attr >= 0 or fg >= 0 or bg >= 0:
-      need_sep = False
-      code = "\033["
+        need_sep = False
+        code = "\033["
 
-      if attr >= 0:
-        code += chr(ord('0') + attr)
-        need_sep = True
+        if attr >= 0:
+            code += chr(ord("0") + attr)
+            need_sep = True
 
-      if fg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+        if fg >= 0:
+            if need_sep:
+                code += ";"
+            need_sep = True
 
-        if fg < 8:
-          code += '3%c' % (ord('0') + fg)
-        else:
-          code += '38;5;%d' % fg
+            if fg < 8:
+                code += "3%c" % (ord("0") + fg)
+            else:
+                code += "38;5;%d" % fg
 
-      if bg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+        if bg >= 0:
+            if need_sep:
+                code += ";"
 
-        if bg < 8:
-          code += '4%c' % (ord('0') + bg)
-        else:
-          code += '48;5;%d' % bg
-      code += 'm'
+            if bg < 8:
+                code += "4%c" % (ord("0") + bg)
+            else:
+                code += "48;5;%d" % bg
+        code += "m"
     else:
-      code = ''
+        code = ""
     return code
 
 
-class Coloring(object):
-  def __init__(self, config, type):
-    self._section = 'color.%s' % type
-    self._config = config
-    self._out = sys.stdout
+DEFAULT = None
 
-    on = self._config.GetString(self._section)
-    if on is None:
-      on = self._config.GetString('color.ui')
 
-    if on == 'auto':
-      if pager.active or os.isatty(1):
-        self._on = True
-      else:
-        self._on = False
-    elif on in ('true', 'always'):
-      self._on = True
-    else:
-      self._on = False
+def SetDefaultColoring(state):
+    """Set coloring behavior to |state|.
 
-  @property
-  def is_on(self):
-    return self._on
+    This is useful for overriding config options via the command line.
+    """
+    if state is None:
+        # Leave it alone -- return quick!
+        return
 
-  def write(self, fmt, *args):
-    self._out.write(fmt % args)
+    global DEFAULT
+    state = state.lower()
+    if state in ("auto",):
+        DEFAULT = state
+    elif state in ("always", "yes", "true", True):
+        DEFAULT = "always"
+    elif state in ("never", "no", "false", False):
+        DEFAULT = "never"
 
-  def nl(self):
-    self._out.write('\n')
 
-  def printer(self, opt=None, fg=None, bg=None, attr=None):
-    s = self
-    c = self.colorer(opt, fg, bg, attr)
-    def f(fmt, *args):
-      s._out.write(c(fmt, *args))
-    return f
+class Coloring:
+    def __init__(self, config, section_type):
+        self._section = "color.%s" % section_type
+        self._config = config
+        self._out = sys.stdout
 
-  def colorer(self, opt=None, fg=None, bg=None, attr=None):
-    if self._on:
-      c = self._parse(opt, fg, bg, attr)
-      def f(fmt, *args):
-        str = fmt % args
-        return ''.join([c, str, RESET])
-      return f
-    else:
-      def f(fmt, *args):
-        return fmt % args
-      return f
+        on = DEFAULT
+        if on is None:
+            on = self._config.GetString(self._section)
+            if on is None:
+                on = self._config.GetString("color.ui")
 
-  def _parse(self, opt, fg, bg, attr):
-    if not opt:
-      return _Color(fg, bg, attr)
+        if on == "auto":
+            if pager.active or os.isatty(1):
+                self._on = True
+            else:
+                self._on = False
+        elif on in ("true", "always"):
+            self._on = True
+        else:
+            self._on = False
 
-    v = self._config.GetString('%s.%s' % (self._section, opt))
-    if v is None:
-      return _Color(fg, bg, attr)
+    def redirect(self, out):
+        self._out = out
 
-    v = v.strip().lower()
-    if v == "reset":
-      return RESET
-    elif v == '':
-      return _Color(fg, bg, attr)
+    @property
+    def is_on(self):
+        return self._on
 
-    have_fg = False
-    for a in v.split(' '):
-      if is_color(a):
-        if have_fg: bg = a
-        else:       fg = a
-      elif is_attr(a):
-        attr = a
+    def write(self, fmt, *args):
+        self._out.write(fmt % args)
 
-    return _Color(fg, bg, attr)
+    def flush(self):
+        self._out.flush()
+
+    def nl(self):
+        self._out.write("\n")
+
+    def printer(self, opt=None, fg=None, bg=None, attr=None):
+        s = self
+        c = self.colorer(opt, fg, bg, attr)
+
+        def f(fmt, *args):
+            s._out.write(c(fmt, *args))
+
+        return f
+
+    def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+        s = self
+        c = self.nofmt_colorer(opt, fg, bg, attr)
+
+        def f(fmt):
+            s._out.write(c(fmt))
+
+        return f
+
+    def colorer(self, opt=None, fg=None, bg=None, attr=None):
+        if self._on:
+            c = self._parse(opt, fg, bg, attr)
+
+            def f(fmt, *args):
+                output = fmt % args
+                return "".join([c, output, RESET])
+
+            return f
+        else:
+
+            def f(fmt, *args):
+                return fmt % args
+
+            return f
+
+    def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+        if self._on:
+            c = self._parse(opt, fg, bg, attr)
+
+            def f(fmt):
+                return "".join([c, fmt, RESET])
+
+            return f
+        else:
+
+            def f(fmt):
+                return fmt
+
+            return f
+
+    def _parse(self, opt, fg, bg, attr):
+        if not opt:
+            return _Color(fg, bg, attr)
+
+        v = self._config.GetString(f"{self._section}.{opt}")
+        if v is None:
+            return _Color(fg, bg, attr)
+
+        v = v.strip().lower()
+        if v == "reset":
+            return RESET
+        elif v == "":
+            return _Color(fg, bg, attr)
+
+        have_fg = False
+        for a in v.split(" "):
+            if is_color(a):
+                if have_fg:
+                    bg = a
+                else:
+                    have_fg = True
+                    fg = a
+            elif is_attr(a):
+                attr = a
+
+        return _Color(fg, bg, attr)

command.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,109 +12,538 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+import contextlib
+import multiprocessing
 import optparse
-import sys
+import os
+import re
 
+from error import InvalidProjectGroupsError
 from error import NoSuchProjectError
+from error import RepoExitError
+from event_log import EventLog
+import progress
 
-class Command(object):
-  """Base class for any command line action in repo.
-  """
 
-  common = False
-  manifest = None
-  _optparse = None
+# Are we generating man-pages?
+GENERATE_MANPAGES = os.environ.get("_REPO_GENERATE_MANPAGES_") == " indeed! "
 
-  @property
-  def OptionParser(self):
-    if self._optparse is None:
-      try:
-        me = 'repo %s' % self.NAME
-        usage = self.helpUsage.strip().replace('%prog', me)
-      except AttributeError:
-        usage = 'repo %s' % self.NAME
-      self._optparse = optparse.OptionParser(usage = usage)
-      self._Options(self._optparse)
-    return self._optparse
 
-  def _Options(self, p):
-    """Initialize the option parser.
-    """
+# Number of projects to submit to a single worker process at a time.
+# This number represents a tradeoff between the overhead of IPC and finer
+# grained opportunity for parallelism. This particular value was chosen by
+# iterating through powers of two until the overall performance no longer
+# improved. The performance of this batch size is not a function of the
+# number of cores on the system.
+WORKER_BATCH_SIZE = 32
 
-  def Usage(self):
-    """Display usage and terminate.
-    """
-    self.OptionParser.print_usage()
-    sys.exit(1)
 
-  def Execute(self, opt, args):
-    """Perform the action, after option parsing is complete.
-    """
-    raise NotImplementedError
- 
-  def GetProjects(self, args, missing_ok=False):
-    """A list of projects that match the arguments.
-    """
-    all = self.manifest.projects
-    result = []
+# How many jobs to run in parallel by default?  This assumes the jobs are
+# largely I/O bound and do not hit the network.
+DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8)
 
-    if not args:
-      for project in all.values():
-        if missing_ok or project.Exists:
-          result.append(project)
-    else:
-      by_path = None
 
-      for arg in args:
-        project = all.get(arg)
+class UsageError(RepoExitError):
+    """Exception thrown with invalid command usage."""
 
-        if not project:
-          path = os.path.abspath(arg)
 
-          if not by_path:
-            by_path = dict()
-            for p in all.values():
-              by_path[p.worktree] = p
+class Command:
+    """Base class for any command line action in repo."""
 
-          if os.path.exists(path):
-            while path \
-              and path != '/' \
-              and path != self.manifest.topdir:
-              try:
-                project = by_path[path]
-                break
-              except KeyError:
-                path = os.path.dirname(path)
-          else:
+    # Singleton for all commands to track overall repo command execution and
+    # provide event summary to callers. Only used by sync subcommand currently.
+    #
+    # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
+    event_log = EventLog()
+
+    # Whether this command is a "common" one, i.e. whether the user would
+    # commonly use it or it's a more uncommon command. This is used by the help
+    # command to show short-vs-full summaries.
+    COMMON = False
+
+    # Whether this command supports running in parallel. If greater than 0,
+    # it is the number of parallel jobs to default to.
+    PARALLEL_JOBS = None
+
+    # Whether this command supports Multi-manifest. If False, then main.py will
+    # iterate over the manifests and invoke the command once per (sub)manifest.
+    # This is only checked after calling ValidateOptions, so that partially
+    # migrated subcommands can set it to False.
+    MULTI_MANIFEST_SUPPORT = True
+
+    # Shared data across parallel execution workers.
+    _parallel_context = None
+
+    @classmethod
+    def get_parallel_context(cls):
+        assert cls._parallel_context is not None
+        return cls._parallel_context
+
+    def __init__(
+        self,
+        repodir=None,
+        client=None,
+        manifest=None,
+        git_event_log=None,
+        outer_client=None,
+        outer_manifest=None,
+    ):
+        self.repodir = repodir
+        self.client = client
+        self.outer_client = outer_client or client
+        self.manifest = manifest
+        self.git_event_log = git_event_log
+        self.outer_manifest = outer_manifest
+
+        # Cache for the OptionParser property.
+        self._optparse = None
+
+    def WantPager(self, _opt):
+        return False
+
+    def ReadEnvironmentOptions(self, opts):
+        """Set options from environment variables."""
+
+        env_options = self._RegisteredEnvironmentOptions()
+
+        for env_key, opt_key in env_options.items():
+            # Get the user-set option value if any
+            opt_value = getattr(opts, opt_key)
+
+            # If the value is set, it means the user has passed it as a command
+            # line option, and we should use that. Otherwise we can try to set
+            # it with the value from the corresponding environment variable.
+            if opt_value is not None:
+                continue
+
+            env_value = os.environ.get(env_key)
+            if env_value is not None:
+                setattr(opts, opt_key, env_value)
+
+        return opts
+
+    @property
+    def OptionParser(self):
+        if self._optparse is None:
             try:
-              project = by_path[path]
+                me = "repo %s" % self.NAME
+                usage = self.helpUsage.strip().replace("%prog", me)
+            except AttributeError:
+                usage = "repo %s" % self.NAME
+            epilog = (
+                "Run `repo help %s` to view the detailed manual." % self.NAME
+            )
+            self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
+            self._CommonOptions(self._optparse)
+            self._Options(self._optparse)
+        return self._optparse
+
+    def _CommonOptions(self, p, opt_v=True):
+        """Initialize the option parser with common options.
+
+        These will show up for *all* subcommands, so use sparingly.
+        NB: Keep in sync with repo:InitParser().
+        """
+        g = p.add_option_group("Logging options")
+        opts = ["-v"] if opt_v else []
+        g.add_option(
+            *opts,
+            "--verbose",
+            dest="output_mode",
+            action="store_true",
+            help="show all output",
+        )
+        g.add_option(
+            "-q",
+            "--quiet",
+            dest="output_mode",
+            action="store_false",
+            help="only show errors",
+        )
+
+        if self.PARALLEL_JOBS is not None:
+            default = "based on number of CPU cores"
+            if not GENERATE_MANPAGES:
+                # Only include active cpu count if we aren't generating man
+                # pages.
+                default = f"%default; {default}"
+            p.add_option(
+                "-j",
+                "--jobs",
+                type=int,
+                default=self.PARALLEL_JOBS,
+                help=f"number of jobs to run in parallel (default: {default})",
+            )
+
+        m = p.add_option_group("Multi-manifest options")
+        m.add_option(
+            "--outer-manifest",
+            action="store_true",
+            default=None,
+            help="operate starting at the outermost manifest",
+        )
+        m.add_option(
+            "--no-outer-manifest",
+            dest="outer_manifest",
+            action="store_false",
+            help="do not operate on outer manifests",
+        )
+        m.add_option(
+            "--this-manifest-only",
+            action="store_true",
+            default=None,
+            help="only operate on this (sub)manifest",
+        )
+        m.add_option(
+            "--no-this-manifest-only",
+            "--all-manifests",
+            dest="this_manifest_only",
+            action="store_false",
+            help="operate on this manifest and its submanifests",
+        )
+
+    def _Options(self, p):
+        """Initialize the option parser with subcommand-specific options."""
+
+    def _RegisteredEnvironmentOptions(self):
+        """Get options that can be set from environment variables.
+
+        Return a dictionary mapping environment variable name
+        to option key name that it can override.
+
+        Example: {'REPO_MY_OPTION': 'my_option'}
+
+        Will allow the option with key value 'my_option' to be set
+        from the value in the environment variable named 'REPO_MY_OPTION'.
+
+        Note: This does not work properly for options that are explicitly
+        set to None by the user, or options that are defined with a
+        default value other than None.
+
+        """
+        return {}
+
+    def Usage(self):
+        """Display usage and terminate."""
+        self.OptionParser.print_usage()
+        raise UsageError()
+
+    def CommonValidateOptions(self, opt, args):
+        """Validate common options."""
+        opt.quiet = opt.output_mode is False
+        opt.verbose = opt.output_mode is True
+        if opt.outer_manifest is None:
+            # By default, treat multi-manifest instances as a single manifest
+            # from the user's perspective.
+            opt.outer_manifest = True
+
+    def ValidateOptions(self, opt, args):
+        """Validate the user options & arguments before executing.
+
+        This is meant to help break the code up into logical steps. Some tips:
+        * Use self.OptionParser.error to display CLI related errors.
+        * Adjust opt member defaults as makes sense.
+        * Adjust the args list, but do so inplace so the caller sees updates.
+        * Try to avoid updating self state. Leave that to Execute.
+        """
+
+    def Execute(self, opt, args):
+        """Perform the action, after option parsing is complete."""
+        raise NotImplementedError
+
+    @classmethod
+    @contextlib.contextmanager
+    def ParallelContext(cls):
+        """Obtains the context, which is shared to ExecuteInParallel workers.
+
+        Callers can store data in the context dict before invocation of
+        ExecuteInParallel. The dict will then be shared to child workers of
+        ExecuteInParallel.
+        """
+        assert cls._parallel_context is None
+        cls._parallel_context = {}
+        try:
+            yield
+        finally:
+            cls._parallel_context = None
+
+    @classmethod
+    def _InitParallelWorker(cls, context, initializer):
+        cls._parallel_context = context
+        if initializer:
+            initializer()
+
+    @classmethod
+    def ExecuteInParallel(
+        cls,
+        jobs,
+        func,
+        inputs,
+        callback,
+        output=None,
+        ordered=False,
+        chunksize=WORKER_BATCH_SIZE,
+        initializer=None,
+    ):
+        """Helper for managing parallel execution boiler plate.
+
+        For subcommands that can easily split their work up.
+
+        Args:
+            jobs: How many parallel processes to use.
+            func: The function to apply to each of the |inputs|. Usually a
+                functools.partial for wrapping additional arguments. It will be
+                run in a separate process, so it must be pickalable, so nested
+                functions won't work. Methods on the subcommand Command class
+                should work.
+            inputs: The list of items to process. Must be a list.
+            callback: The function to pass the results to for processing. It
+                will be executed in the main thread and process the results of
+                |func| as they become available. Thus it may be a local nested
+                function. Its return value is passed back directly. It takes
+                three arguments:
+                - The processing pool (or None with one job).
+                - The |output| argument.
+                - An iterator for the results.
+            output: An output manager. May be progress.Progess or
+                color.Coloring.
+            ordered: Whether the jobs should be processed in order.
+            chunksize: The number of jobs processed in batch by parallel
+                workers.
+            initializer: Worker initializer.
+
+        Returns:
+            The |callback| function's results are returned.
+        """
+        try:
+            # NB: Multiprocessing is heavy, so don't spin it up for one job.
+            if len(inputs) == 1 or jobs == 1:
+                return callback(None, output, (func(x) for x in inputs))
+            else:
+                with multiprocessing.Pool(
+                    jobs,
+                    initializer=cls._InitParallelWorker,
+                    initargs=(cls._parallel_context, initializer),
+                ) as pool:
+                    submit = pool.imap if ordered else pool.imap_unordered
+                    return callback(
+                        pool,
+                        output,
+                        submit(func, inputs, chunksize=chunksize),
+                    )
+        finally:
+            if isinstance(output, progress.Progress):
+                output.end()
+
+    def _ResetPathToProjectMap(self, projects):
+        self._by_path = {p.worktree: p for p in projects}
+
+    def _UpdatePathToProjectMap(self, project):
+        self._by_path[project.worktree] = project
+
+    def _GetProjectByPath(self, manifest, path):
+        project = None
+        if os.path.exists(path):
+            oldpath = None
+            while path and path != oldpath and path != manifest.topdir:
+                try:
+                    project = self._by_path[path]
+                    break
+                except KeyError:
+                    oldpath = path
+                    path = os.path.dirname(path)
+            if not project and path == manifest.topdir:
+                try:
+                    project = self._by_path[path]
+                except KeyError:
+                    pass
+        else:
+            try:
+                project = self._by_path[path]
             except KeyError:
-              pass
+                pass
+        return project
 
-        if not project:
-          raise NoSuchProjectError(arg)
-        if not missing_ok and not project.Exists:
-          raise NoSuchProjectError(arg)
+    def GetProjects(
+        self,
+        args,
+        manifest=None,
+        groups="",
+        missing_ok=False,
+        submodules_ok=False,
+        all_manifests=False,
+    ):
+        """A list of projects that match the arguments.
 
-        result.append(project)
+        Args:
+            args: a list of (case-insensitive) strings, projects to search for.
+            manifest: an XmlManifest, the manifest to use, or None for default.
+            groups: a string, the manifest groups in use.
+            missing_ok: a boolean, whether to allow missing projects.
+            submodules_ok: a boolean, whether to allow submodules.
+            all_manifests: a boolean, if True then all manifests and
+                submanifests are used. If False, then only the local
+                (sub)manifest is used.
+
+        Returns:
+            A list of matching Project instances.
+        """
+        if all_manifests:
+            if not manifest:
+                manifest = self.manifest.outer_client
+            all_projects_list = manifest.all_projects
+        else:
+            if not manifest:
+                manifest = self.manifest
+            all_projects_list = manifest.projects
+        result = []
+
+        if not groups:
+            groups = manifest.GetGroupsStr()
+        groups = [x for x in re.split(r"[,\s]+", groups) if x]
+
+        if not args:
+            derived_projects = {}
+            for project in all_projects_list:
+                if submodules_ok or project.sync_s:
+                    derived_projects.update(
+                        (p.name, p) for p in project.GetDerivedSubprojects()
+                    )
+            all_projects_list.extend(derived_projects.values())
+            for project in all_projects_list:
+                if (missing_ok or project.Exists) and project.MatchesGroups(
+                    groups
+                ):
+                    result.append(project)
+        else:
+            self._ResetPathToProjectMap(all_projects_list)
+
+            for arg in args:
+                # We have to filter by manifest groups in case the requested
+                # project is checked out multiple times or differently based on
+                # them.
+                projects = [
+                    project
+                    for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests
+                    )
+                    if project.MatchesGroups(groups)
+                ]
+
+                if not projects:
+                    path = os.path.abspath(arg).replace("\\", "/")
+                    tree = manifest
+                    if all_manifests:
+                        # Look for the deepest matching submanifest.
+                        for tree in reversed(list(manifest.all_manifests)):
+                            if path.startswith(tree.topdir):
+                                break
+                    project = self._GetProjectByPath(tree, path)
+
+                    # If it's not a derived project, update path->project
+                    # mapping and search again, as arg might actually point to
+                    # a derived subproject.
+                    if (
+                        project
+                        and not project.Derived
+                        and (submodules_ok or project.sync_s)
+                    ):
+                        search_again = False
+                        for subproject in project.GetDerivedSubprojects():
+                            self._UpdatePathToProjectMap(subproject)
+                            search_again = True
+                        if search_again:
+                            project = (
+                                self._GetProjectByPath(manifest, path)
+                                or project
+                            )
+
+                    if project:
+                        projects = [project]
+
+                if not projects:
+                    raise NoSuchProjectError(arg)
+
+                for project in projects:
+                    if not missing_ok and not project.Exists:
+                        raise NoSuchProjectError(
+                            "%s (%s)"
+                            % (arg, project.RelPath(local=not all_manifests))
+                        )
+                    if not project.MatchesGroups(groups):
+                        raise InvalidProjectGroupsError(arg)
+
+                result.extend(projects)
+
+        def _getpath(x):
+            return x.relpath
+
+        result.sort(key=_getpath)
+        return result
+
+    def FindProjects(self, args, inverse=False, all_manifests=False):
+        """Find projects from command line arguments.
+
+        Args:
+            args: a list of (case-insensitive) strings, projects to search for.
+            inverse: a boolean, if True, then projects not matching any |args|
+                are returned.
+            all_manifests: a boolean, if True then all manifests and
+                submanifests are used. If False, then only the local
+                (sub)manifest is used.
+        """
+        result = []
+        patterns = [re.compile(r"%s" % a, re.IGNORECASE) for a in args]
+        for project in self.GetProjects("", all_manifests=all_manifests):
+            paths = [project.name, project.RelPath(local=not all_manifests)]
+            for pattern in patterns:
+                match = any(pattern.search(x) for x in paths)
+                if not inverse and match:
+                    result.append(project)
+                    break
+                if inverse and match:
+                    break
+            else:
+                if inverse:
+                    result.append(project)
+        result.sort(
+            key=lambda project: (project.manifest.path_prefix, project.relpath)
+        )
+        return result
+
+    def ManifestList(self, opt):
+        """Yields all of the manifests to traverse.
+
+        Args:
+            opt: The command options.
+        """
+        top = self.outer_manifest
+        if not opt.outer_manifest or opt.this_manifest_only:
+            top = self.manifest
+        yield top
+        if not opt.this_manifest_only:
+            yield from top.all_children
 
-    def _getpath(x):
-      return x.relpath
-    result.sort(key=_getpath)
-    return result
 
 class InteractiveCommand(Command):
-  """Command which requires user interaction on the tty and
-     must not run within a pager, even if the user asks to.
-  """
+    """Command which requires user interaction on the tty and must not run
+    within a pager, even if the user asks to.
+    """
+
+    def WantPager(self, _opt):
+        return False
+
 
 class PagedCommand(Command):
-  """Command which defaults to output in a pager, as its
-     display tends to be larger than one screen full.
-  """
+    """Command which defaults to output in a pager, as its display tends to be
+    larger than one screen full.
+    """
 
-class MirrorSafeCommand(object):
-  """Command permits itself to run within a mirror,
-     and does not require a working directory.
-  """
+    def WantPager(self, _opt):
+        return True
+
+
+class MirrorSafeCommand:
+    """Command permits itself to run within a mirror, and does not require a
+    working directory.
+    """

completion.bash

@@ -0,0 +1,156 @@
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Programmable bash completion.  https://github.com/scop/bash-completion
+
+# TODO: Handle interspersed options.  We handle `repo h<tab>`, but not
+# `repo --time h<tab>`.
+
+# Complete the list of repo subcommands.
+__complete_repo_list_commands() {
+  local repo=${COMP_WORDS[0]}
+  (
+  # Handle completions if running outside of a checkout.
+  if ! "${repo}" help --all 2>/dev/null; then
+    repo help 2>/dev/null
+  fi
+  ) | sed -n '/^  /{s/  \([^ ]\+\) .\+/\1/;p}'
+}
+
+# Complete list of all branches available in all projects in the repo client
+# checkout.
+__complete_repo_list_branches() {
+  local repo=${COMP_WORDS[0]}
+  "${repo}" branches 2>/dev/null | \
+    sed -n '/|/{s/[ *][Pp ] *\([^ ]\+\) .*/\1/;p}'
+}
+
+# Complete list of all projects available in the repo client checkout.
+__complete_repo_list_projects() {
+  local repo=${COMP_WORDS[0]}
+  "${repo}" list -n 2>/dev/null
+  "${repo}" list -p --relative-to=. 2>/dev/null
+}
+
+# Complete the repo <command> argument.
+__complete_repo_command() {
+  if [[ ${COMP_CWORD} -ne 1 ]]; then
+    return 1
+  fi
+
+  local command=${COMP_WORDS[1]}
+  COMPREPLY=($(compgen -W "$(__complete_repo_list_commands)" -- "${command}"))
+  return 0
+}
+
+# Complete repo subcommands that take <branch> <projects>.
+__complete_repo_command_branch_projects() {
+  local current=$1
+  if [[ ${COMP_CWORD} -eq 2 ]]; then
+    COMPREPLY=($(compgen -W "$(__complete_repo_list_branches)" -- "${current}"))
+  else
+    COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
+  fi
+}
+
+# Complete repo subcommands that take only <projects>.
+__complete_repo_command_projects() {
+  local current=$1
+  COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
+}
+
+# Complete `repo help`.
+__complete_repo_command_help() {
+  local current=$1
+  # CWORD=1 is "start".
+  # CWORD=2 is the <subcommand> which we complete here.
+  if [[ ${COMP_CWORD} -eq 2 ]]; then
+    COMPREPLY=(
+      $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
+    )
+  fi
+}
+
+# Complete `repo forall`.
+__complete_repo_command_forall() {
+  local current=$1
+  # CWORD=1 is "forall".
+  # CWORD=2+ are <projects> *until* we hit the -c option.
+  local i
+  for (( i = 0; i < COMP_CWORD; ++i )); do
+    if [[ "${COMP_WORDS[i]}" == "-c" ]]; then
+      return 0
+    fi
+  done
+
+  COMPREPLY=(
+    $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
+  )
+}
+
+# Complete `repo start`.
+__complete_repo_command_start() {
+  local current=$1
+  # CWORD=1 is "start".
+  # CWORD=2 is the <branch> which we don't complete.
+  # CWORD=3+ are <projects> which we complete here.
+  if [[ ${COMP_CWORD} -gt 2 ]]; then
+    COMPREPLY=(
+      $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
+    )
+  fi
+}
+
+# Complete the repo subcommand arguments.
+__complete_repo_arg() {
+  if [[ ${COMP_CWORD} -le 1 ]]; then
+    return 1
+  fi
+
+  local command=${COMP_WORDS[1]}
+  local current=${COMP_WORDS[COMP_CWORD]}
+  case ${command} in
+  abandon|checkout)
+    __complete_repo_command_branch_projects "${current}"
+    return 0
+    ;;
+
+  branch|branches|diff|info|list|overview|prune|rebase|smartsync|stage|status|\
+  sync|upload)
+    __complete_repo_command_projects "${current}"
+    return 0
+    ;;
+
+  help|start|forall)
+    __complete_repo_command_${command} "${current}"
+    return 0
+    ;;
+
+  *)
+    return 1
+    ;;
+  esac
+}
+
+# Complete the repo arguments.
+__complete_repo() {
+  COMPREPLY=()
+  __complete_repo_command && return 0
+  __complete_repo_arg && return 0
+  return 0
+}
+
+# Fallback to the default complete methods if we aren't able to provide anything
+# useful.  This will allow e.g. local paths to be used when it makes sense.
+complete -F __complete_repo -o bashdefault -o default repo

constraints.txt

@@ -0,0 +1,2 @@
+# NB: Keep in sync with run_tests.vpython3.
+black<26

docs/internal-fs-layout.md

@@ -0,0 +1,276 @@
+# Repo internal filesystem layout
+
+A reference to the `.repo/` tree in repo client checkouts.
+Hopefully it's complete & up-to-date, but who knows!
+
+*** note
+**Warning**:
+This is meant for developers of the repo project itself as a quick reference.
+**Nothing** in here must be construed as ABI, or that repo itself will never
+change its internals in backwards incompatible ways.
+***
+
+[TOC]
+
+## .repo/ layout
+
+All content under `.repo/` is managed by `repo` itself with few exceptions.
+
+In general, you should not make manual changes in here.
+If a setting was initialized using an option to `repo init`, you should use that
+command to change the setting later on.
+It is always safe to re-run `repo init` in existing repo client checkouts.
+For example, if you want to change the manifest branch, you can simply run
+`repo init --manifest-branch=<new name>` and repo will take care of the rest.
+
+*   `config`: Per-repo client checkout settings using [git-config] file format.
+*   `.repo_config.json`: JSON cache of the `config` file for repo to
+    read/process quickly.
+
+### repo/ state
+
+*   `repo/`: A git checkout of the repo project.  This is how `repo` re-execs
+    itself to get the latest released version.
+
+    It tracks the git repository at `REPO_URL` using the `REPO_REV` branch.
+    Those are specified at `repo init` time using the `--repo-url=<REPO_URL>`
+    and `--repo-rev=<REPO_REV>` options.
+
+    Any changes made to this directory will usually be automatically discarded
+    by repo itself when it checks for updates.  If you want to update to the
+    latest version of repo, use `repo selfupdate` instead.  If you want to
+    change the git URL/branch that this tracks, re-run `repo init` with the new
+    settings.
+
+*   `.repo_fetchtimes.json`: Used by `repo sync` to record fetch times when
+    syncing the various projects.
+
+*   `.repo_localsyncstate.json`: Used by `repo sync` to detect and warn on
+    on partial tree syncs.  Partial syncs are allowed by `repo` itself, but are
+    unsupported by many projects where `repo` is used.
+
+### Manifests
+
+For more documentation on the manifest format, including the local_manifests
+support, see the [manifest-format.md] file.
+
+*   `submanifests/{submanifest.path}/`: The path prefix to the manifest state of
+    a submanifest included in a multi-manifest checkout.  The outermost manifest
+    manifest state is found adjacent to `submanifests/`.
+
+*   `manifests/`: A git checkout of the manifest project.  Its `.git/` state
+    points to the `manifest.git` bare checkout (see below).  It tracks the git
+    branch specified at `repo init` time via `--manifest-branch`.
+
+    The local branch name is always `default` regardless of the remote tracking
+    branch.  Do not get confused if the remote branch is not `default`, or if
+    there is a remote `default` that is completely different!
+
+    No manual changes should be made in here as it will just confuse repo and
+    it won't automatically recover causing no new changes to be picked up.
+
+*   `manifests.git/`: A bare checkout of the manifest project.  It tracks the
+    git repository specified at `repo init` time via `--manifest-url`.
+
+    No manual changes should be made in here as it will just confuse repo.
+    If you want to switch the tracking settings, re-run `repo init` with the
+    new settings.
+
+*   `manifest.xml`: The manifest that repo uses.  It is generated at `repo init`
+    and uses the `--manifest-name` to determine what manifest file to load next
+    out of `manifests/`.
+
+    Do not try to modify this to load other manifests as it will confuse repo.
+    If you want to switch manifest files, re-run `repo init` with the new
+    setting.
+
+    Older versions of repo managed this with symlinks.
+
+*   `manifest.xml -> manifests/<manifest-name>.xml`: A symlink to the manifest
+    that the user wishes to sync.  It is specified at `repo init` time via
+    `--manifest-name`.
+
+
+*   `manifests.git/.repo_config.json`: JSON cache of the `manifests.git/config`
+    file for repo to read/process quickly.
+
+*   `local_manifest.xml` (*Deprecated*): User-authored tweaks to the manifest
+    used to sync.  See [local manifests] for more details.
+*   `local_manifests/`: Directory of user-authored manifest fragments to tweak
+    the manifest used to sync.  See [local manifests] for more details.
+
+### Project objects
+
+*** note
+**Warning**: Please do not use repo's approach to projects/ & project-objects/
+layouts as a model for other tools to implement similar approaches.
+It has a number of known downsides like:
+*   [Symlinks do not work well under Windows](./windows.md).
+*   Git sometimes replaces symlinks under .git/ with real files (under unknown
+    circumstances), and then the internal state gets out of sync, and data loss
+    may ensue.
+*   When sharing project-objects between multiple project checkouts, Git might
+    automatically run `gc` or `prune` which may lead to data loss or corruption
+    (since those operate on leaf projects and miss refs in other leaves).  See
+    https://gerrit-review.googlesource.com/c/git-repo/+/254392 for more details.
+
+Instead, you should use standard Git workflows like [git worktree] or
+[gitsubmodules] with [superprojects].
+***
+
+*   `copy-link-files.json`: Tracking file used by `repo sync` to determine when
+    copyfile or linkfile are added or removed and need corresponding updates.
+*   `project.list`: Tracking file used by `repo sync` to determine when projects
+    are added or removed and need corresponding updates in the checkout.
+*   `projects/`: Bare checkouts of every project synced by the manifest.  The
+    filesystem layout matches the `<project path=...` setting in the manifest
+    (i.e. where it's checked out in the repo client source tree).  Those
+    checkouts will symlink their `.git/` state to paths under here.
+
+    Some git state is further split out under `project-objects/`.
+*   `project-objects/`: Git objects that are safe to share across multiple
+    git checkouts.  The filesystem layout matches the `<project name=...`
+    setting in the manifest (i.e. the path on the remote server) with a `.git`
+    suffix.  This allows for multiple checkouts of the same remote git repo to
+    share their objects.  For example, you could have different branches of
+    `foo/bar.git` checked out to `foo/bar-main`, `foo/bar-release`, etc...
+    There will be multiple trees under `projects/` for each one, but only one
+    under `project-objects/`.
+
+    This layout is designed to allow people to sync against different remotes
+    (e.g. a local mirror & a public review server) while avoiding duplicating
+    the content.  However, this can run into problems if different remotes use
+    the same path on their respective servers.  Best to avoid that.
+*   `modules/`: Like `projects/`, but for git submodules.
+*   `subproject-objects/`: Like `project-objects/`, but for git submodules.
+*   `worktrees/`: Bare checkouts of every project synced by the manifest.  The
+    filesystem layout matches the `<project name=...` setting in the manifest
+    (i.e. the path on the remote server) with a `.git` suffix.  This has the
+    same advantages as the `project-objects/` layout above.
+
+    This is used when [git worktree]'s are enabled.
+
+### Global settings
+
+The `.repo/manifests.git/config` file is used to track settings for the entire
+repo client checkout.
+
+Most settings use the `[repo]` section to avoid conflicts with git.
+
+Everything under `[repo.syncstate.*]` is used to keep track of sync details for logging
+purposes.
+
+User controlled settings are initialized when running `repo init`.
+
+| Setting                  | `repo init` Option        | Use/Meaning |
+|-------------------       |---------------------------|-------------|
+| manifest.groups          | `--groups` & `--platform` | The manifest groups to sync |
+| manifest.standalone      | `--standalone-manifest`   | Download manifest as static file instead of creating checkout |
+| repo.archive             | `--archive`               | Use `git archive` for checkouts |
+| repo.clonebundle         | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
+| repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
+| repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
+| repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
+| repo.mirror              | `--mirror`                | Checkout is a repo mirror |
+| repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
+| repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
+| repo.reference           | `--reference`             | Reference repo client checkout |
+| repo.submodules          | `--submodules`            | Sync git submodules |
+| repo.superproject        | `--use-superproject`      | Sync [superproject] |
+| repo.worktree            | `--worktree`              | Use [git worktree] for checkouts |
+| user.email               | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
+| user.name                | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |
+
+[partial git clones]: https://git-scm.com/docs/gitrepository-layout#_code_partialclone_code
+[superproject]: https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
+
+### Repo hooks settings
+
+For more details on this feature, see the [repo-hooks docs](./repo-hooks.md).
+We'll just discuss the internal configuration settings.
+These are stored in the registered `<repo-hooks>` project itself, so if the
+manifest switches to a different project, the settings will not be copied.
+
+| Setting                              | Use/Meaning |
+|--------------------------------------|-------------|
+| repo.hooks.\<hook\>.approvedmanifest | User approval for secure manifest sources (e.g. https://) |
+| repo.hooks.\<hook\>.approvedhash     | User approval for insecure manifest sources (e.g. http://) |
+
+
+For example, if our manifest had the following entries, we would store settings
+under `.repo/projects/src/repohooks.git/config` (which would be reachable via
+`git --git-dir=src/repohooks/.git config`).
+```xml
+  <project path="src/repohooks" name="chromiumos/repohooks" ... />
+  <repo-hooks in-project="chromiumos/repohooks" ... />
+```
+
+If `<hook>` is `pre-upload`, the `.git/config` setting might be:
+```ini
+[repo "hooks.pre-upload"]
+	approvedmanifest = https://chromium.googlesource.com/chromiumos/manifest
+```
+
+## Per-project settings
+
+These settings are somewhat meant to be tweaked by the user on a per-project
+basis (e.g. `git config` in a checked out source repo).
+
+Where possible, we re-use standard git settings to avoid confusion, and we
+refrain from documenting those, so see [git-config] documentation instead.
+
+See `repo help upload` for documentation on `[review]` settings.
+
+The `[remote]` settings are automatically populated/updated from the manifest.
+
+The `[branch]` settings are updated by `repo start` and `git branch`.
+
+| Setting                               | Subcommands   | Use/Meaning |
+|---------------------------------------|---------------|-------------|
+| review.\<url\>.autocopy               | upload        | Automatically add to `--cc=<value>` |
+| review.\<url\>.autoreviewer           | upload        | Automatically add to `--reviewers=<value>` |
+| review.\<url\>.autoupload             | upload        | Automatically answer "yes" or "no" to all prompts |
+| review.\<url\>.uploadhashtags         | upload        | Automatically add to `--hashtag=<value>` |
+| review.\<url\>.uploadlabels           | upload        | Automatically add to `--label=<value>` |
+| review.\<url\>.uploadnotify           | upload        | [Notify setting][upload-notify] to use |
+| review.\<url\>.uploadtopic            | upload        | Default [topic] to use |
+| review.\<url\>.uploadwarningthreshold | upload        | Warn when attempting to upload more than this many CLs |
+| review.\<url\>.username               | upload        | Override username with `ssh://` review URIs |
+| remote.\<remote\>.fetch               | sync          | Set of refs to fetch |
+| remote.\<remote\>.projectname         | \<network\>   | The name of the project as it exists in Gerrit review |
+| remote.\<remote\>.pushurl             | upload        | The base URI for pushing CLs |
+| remote.\<remote\>.review              | upload        | The URI of the Gerrit review server |
+| remote.\<remote\>.url                 | sync & upload | The URI of the git project to fetch |
+| branch.\<branch\>.merge               | sync & upload | The branch to merge & upload & track |
+| branch.\<branch\>.remote              | sync & upload | The remote to track |
+
+## ~/ dotconfig layout
+
+Repo will create & maintain a few files under the `.repoconfig/` directory.
+This is placed in the user's home directory by default but can be changed by
+setting `REPO_CONFIG_DIR`.
+
+*   `.repoconfig/`: Repo's per-user directory for all random config files/state.
+*   `.repoconfig/config`: Per-user settings using [git-config] file format.
+*   `.repoconfig/keyring-version`: Cache file for checking if the gnupg subdir
+    has all the same keys as the repo launcher.  Used to avoid running gpg
+    constantly as that can be quite slow.
+*   `.repoconfig/gnupg/`: GnuPG's internal state directory used when repo needs
+    to run `gpg`.  This provides isolation from the user's normal `~/.gnupg/`.
+
+*   `.repoconfig/.repo_config.json`: JSON cache of the `.repoconfig/config`
+    file for repo to read/process quickly.
+*   `.repo_.gitconfig.json`: JSON cache of the `.gitconfig` file for repo to
+    read/process quickly.
+
+
+[git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
+[git worktree]: https://git-scm.com/docs/git-worktree
+[gitsubmodules]: https://git-scm.com/docs/gitsubmodules
+[manifest-format.md]: ./manifest-format.md
+[local manifests]: ./manifest-format.md#Local-Manifests
+[superprojects]: https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
+[topic]: https://gerrit-review.googlesource.com/Documentation/intro-user.html#topics
+[upload-notify]: https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify

docs/manifest-format.md

@@ -0,0 +1,606 @@
+# repo Manifest Format
+
+A repo manifest describes the structure of a repo client; that is
+the directories that are visible and where they should be obtained
+from with git.
+
+The basic structure of a manifest is a bare Git repository holding
+a single `default.xml` XML file in the top level directory.
+
+Manifests are inherently version controlled, since they are kept
+within a Git repository.  Updates to manifests are automatically
+obtained by clients during `repo sync`.
+
+[TOC]
+
+
+## XML File Format
+
+A manifest XML file (e.g. `default.xml`) roughly conforms to the
+following DTD:
+
+```xml
+<!DOCTYPE manifest [
+
+  <!ELEMENT manifest (notice?,
+                      remote*,
+                      default?,
+                      manifest-server?,
+                      submanifest*?,
+                      remove-project*,
+                      project*,
+                      extend-project*,
+                      repo-hooks?,
+                      superproject?,
+                      contactinfo?,
+                      include*)>
+
+  <!ELEMENT notice (#PCDATA)>
+
+  <!ELEMENT remote (annotation*)>
+  <!ATTLIST remote name         ID    #REQUIRED>
+  <!ATTLIST remote alias        CDATA #IMPLIED>
+  <!ATTLIST remote fetch        CDATA #REQUIRED>
+  <!ATTLIST remote pushurl      CDATA #IMPLIED>
+  <!ATTLIST remote review       CDATA #IMPLIED>
+  <!ATTLIST remote revision     CDATA #IMPLIED>
+
+  <!ELEMENT default EMPTY>
+  <!ATTLIST default remote      IDREF #IMPLIED>
+  <!ATTLIST default revision    CDATA #IMPLIED>
+  <!ATTLIST default dest-branch CDATA #IMPLIED>
+  <!ATTLIST default upstream    CDATA #IMPLIED>
+  <!ATTLIST default sync-j      CDATA #IMPLIED>
+  <!ATTLIST default sync-c      CDATA #IMPLIED>
+  <!ATTLIST default sync-s      CDATA #IMPLIED>
+  <!ATTLIST default sync-tags   CDATA #IMPLIED>
+
+  <!ELEMENT manifest-server EMPTY>
+  <!ATTLIST manifest-server url CDATA #REQUIRED>
+
+  <!ELEMENT submanifest EMPTY>
+  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest remote         IDREF #IMPLIED>
+  <!ATTLIST submanifest project        CDATA #IMPLIED>
+  <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
+  <!ATTLIST submanifest revision       CDATA #IMPLIED>
+  <!ATTLIST submanifest path           CDATA #IMPLIED>
+  <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
+
+  <!ELEMENT project (annotation*,
+                     project*,
+                     copyfile*,
+                     linkfile*)>
+  <!ATTLIST project name        CDATA #REQUIRED>
+  <!ATTLIST project path        CDATA #IMPLIED>
+  <!ATTLIST project remote      IDREF #IMPLIED>
+  <!ATTLIST project revision    CDATA #IMPLIED>
+  <!ATTLIST project dest-branch CDATA #IMPLIED>
+  <!ATTLIST project groups      CDATA #IMPLIED>
+  <!ATTLIST project sync-c      CDATA #IMPLIED>
+  <!ATTLIST project sync-s      CDATA #IMPLIED>
+  <!ATTLIST project sync-tags   CDATA #IMPLIED>
+  <!ATTLIST project upstream CDATA #IMPLIED>
+  <!ATTLIST project clone-depth CDATA #IMPLIED>
+  <!ATTLIST project force-path CDATA #IMPLIED>
+
+  <!ELEMENT annotation EMPTY>
+  <!ATTLIST annotation name  CDATA #REQUIRED>
+  <!ATTLIST annotation value CDATA #REQUIRED>
+  <!ATTLIST annotation keep  CDATA "true">
+
+  <!ELEMENT copyfile EMPTY>
+  <!ATTLIST copyfile src  CDATA #REQUIRED>
+  <!ATTLIST copyfile dest CDATA #REQUIRED>
+
+  <!ELEMENT linkfile EMPTY>
+  <!ATTLIST linkfile src CDATA #REQUIRED>
+  <!ATTLIST linkfile dest CDATA #REQUIRED>
+
+  <!ELEMENT extend-project EMPTY>
+  <!ATTLIST extend-project name CDATA #REQUIRED>
+  <!ATTLIST extend-project path CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-path CDATA #IMPLIED>
+  <!ATTLIST extend-project groups CDATA #IMPLIED>
+  <!ATTLIST extend-project revision CDATA #IMPLIED>
+  <!ATTLIST extend-project remote CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
+  <!ATTLIST extend-project upstream CDATA #IMPLIED>
+  <!ATTLIST extend-project base-rev CDATA #IMPLIED>
+
+  <!ELEMENT remove-project EMPTY>
+  <!ATTLIST remove-project name     CDATA #IMPLIED>
+  <!ATTLIST remove-project path     CDATA #IMPLIED>
+  <!ATTLIST remove-project optional CDATA #IMPLIED>
+  <!ATTLIST remove-project base-rev CDATA #IMPLIED>
+
+  <!ELEMENT repo-hooks EMPTY>
+  <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
+  <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
+
+  <!ELEMENT superproject EMPTY>
+  <!ATTLIST superproject name     CDATA #REQUIRED>
+  <!ATTLIST superproject remote   IDREF #IMPLIED>
+  <!ATTLIST superproject revision CDATA #IMPLIED>
+
+  <!ELEMENT contactinfo EMPTY>
+  <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
+
+  <!ELEMENT include EMPTY>
+  <!ATTLIST include name     CDATA #REQUIRED>
+  <!ATTLIST include groups   CDATA #IMPLIED>
+  <!ATTLIST include revision CDATA #IMPLIED>
+]>
+```
+
+For compatibility purposes across repo releases, all unknown elements are
+silently ignored.  However, repo reserves all possible names for itself for
+future use.  If you want to use custom elements, the `x-*` namespace is
+reserved for that purpose, and repo guarantees to never allocate any
+corresponding names.
+
+A description of the elements and their attributes follows.
+
+
+### Element manifest
+
+The root element of the file.
+
+### Element notice
+
+Arbitrary text that is displayed to users whenever `repo sync` finishes.
+The content is simply passed through as it exists in the manifest.
+
+### Element remote
+
+One or more remote elements may be specified.  Each remote element
+specifies a Git URL shared by one or more projects and (optionally)
+the Gerrit review server those projects upload changes through.
+
+Attribute `name`: A short name unique to this manifest file.  The
+name specified here is used as the remote name in each project's
+.git/config, and is therefore automatically available to commands
+like `git fetch`, `git remote`, `git pull` and `git push`.
+
+Attribute `alias`: The alias, if specified, is used to override
+`name` to be set as the remote name in each project's .git/config.
+Its value can be duplicated while attribute `name` has to be unique
+in the manifest file. This helps each project to be able to have
+same remote name which actually points to different remote url.
+
+Attribute `fetch`: The Git URL prefix for all projects which use
+this remote.  Each project's name is appended to this prefix to
+form the actual URL used to clone the project.
+
+Attribute `pushurl`: The Git "push" URL prefix for all projects
+which use this remote.  Each project's name is appended to this
+prefix to form the actual URL used to "git push" the project.
+This attribute is optional; if not specified then "git push"
+will use the same URL as the `fetch` attribute.
+
+Attribute `review`: Hostname of the Gerrit server where reviews
+are uploaded to by `repo upload`.  This attribute is optional;
+if not specified then `repo upload` will not function.
+
+Attribute `revision`: Name of a Git branch (e.g. `main` or
+`refs/heads/main`). Remotes with their own revision will override
+the default revision.
+
+### Element default
+
+At most one default element may be specified.  Its remote and
+revision attributes are used when a project element does not
+specify its own remote or revision attribute.
+
+Attribute `remote`: Name of a previously defined remote element.
+Project elements lacking a remote attribute of their own will use
+this remote.
+
+Attribute `revision`: Name of a Git branch (e.g. `main` or
+`refs/heads/main`).  Project elements lacking their own
+revision attribute will use this revision.
+
+Attribute `dest-branch`: Name of a Git branch (e.g. `main`).
+Project elements not setting their own `dest-branch` will inherit
+this value. If this value is not set, projects will use `revision`
+by default instead.
+
+Attribute `upstream`: Name of the Git ref in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space. Project elements
+not setting their own `upstream` will inherit this value.
+
+Attribute `sync-j`: Number of parallel jobs to use when synching.
+
+Attribute `sync-c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.  Project elements lacking a sync-c element of
+their own will use this value.
+
+Attribute `sync-s`: Set to true to also sync sub-projects.
+
+Attribute `sync-tags`: Set to false to only sync the given Git
+branch (specified in the `revision` attribute) rather than
+the other ref tags.
+
+
+### Element manifest-server
+
+At most one manifest-server may be specified. The url attribute
+is used to specify the URL of a manifest server, which is an
+XML RPC service.
+
+See the [smart sync documentation](./smart-sync.md) for more details.
+
+
+### Element submanifest
+
+One or more submanifest elements may be specified.  Each element describes a
+single manifest to be checked out as a child.
+
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below.  The same name can be
+used for submanifests with different parent (sub)manifests.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `project`: The manifest project name.  The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+
+Attribute `manifest-name`: The manifest filename in the manifest project.  If
+not supplied, `default.xml` is used.
+
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash.  If not supplied, `name` is
+used.
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the submanifest repo client top directory
+should be placed.  If not supplied, `revision` is used.
+
+`path` may not be an absolute path or use "." or ".." path components.
+
+Attribute `groups`: List of additional groups to which all projects
+in the included submanifest belong. This appends and recurses, meaning
+all projects in submanifests carry all parent submanifest groups.
+Same syntax as the corresponding element of `project`.
+
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+
+### Element project
+
+One or more project elements may be specified.  Each element
+describes a single Git repository to be cloned into the repo
+client workspace.  You may specify Git-submodules by creating a
+nested project.  Git-submodules will be automatically
+recognized and inherit their parent's attributes, but those
+may be overridden by an explicitly specified project element.
+
+Attribute `name`: A unique name for this project.  The project's
+name is appended onto its remote's fetch URL to generate the actual
+URL to configure the Git remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+"name" must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+These restrictions are not enforced for [Local Manifests].
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the Git working directory for this project
+should be placed.  If not supplied the project "name" is used.
+If the project has a parent element, its path will be prefixed
+by the parent's.
+
+"path" may not be an absolute path or use "." or ".." path components.
+These restrictions are not enforced for [Local Manifests].
+
+If you want to place files into the root of the checkout (e.g. a README or
+Makefile or another build script), use the [copyfile] or [linkfile] elements
+instead.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `revision`: Name of the Git branch the manifest wants
+to track for this project.  Names can be relative to refs/heads
+(e.g. just "main") or absolute (e.g. "refs/heads/main").
+Tags and/or explicit SHA-1s should work in theory, but have not
+been extensively tested.  If not supplied the revision given by
+the remote element is used if applicable, else the default
+element is used.
+
+Attribute `dest-branch`: Name of a Git branch (e.g. `main`).
+When using `repo upload`, changes will be submitted for code
+review on this branch. If unspecified both here and in the
+default element, `revision` is used instead.
+
+Attribute `groups`: List of groups to which this project belongs,
+whitespace or comma separated.  All projects belong to the group
+"all", and each project automatically belongs to a group of
+its name:`name` and path:`path`.  E.g. for
+`<project name="monkeys" path="barrel-of"/>`, that project
+definition is implicitly in the following manifest groups:
+default, name:monkeys, and path:barrel-of.  If you place a project in the
+group "notdefault", it will not be automatically downloaded by repo.
+If the project has a parent element, the `name` and `path` here
+are the prefixed ones.
+
+Attribute `sync-c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.
+
+Attribute `sync-s`: Set to true to also sync sub-projects.
+
+Attribute `upstream`: Name of the Git ref in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space.
+
+Attribute `clone-depth`: Set the depth to use when fetching this
+project.  If specified, this value will override any value given
+to repo init with the --depth option on the command line.
+
+Attribute `force-path`: Set to true to force this project to create the
+local mirror repository according to its `path` attribute (if supplied)
+rather than the `name` attribute.  This attribute only applies to the
+local mirrors syncing, it will be ignored when syncing the projects in a
+client working directory.
+
+### Element extend-project
+
+Modify the attributes of the named project.
+
+This element is mostly useful in a local manifest file, to modify the
+attributes of an existing project without completely replacing the
+existing project definition.  This makes the local manifest more robust
+against changes to the original manifest.
+
+Attribute `path`: If specified, limit the change to projects checked out
+at the specified path, rather than all projects with the given name.
+
+Attribute `dest-path`: If specified, a path relative to the top directory
+of the repo client where the Git working directory for this project
+should be placed.  This is used to move a project in the checkout by
+overriding the existing `path` setting.
+
+Attribute `groups`: List of additional groups to which this project
+belongs.  Same syntax as the corresponding element of `project`.
+
+Attribute `revision`: If specified, overrides the revision of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `remote`: If specified, overrides the remote of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `dest-branch`: If specified, overrides the dest-branch of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `upstream`: If specified, overrides the upstream of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `base-rev`: If specified, adds a check against the revision
+to be extended. Manifest parse will fail and give a list of mismatch extends
+if the revisions being extended have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
+### Element annotation
+
+Zero or more annotation elements may be specified as children of a
+project or remote element. Each element describes a name-value pair.
+For projects, this name-value pair will be exported into each project's
+environment during a 'forall' command, prefixed with `REPO__`.  In addition,
+there is an optional attribute "keep" which accepts the case insensitive values
+"true" (default) or "false".  This attribute determines whether or not the
+annotation will be kept when exported with the manifest subcommand.
+
+### Element copyfile
+
+Zero or more copyfile elements may be specified as children of a
+project element. Each element describes a src-dest pair of files;
+the "src" file will be copied to the "dest" place during `repo sync`
+command.
+
+"src" is project relative, "dest" is relative to the top of the tree.
+Copying from paths outside of the project or to paths outside of the repo
+client is not allowed.
+
+"src" and "dest" must be files.  Directories or symlinks are not allowed.
+Intermediate paths must not be symlinks either.
+
+Parent directories of "dest" will be automatically created if missing.
+
+### Element linkfile
+
+It's just like copyfile and runs at the same time as copyfile but
+instead of copying it creates a symlink.
+
+The symlink is created at "dest" (relative to the top of the tree) and
+points to the path specified by "src" which is a path in the project.
+
+Parent directories of "dest" will be automatically created if missing.
+
+The symlink target may be a file or directory, but it may not point outside
+of the repo client.
+
+### Element remove-project
+
+Deletes a project from the internal manifest table, possibly
+allowing a subsequent project element in the same manifest file to
+replace the project with a different source.
+
+This element is mostly useful in a local manifest file, where
+the user can remove a project, and possibly replace it with their
+own definition.
+
+The project `name` or project `path` can be used to specify the remove target
+meaning one of them is required. If only name is specified, all
+projects with that name are removed.
+
+If both name and path are specified, only projects with the same name and
+path are removed, meaning projects with the same name but in other
+locations are kept.
+
+If only path is specified, a matching project is removed regardless of its
+name. Logic otherwise behaves like both are specified.
+
+Attribute `optional`: Set to true to ignore remove-project elements with no
+matching `project` element.
+
+Attribute `base-rev`: If specified, adds a check against the revision
+to be removed. Manifest parse will fail and give a list of mismatch removes
+if the revisions being removed have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
+### Element repo-hooks
+
+NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.
+
+Only one repo-hooks element may be specified at a time.
+Attempting to redefine it will fail to parse.
+
+Attribute `in-project`: The project where the hooks are defined.  The value
+must match the `name` attribute (**not** the `path` attribute) of a previously
+defined `project` element.
+
+Attribute `enabled-list`: List of hooks to use, whitespace or comma separated.
+
+### Element superproject
+
+***
+*Note*: This is currently a WIP.
+***
+
+NB: See the [git superprojects documentation](
+https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects) for background
+information.
+
+This element is used to specify the URL of the superproject. It has "name" and
+"remote" as atrributes. Only "name" is required while the others have
+reasonable defaults. At most one superproject may be specified.
+Attempting to redefine it will fail to parse.
+
+Attribute `name`: A unique name for the superproject. This attribute has the
+same meaning as project's name attribute. See the
+[element project](#element-project) for more information.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `revision`: Name of the Git branch the manifest wants
+to track for this superproject. If not supplied the revision given
+by the remote element is used if applicable, else the default
+element is used.
+
+### Element contactinfo
+
+***
+*Note*: This is currently a WIP.
+***
+
+This element is used to let manifest authors self-register contact info.
+It has "bugurl" as a required atrribute. This element can be repeated,
+and any later entries will clobber earlier ones. This would allow manifest
+authors who extend manifests to specify their own contact info.
+
+Attribute `bugurl`: The URL to file a bug against the manifest owner.
+
+### Element include
+
+This element provides the capability of including another manifest
+file into the originating manifest.  Normal rules apply for the
+target manifest to include - it must be a usable manifest on its own.
+
+Attribute `name`: the manifest to include, specified relative to
+the manifest repository's root.
+
+"name" may not be an absolute path or use "." or ".." path components.
+These restrictions are not enforced for [Local Manifests].
+
+Attribute `groups`: List of additional groups to which all projects
+in the included manifest belong. This appends and recurses, meaning
+all projects in included manifests carry all parent include groups.
+Same syntax as the corresponding element of `project`.
+
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`)
+default to which all projects in the included manifest belong.
+
+## Local Manifests {#local-manifests}
+
+Additional remotes and projects may be added through local manifest
+files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
+
+For example:
+
+    $ ls .repo/local_manifests
+    local_manifest.xml
+    another_local_manifest.xml
+
+    $ cat .repo/local_manifests/local_manifest.xml
+    <?xml version="1.0" encoding="UTF-8"?>
+    <manifest>
+      <project path="manifest"
+               name="tools/manifest" />
+      <project path="platform-manifest"
+               name="platform/manifest" />
+    </manifest>
+
+Users may add projects to the local manifest(s) prior to a `repo sync`
+invocation, instructing repo to automatically download and manage
+these extra projects.
+
+Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
+be loaded in alphabetical order.
+
+Projects from local manifest files are added into
+local::<local manifest filename> group.
+
+The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
+
+
+[copyfile]: #Element-copyfile
+[linkfile]: #Element-linkfile
+[Local Manifests]: #local-manifests

docs/manifest-format.txt

@@ -1,198 +0,0 @@
-repo Manifest Format
-====================
-
-A repo manifest describes the structure of a repo client; that is
-the directories that are visible and where they should be obtained
-from with git.
-
-The basic structure of a manifest is a bare Git repository holding
-a single 'default.xml' XML file in the top level directory.
-
-Manifests are inherently version controlled, since they are kept
-within a Git repository.  Updates to manifests are automatically
-obtained by clients during `repo sync`.
-
-
-XML File Format
----------------
-
-A manifest XML file (e.g. 'default.xml') roughly conforms to the
-following DTD:
-
-  <!DOCTYPE manifest [
-    <!ELEMENT manifest (remote*,
-                        default?,
-                        remove-project*,
-                        project*,
-                        add-remote*)>
-  
-    <!ELEMENT remote (EMPTY)>
-    <!ATTLIST remote name         ID    #REQUIRED>
-    <!ATTLIST remote fetch        CDATA #REQUIRED>
-    <!ATTLIST remote review       CDATA #IMPLIED>
-    <!ATTLIST remote project-name CDATA #IMPLIED>
-  
-    <!ELEMENT default (EMPTY)>
-    <!ATTLIST default remote   IDREF #IMPLIED>
-    <!ATTLIST default revision CDATA #IMPLIED>
-  
-    <!ELEMENT project (remote*)>
-    <!ATTLIST project name     CDATA #REQUIRED>
-    <!ATTLIST project path     CDATA #IMPLIED>
-    <!ATTLIST project remote   IDREF #IMPLIED>
-    <!ATTLIST project revision CDATA #IMPLIED>
-  
-    <!ELEMENT add-remote (EMPTY)>
-    <!ATTLIST add-remote to-project   ID    #REQUIRED>
-    <!ATTLIST add-remote name         ID    #REQUIRED>
-    <!ATTLIST add-remote fetch        CDATA #REQUIRED>
-    <!ATTLIST add-remote review       CDATA #IMPLIED>
-    <!ATTLIST add-remote project-name CDATA #IMPLIED>
-  
-    <!ELEMENT remove-project (EMPTY)>
-    <!ATTLIST remove-project name  CDATA #REQUIRED>
-  ]>
-
-A description of the elements and their attributes follows.
-
-
-Element manifest
-----------------
-
-The root element of the file.
-
-
-Element remote
---------------
-
-One or more remote elements may be specified.  Each remote element
-specifies a Git URL shared by one or more projects and (optionally)
-the Gerrit review server those projects upload changes through.
-
-Attribute `name`: A short name unique to this manifest file.  The
-name specified here is used as the remote name in each project's
-.git/config, and is therefore automatically available to commands
-like `git fetch`, `git remote`, `git pull` and `git push`.
-
-Attribute `fetch`: The Git URL prefix for all projects which use
-this remote.  Each project's name is appended to this prefix to
-form the actual URL used to clone the project.
-
-Attribute `review`: Hostname of the Gerrit server where reviews
-are uploaded to by `repo upload`.  This attribute is optional;
-if not specified then `repo upload` will not function.
-
-Attribute `project-name`: Specifies the name of this project used
-by the review server given in the review attribute of this element.
-Only permitted when the remote element is nested inside of a project
-element (see below).  If not given, defaults to the name supplied
-in the project's name attribute.
-
-Element add-remote
-------------------
-
-Adds a remote to an existing project, whose name is given by the
-to-project attribute.  This is functionally equivalent to nesting
-a remote element under the project, but has the advantage that it
-can be specified in the uesr's `local_manifest.xml` to add a remote
-to a project declared by the normal manifest.
-
-The element can be used to add a fork of an existing project that
-the user needs to work with.
-
-
-Element default
----------------
-
-At most one default element may be specified.  Its remote and
-revision attributes are used when a project element does not
-specify its own remote or revision attribute.
-
-Attribute `remote`: Name of a previously defined remote element.
-Project elements lacking a remote attribute of their own will use
-this remote.
-
-Attribute `revision`: Name of a Git branch (e.g. `master` or
-`refs/heads/master`).  Project elements lacking their own
-revision attribute will use this revision.
-
-
-Element project
----------------
-
-One or more project elements may be specified.  Each element
-describes a single Git repository to be cloned into the repo
-client workspace.
-
-Attribute `name`: A unique name for this project.  The project's
-name is appended onto its remote's fetch URL to generate the actual
-URL to configure the Git remote with.  The URL gets formed as:
-
-  ${remote_fetch}/${project_name}.git
-
-where ${remote_fetch} is the remote's fetch attribute and
-${project_name} is the project's name attribute.  The suffix ".git"
-is always appended as repo assumes the upstream is a forrest of
-bare Git repositories.
-
-The project name must match the name Gerrit knows, if Gerrit is
-being used for code reviews.
-
-Attribute `path`: An optional path relative to the top directory
-of the repo client where the Git working directory for this project
-should be placed.  If not supplied the project name is used.
-
-Attribute `remote`: Name of a previously defined remote element.
-If not supplied the remote given by the default element is used.
-
-Attribute `revision`: Name of the Git branch the manifest wants
-to track for this project.  Names can be relative to refs/heads
-(e.g. just "master") or absolute (e.g. "refs/heads/master").
-Tags and/or explicit SHA-1s should work in theory, but have not
-been extensively tested.  If not supplied the revision given by
-the default element is used.
-
-Child element `remote`: Described like the top-level remote element,
-but adds an additional remote to only this project.  These additional
-remotes are fetched from first on the initial `repo sync`, causing
-the majority of the project's object database to be obtained through
-these additional remotes.
-
-
-Element remove-project
-----------------------
-
-Deletes the named project from the internal manifest table, possibly
-allowing a subsequent project element in the same manifest file to
-replace the project with a different source.
-
-This element is mostly useful in the local_manifest.xml, where
-the user can remove a project, and possibly replace it with their
-own definition.
-
-
-Local Manifest
-==============
-
-Additional remotes and projects may be added through a local
-manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
-
-For example:
-
-  $ cat .repo/local_manifest.xml
-  <?xml version="1.0" encoding="UTF-8"?>
-  <manifest>
-    <project path="manifest"
-             name="tools/manifest" />
-    <project path="platform-manifest"
-             name="platform/manifest" />
-  </manifest>
-
-Users may add projects to the local manifest prior to a `repo sync`
-invocation, instructing repo to automatically download and manage
-these extra projects.
-
-Currently the only supported feature of a local manifest is to
-add new remotes and/or projects.  In the future a local manifest
-may support picking different revisions of a project, or deleting
-projects specified in the default manifest.

docs/python-support.md

@@ -0,0 +1,92 @@
+# Supported Python Versions
+
+This documents the current supported Python versions, and tries to provide
+guidance for when we decide to drop support for older versions.
+
+## Summary
+
+*   Python 3.6 (released Dec 2016) is required starting with repo-2.0.
+*   Older versions of Python (e.g. v2.7) may use old releases via the repo-1.x
+    branch, but no support is provided.
+
+## repo hooks
+
+Projects that use [repo hooks] run on independent schedules.
+Since it's not possible to detect what version of Python the hooks were written
+or tested against, we always import & exec them with the active Python version.
+
+If the user's Python is too new for the [repo hooks], then it is up to the hooks
+maintainer to update.
+
+## Repo launcher
+
+The [repo launcher] is an independent script that can support older versions of
+Python without holding back the rest of the codebase.
+If it detects the current version of Python is too old, it will try to reexec
+via a newer version of Python via standard `pythonX.Y` interpreter names.
+
+However, this is provided as a nicety when it is not onerous, and there is no
+official support for older versions of Python than the rest of the codebase.
+
+If your default python interpreters are too old to run the launcher even though
+you have newer versions installed, your choices are:
+
+*   Modify the [repo launcher]'s shebang to suite your environment.
+*   Download an older version of the [repo launcher] and don't upgrade it.
+    Be aware that we do not guarantee old repo launchers will work with current
+    versions of repo.  Bug reports using old launchers will not be accepted.
+
+## When to drop support
+
+So far, Python 3.6 has provided most of the interesting features that we want
+(e.g. typing & f-strings), and there haven't been features in newer versions
+that are critical to us.
+
+That said, let's assume we need functionality that only exists in Python 3.7.
+How do we decide when it's acceptable to drop Python 3.6?
+
+1.  Review the [Project References](./release-process.md#project-references) to
+    see what major distros are using the previous version of Python, and when
+    they go EOL.  Generally we care about Ubuntu LTS & current/previous Debian
+    stable versions.
+    *   If they're all EOL already, then go for it, drop support.
+    *   If they aren't EOL, start a thread on [repo-discuss] to see how the user
+        base feels about the proposal.
+1.  Update the "soft" versions in the codebase.  This will start warning users
+    that the older version is deprecated.
+    *   Update [repo](/repo) if the launcher needs updating.
+        This only helps with people who download newer launchers.
+    *   Update [main.py](/main.py) for the main codebase.
+        This warns for everyone regardless of [repo launcher] version.
+    *   Update [requirements.json](/requirements.json).
+        This allows [repo launcher] to display warnings/errors without having
+        to execute the new codebase.  This helps in case of syntax or module
+        changes where older versions won't even be able to import the new code.
+1.  After some grace period (ideally at least 2 quarters after the first release
+    with the updated soft requirements), update the "hard" versions, and then
+    start using the new functionality.
+
+## Python 2.7 & 3.0-3.5
+
+> **There is no support for these versions.**
+> **Do not file bugs if you are using old Python versions.**
+> **Any such reports will be marked invalid and ignored.**
+> **Upgrade your distro and/or runtime instead.**
+
+Fetch an old version of the [repo launcher]:
+
+```sh
+$ curl https://storage.googleapis.com/git-repo-downloads/repo-2.32 > ~/.bin/repo-2.32
+$ chmod a+rx ~/.bin/repo-2.32
+```
+
+Then initialize an old version of repo:
+
+```sh
+$ repo-2.32 init --repo-rev=repo-1 ...
+```
+
+
+[repo-discuss]: https://groups.google.com/forum/#!forum/repo-discuss
+[repo hooks]: ./repo-hooks.md
+[repo launcher]: ../repo

docs/release-process.md

@@ -0,0 +1,356 @@
+# repo release process
+
+This is the process for creating a new release of repo, as well as all the
+related topics and flows.
+
+[TOC]
+
+## Schedule
+
+There is no specific schedule for when releases are made.
+Usually it's more along the lines of "enough minor changes have been merged",
+or "there's a known issue the maintainers know should get fixed".
+If you find a fix has been merged for an issue important to you, but hasn't been
+released after a week or so, feel free to [contact] us to request a new release.
+
+### Release Freezes {#freeze}
+
+We try to observe a regular schedule for when **not** to release.
+If something goes wrong, staff need to be active in order to respond quickly &
+effectively.
+We also don't want to disrupt non-Google organizations if possible.
+
+We generally follow the rules:
+
+* Release during Mon - Thu, 9:00 - 14:00 [US PT]
+* Avoid holidays
+  * All regular [US holidays]
+  * Large international ones if possible
+  * All the various [New Years]
+    * Jan 1 in Gregorian calendar is the most obvious
+    * Check for large Lunar New Years too
+* Follow the normal [Google production freeze schedule]
+
+[US holidays]: https://en.wikipedia.org/wiki/Federal_holidays_in_the_United_States
+[US PT]: https://en.wikipedia.org/wiki/Pacific_Time_Zone
+[New Years]: https://en.wikipedia.org/wiki/New_Year
+[Google production freeze schedule]: http://goto.google.com/prod-freeze
+
+## Launcher script
+
+The main repo script serves as a standalone program and is often referred to as
+the "launcher script".
+This makes it easy to copy around and install as you don't have to install any
+other files from the git repo.
+
+Whenever major changes are made to the launcher script, you should increment the
+`VERSION` variable in the launcher itself.
+At runtime, repo will check this to see if it needs to be updated (and notify
+the user automatically).
+
+## Key management
+
+Every release has a git tag that is signed with a key that repo recognizes.
+Those keys are hardcoded inside of the repo launcher itself -- look for the
+`KEYRING_VERSION` and `MAINTAINER_KEYS` settings.
+
+Adding new keys to the repo launcher will allow tags to be recognized by new
+keys, but only people using that updated version will be able to.
+Since the majority of users will be using an official launcher version, their
+version will simply ignore any new signed tags.
+
+If you want to add new keys, it's best to register them long ahead of time,
+and then wait for that updated launcher to make its way out to everyone.
+Even then, there will be a long tail of users with outdated launchers, so be
+prepared for people asking questions.
+
+### Registering a new key
+
+The process of actually adding a new key is quite simple.
+
+1.  Add the public half of the key to `MAINTAINER_KEYS`.
+2.  Increment `KEYRING_VERSION` so repo knows it needs to update.
+3.  Wait a long time after that version is in a release (~months) before trying
+    to create a new release using those new keys.
+
+## Self update algorithm
+
+When creating a new repo checkout with `repo init`, there are a few options that
+control how repo finds updates:
+
+*   `--repo-url`: This tells repo where to clone the full repo project itself.
+    It defaults to the official project (`REPO_URL` in the launcher script).
+*   `--repo-rev`: This tells repo which branch to use for the full project.
+    It defaults to the `stable` branch (`REPO_REV` in the launcher script).
+
+Whenever `repo sync` is run, repo will, once every 24 hours, see if an update
+is available.
+It fetches the latest repo-rev from the repo-url.
+Then it verifies that the latest commit in the branch has a valid signed tag
+using `git tag -v` (which uses gpg).
+If the tag is valid, then repo will update its internal checkout to it.
+
+If the latest commit doesn't have a signed tag, repo will fall back to the
+most recent tag it can find (via `git describe`).
+If that tag is valid, then repo will warn and use that commit instead.
+
+If that tag cannot be verified, it gives up and forces the user to resolve.
+
+If env variable `REPO_SKIP_SELF_UPDATE` is defined, this will
+bypass the self update algorithm.
+
+### Force an update
+
+The `repo selfupdate` command can be used to force an immediate update.
+It is not subject to the 24 hour limitation.
+
+## Branch management
+
+All development happens on the `main` branch and should generally be stable.
+
+Since the repo launcher defaults to tracking the `stable` branch, it is not
+normally updated until a new release is available.
+If something goes wrong with a new release, an older release can be force pushed
+and clients will automatically downgrade.
+
+The `maint` branch is used to track the previous major release of repo.
+It is not normally meant to be used by people as `stable` should be good enough.
+Once a new major release is pushed to the `stable` branch, then the previous
+major release can be pushed to `maint`.
+For example, when `stable` moves from `v1.10.x` to `v1.11.x`, then the `maint`
+branch will be updated from `v1.9.x` to `v1.10.x`.
+
+We don't have parallel release branches/series.
+Typically all tags are made against the `main` branch and then pushed to the
+`stable` branch to make it available to the rest of the world.
+Since repo doesn't typically see a lot of changes, this tends to be OK.
+
+## Creating a new release
+
+When you want to create a new release, you'll need to select a good version and
+create a signed tag using a key registered in repo itself.
+Typically we just tag the latest version of the `main` branch.
+The tag could be pushed now, but it won't be used by clients normally (since the
+default `repo-rev` setting is `stable`).
+This would allow some early testing on systems who explicitly select `main`.
+
+### Creating a signed tag
+
+Lets assume your keys live in a dedicated directory, e.g. `~/.gnupg/repo/`.
+
+*** note
+If you need access to the official keys, check out the internal documentation
+at [go/repo-release].
+Note that only official maintainers of repo will have access as it describes
+internal processes for accessing the restricted keys.
+***
+
+```sh
+# Pick the new version.
+$ t=v2.30
+
+# Create a new signed tag with the current HEAD.
+$ ./release/sign-tag.py $t
+
+# Verify the signed tag.
+$ git show $t
+```
+
+### Push the new release
+
+Once you're ready to make the release available to everyone, push it to the
+`stable` branch.
+
+Make sure you never push the tag itself to the stable branch!
+Only push the commit -- note the use of `^0` below.
+
+```sh
+$ git push https://gerrit-review.googlesource.com/git-repo $t
+$ git push https://gerrit-review.googlesource.com/git-repo $t^0:stable
+```
+
+If something goes horribly wrong, you can force push the previous version to the
+`stable` branch and people should automatically recover.
+Again, make sure you never push the tag itself!
+
+```sh
+$ oldrev="whatever-old-commit"
+$ git push https://gerrit-review.googlesource.com/git-repo $oldrev:stable --force
+```
+
+### Announce the release
+
+Once you do push a new release to `stable`, make sure to announce it on the
+[repo-discuss@googlegroups.com] group.
+Here is an [example announcement].
+
+You can create a short changelog using the command:
+
+```sh
+# If you haven't pushed to the stable branch yet, you can use origin/stable.
+# If you have pushed, change origin/stable to the previous release tag.
+# This assumes "main" is the current tagged release.  If it's newer, change it
+# to the current release tag too.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..main
+```
+
+## Project References
+
+Here's a table showing the relationship of major tools, their EOL dates, and
+their status in Ubuntu & Debian.
+Those distros tend to be good indicators of how long we need to support things.
+
+Things in bold indicate stuff to take note of, but does not guarantee that we
+still support them.
+Things in italics are things we used to care about but probably don't anymore.
+
+|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
+|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|:---:|:------:|:---:|
+| Apr 2008 |              |              |                 | 5.0          |
+| Jun 2008 |              |              |                 | 5.1          |
+| Oct 2008 | *Oct 2013*   |              | 2.6.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Dec 2008 | *Feb 2009*   |              | 3.0.0           |
+| Feb 2009 |              |              |                 | 5.2          |
+| Feb 2009 | *Mar 2012*   |              |                 |              | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
+| Jun 2009 | *Jun 2016*   |              | 3.1.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Sep 2009 |              |              |                 | 5.3          | *10.04 Lucid* |
+| Feb 2010 | *Oct 2012*   | 1.7.0        |                 |              | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
+| Mar 2010 |              |              |                 | 5.4          |
+| Apr 2010 |              |              |                 | 5.5          | 10.10 Maverick |
+| Apr 2010 | *Apr 2015*   |              |                 |              | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  | 5.3 |
+| Jul 2010 | *Dec 2019*   |              | *2.7.0*         |              | 11.04 Natty - *<current>* |
+| Aug 2010 |              |              |                 | 5.6          |
+| Oct 2010 |              |              |                 |              | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  | 5.5 |
+| Jan 2011 |              |              |                 | 5.7          |
+| Feb 2011 |              |              |                 | 5.8          | 11.04 Natty |
+| Feb 2011 | *Feb 2016*   |              |                 |              | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
+| Apr 2011 |              |              |                 |              | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  | 5.8 |
+| Sep 2011 |              |              |                 | 5.9          | *12.04 Precise* |
+| Oct 2011 | *Feb 2016*   |              | 3.2.0           |              | 11.04 Natty - 12.10 Quantal |
+| Oct 2011 |              |              |                 |              | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  | 5.8 |
+| Apr 2012 |              |              |                 | 6.0          | 12.10 Quantal |
+| Apr 2012 | *Apr 2019*   |              |                 |              | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  | 5.9 |
+| Aug 2012 |              |              |                 | 6.1          | 13.04 Raring |
+| Sep 2012 | *Sep 2017*   |              | 3.3.0           |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 | *Dec 2014*   | 1.8.0        |                 |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 |              |              |                 |              | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  | 6.0 |
+| Mar 2013 |              |              |                 | 6.2          | 13.10 Saucy |
+| Apr 2013 |              |              |                 |              | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  | 6.1 |
+| May 2013 | *May 2018*   |              |                 |              | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
+| Sep 2013 |              |              |                 | 6.3          |
+| Oct 2013 |              |              |                 |              | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  | 6.2 |
+| Nov 2013 |              |              |                 | 6.4          |
+| Jan 2014 |              |              |                 | 6.5          |
+| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 |              | *14.04 Trusty* |
+| Mar 2014 | *Mar 2019*   |              | *3.4.0*         |              | *14.04 Trusty* - 15.10 Wily / *Jessie* |
+| Mar 2014 |              |              |                 | 6.6          | *14.04 Trusty* - 14.10 Utopic |
+| Apr 2014 | *Apr 2024*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
+| May 2014 | *Dec 2014*   | 2.0.0        |
+| Aug 2014 | *Dec 2014*   | *2.1.0*      |                 |              | 14.10 Utopic - 15.04 Vivid / *Jessie* |
+| Oct 2014 |              |              |                 | 6.7          | 15.04 Vivid |
+| Oct 2014 |              |              |                 |              | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  | 6.6 |
+| Nov 2014 | *Sep 2015*   | 2.2.0        |
+| Feb 2015 | *Sep 2015*   | 2.3.0        |
+| Mar 2015 |              |              |                 | 6.8          |
+| Apr 2015 | *May 2017*   | 2.4.0        |
+| Apr 2015 | *Jun 2020*   |              |                 |              | *Debian 8 Jessie*    | 2.1.4    | 2.7.9 3.4.2  |
+| Apr 2015 |              |              |                 |              | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  | 6.7 |
+| Jul 2015 | *May 2017*   | 2.5.0        |                 |              | 15.10 Wily |
+| Jul 2015 |              |              |                 | 6.9          | 15.10 Wily |
+| Aug 2015 |              |              |                 | 7.0          |
+| Aug 2015 |              |              |                 | 7.1          |
+| Sep 2015 | *May 2017*   | 2.6.0        |
+| Sep 2015 | *Sep 2020*   |              | *3.5.0*         |              | *16.04 Xenial* - 17.04 Zesty / *Stretch* |
+| Oct 2015 |              |              |                 |              | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  | 6.9 |
+| Jan 2016 | *Jul 2017*   | *2.7.0*      |                 |              | *16.04 Xenial* |
+| Feb 2016 |              |              |                 | 7.2          | *16.04 Xenial* |
+| Mar 2016 | *Jul 2017*   | 2.8.0        |
+| Apr 2016 | *Apr 2026*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
+| Jun 2016 | *Jul 2017*   | 2.9.0        |                 |              | 16.10 Yakkety |
+| Jul 2016 |              |              |                 | 7.3          | 16.10 Yakkety |
+| Sep 2016 | *Sep 2017*   | 2.10.0       |
+| Oct 2016 |              |              |                 |              | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 | 7.3 |
+| Nov 2016 | *Sep 2017*   | *2.11.0*     |                 |              | 17.04 Zesty / *Stretch* |
+| Dec 2016 | **Dec 2021** |              | **3.6.0**       |              | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Dec 2016 |              |              |                 | 7.4          | 17.04 Zesty / *Debian 9 Stretch* |
+| Feb 2017 | *Sep 2017*   | 2.12.0       |
+| Mar 2017 |              |              |                 | 7.5          | 17.10 Artful |
+| Apr 2017 |              |              |                 |              | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
+| May 2017 | *May 2018*   | 2.13.0       |
+| Jun 2017 | *Jun 2022*   |              |                 |              | *Debian 9 Stretch*   | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
+| Aug 2017 | *Dec 2019*   | 2.14.0       |                 |              | 17.10 Artful |
+| Oct 2017 | *Dec 2019*   | 2.15.0       |
+| Oct 2017 |              |              |                 | 7.6          | **18.04 Bionic** |
+| Oct 2017 |              |              |                 |              | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 | 7.5 |
+| Jan 2018 | *Dec 2019*   | 2.16.0       |
+| Apr 2018 | *Mar 2021*   | **2.17.0**   |                 |              | **18.04 Bionic**     |
+| Apr 2018 |              |              |                 | 7.7          | 18.10 Cosmic |
+| Apr 2018 | **Apr 2028** |              |                 |              | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 | 7.6 |
+| Jun 2018 | *Mar 2021*   | 2.18.0       |
+| Jun 2018 | **Jun 2023** |              | 3.7.0           |              | 19.04 Disco - **Buster** |
+| Aug 2018 |              |              |                 | 7.8          |
+| Sep 2018 | *Mar 2021*   | 2.19.0       |                 |              | 18.10 Cosmic |
+| Oct 2018 |              |              |                 | 7.9          | 19.04 Disco / **Buster** |
+| Oct 2018 |              |              |                 |              | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 | 7.7 |
+| Dec 2018 | *Mar 2021*   | **2.20.0**   |                 |              | 19.04 Disco - 19.10 Eoan / **Buster** |
+| Feb 2019 | *Mar 2021*   | 2.21.0       |
+| Apr 2019 |              |              |                 | 8.0          | 19.10 Eoan |
+| Apr 2019 |              |              |                 |              | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
+| Jun 2019 |              | 2.22.0       |
+| Jul 2019 | **Jul 2024** |              |                 |              | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
+| Aug 2019 | *Mar 2021*   | 2.23.0       |
+| Oct 2019 | **Oct 2024** |              | 3.8.0           |              | **20.04 Focal** - 20.10 Groovy |
+| Oct 2019 |              |              |                 | 8.1          |
+| Oct 2019 |              |              |                 |              | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 | 8.0 |
+| Nov 2019 | *Mar 2021*   | 2.24.0       |
+| Jan 2020 | *Mar 2021*   | 2.25.0       |                 |              | **20.04 Focal** |
+| Feb 2020 |              |              |                 | 8.2          | **20.04 Focal** |
+| Mar 2020 | *Mar 2021*   | 2.26.0       |
+| Apr 2020 | **Apr 2030** |              |                 |              | **20.04 Focal**      | 2.25.1   | 2.7.17 3.8.2 | 8.2 |
+| May 2020 | *Mar 2021*   | 2.27.0       |                 |              | 20.10 Groovy |
+| May 2020 |              |              |                 | 8.3          |
+| Jul 2020 | *Mar 2021*   | 2.28.0       |
+| Sep 2020 |              |              |                 | 8.4          | 21.04 Hirsute / **Bullseye** |
+| Oct 2020 | *Mar 2021*   | 2.29.0       |
+| Oct 2020 |              |              |                 |              | 20.10 Groovy         | 2.27.0   | 2.7.18 3.8.6 | 8.3 |
+| Oct 2020 | **Oct 2025** |              | 3.9.0           |              | 21.04 Hirsute / **Bullseye** |
+| Dec 2020 | *Mar 2021*   | 2.30.0       |                 |              | 21.04 Hirsute / **Bullseye** |
+| Mar 2021 |              | 2.31.0       |                 | 8.5          |
+| Apr 2021 |              |              |                 | 8.6          |
+| Apr 2021 | *Jan 2022*   |              |                 |              | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 | 8.4 |
+| Jun 2021 |              | 2.32.0       |
+| Aug 2021 |              | 2.33.0       |                 | 8.7          |
+| Aug 2021 | **Aug 2026** |              |                 |              | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
+| Sep 2021 |              |              |                 | 8.8          |
+| Oct 2021 |              | 2.34.0       | 3.10.0          |              | **22.04 Jammy** |
+| Jan 2022 |              | 2.35.0       |
+| Feb 2022 |              |              |                 | 8.9          | **22.04 Jammy** |
+| Apr 2022 |              | 2.36.0       |                 | 9.0          |
+| Apr 2022 | **Apr 2032** |              |                 |              | **22.04 Jammy**      | 2.34.1 | 2.7.18 3.10.6 | 8.9 |
+| Jun 2022 |              | 2.37.0       |
+| Oct 2022 |              | 2.38.0       |                 | 9.1          |
+| Oct 2022 |              |              | 3.11.0          |              | **Bookworm** |
+| Dec 2022 |              | 2.39.0       |                 |              | **Bookworm** |
+| Feb 2023 |              |              |                 | 9.2          | **Bookworm** |
+| Mar 2023 |              | 2.40.0       |                 | 9.3          |
+| Jun 2023 |              | 2.41.0       |
+| Jun 2023 | **Jun 2028** |              |                 |              | **Debian 12 Bookworm** | 2.39.2 | 3.11.2 | 9.2 |
+| Aug 2023 |              | 2.42.0       |                 | 9.4          |
+| Oct 2023 |              |              | 3.12.0          | 9.5          |
+| Nov 2022 |              | 2.43.0       |
+| Dec 2023 |              |              |                 | 9.6          |
+| Feb 2024 |              | 2.44.0       |
+| Mar 2024 |              |              |                 | 9.7          |
+| Oct 2024 |              |              | 3.13.0          |
+| **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
+
+
+[contact]: ../README.md#contact
+[rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
+[rel-g]: https://en.wikipedia.org/wiki/Git#Releases
+[rel-o]: https://www.openssh.com/releasenotes.html
+[rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
+[rel-u]: https://wiki.ubuntu.com/Releases
+[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
+[repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
+[go/repo-release]: https://goto.google.com/repo-release

docs/repo-hooks.md

@@ -0,0 +1,135 @@
+# repo hooks
+
+[TOC]
+
+Repo provides a mechanism to hook specific stages of the runtime with custom
+python modules.  All the hooks live in one git project which is checked out by
+the manifest (specified during `repo init`), and the manifest itself defines
+which hooks are registered.
+
+These are useful to run linters, check formatting, and run quick unittests
+before allowing a step to proceed (e.g. before uploading a commit to Gerrit).
+
+A complete example can be found in the Android project.  It can be easily
+re-used by any repo based project and is not specific to Android.<br>
+https://android.googlesource.com/platform/tools/repohooks
+
+## Approvals
+
+When a hook is processed the first time, the user is prompted for approval.
+We don't want to execute arbitrary code without explicit consent.  For manifests
+fetched via secure protocols (e.g. https://), the user is prompted once.  For
+insecure protocols (e.g. http://), the user is prompted whenever the registered
+repohooks project is updated and a hook is triggered.
+
+## Manifest Settings
+
+For the full syntax, see the [repo manifest format](./manifest-format.md).
+
+Here's a short example from
+[Android](https://android.googlesource.com/platform/manifest/+/HEAD/default.xml).
+The `<project>` line checks out the repohooks git repo to the local
+`tools/repohooks/` path.  The `<repo-hooks>` line says to look in the project
+with the name `platform/tools/repohooks` for hooks to run during the
+`pre-upload` phase.
+
+```xml
+<project path="tools/repohooks" name="platform/tools/repohooks" />
+<repo-hooks in-project="platform/tools/repohooks" enabled-list="pre-upload" />
+```
+
+## Source Layout
+
+The repohooks git repo should have a python file with the same name as the hook.
+So if you want to support the `pre-upload` hook, you'll need to create a file
+named `pre-upload.py`.  Repo will dynamically load that module when processing
+the hook and then call the `main` function in it.
+
+Hooks should have their `main` accept `**kwargs` for future compatibility.
+
+## Runtime
+
+Hook return values are ignored.
+
+Any uncaught exceptions from the hook will cause the step to fail.  This is
+intended as a fallback safety check though rather than the normal flow.  If
+you want your hook to trigger a failure, it should call `sys.exit()` (after
+displaying relevant diagnostics).
+
+Output (stdout & stderr) are not filtered in any way.  Hooks should generally
+not be too verbose.  A short summary is nice, and some status information when
+long running operations occur, but long/verbose output should be used only if
+the hook ultimately fails.
+
+The hook runs from the top level of the repo client where the operation is
+started.
+For example, if the repo client is under `~/tree/`, then that is where the hook
+runs, even if you ran repo in a git repository at `~/tree/src/foo/`, or in a
+subdirectory of that git repository in `~/tree/src/foo/bar/`.
+Hooks frequently start off by doing a `os.chdir` to the specific project they're
+called on (see below) and then changing back to the original dir when they're
+finished.
+
+Python's `sys.path` is modified so that the top of repohooks directory comes
+first.  This should help simplify the hook logic to easily allow importing of
+local modules.
+
+Repo does not modify the state of the git checkout.  This means that the hooks
+might be running in a dirty git repo with many commits and checked out to the
+latest one.  If the hook wants to operate on specific git commits, it needs to
+manually discover the list of pending commits, extract the diff/commit, and
+then check it directly.  Hooks should not normally modify the active git repo
+(such as checking out a specific commit to run checks) without first prompting
+the user.  Although user interaction is discouraged in the common case, it can
+be useful when deploying automatic fixes.
+
+### Shebang Handling
+
+*** note
+This is intended as a transitional feature.  Hooks are expected to eventually
+migrate to Python 3 only as Python 2 is EOL & deprecated.
+***
+
+If the hook is written against a specific version of Python (either 2 or 3),
+the script can declare that explicitly.  Repo will then attempt to execute it
+under the right version of Python regardless of the version repo itself might
+be executing under.
+
+Here are the shebangs that are recognized.
+
+* `#!/usr/bin/env python` & `#!/usr/bin/python`: The hook is compatible with
+  Python 2 & Python 3.  For maximum compatibility, these are recommended.
+* `#!/usr/bin/env python2` & `#!/usr/bin/python2`: The hook requires Python 2.
+  Version specific names like `python2.7` are also recognized.
+* `#!/usr/bin/env python3` & `#!/usr/bin/python3`: The hook requires Python 3.
+  Version specific names like `python3.6` are also recognized.
+
+If no shebang is detected, or does not match the forms above, we assume that the
+hook is compatible with both Python 2 & Python 3 as if `#!/usr/bin/python` was
+used.
+
+## Hooks
+
+Here are all the points available for hooking.
+
+### pre-upload
+
+This hook runs when people run `repo upload`.
+
+The `pre-upload.py` file should be defined like:
+
+```py
+def main(project_list, worktree_list=None, **kwargs):
+    """Main function invoked directly by repo.
+
+    We must use the name "main" as that is what repo requires.
+
+    Args:
+      project_list: List of projects to run on.
+      worktree_list: A list of directories.  It should be the same length as
+          project_list, so that each entry in project_list matches with a
+          directory in worktree_list.  If None, we will attempt to calculate
+          the directories automatically.
+      kwargs: Leave this here for forward-compatibility.
+    """
+```

docs/smart-sync.md

@@ -0,0 +1,129 @@
+# repo Smart Syncing
+
+Repo normally fetches & syncs manifests from the same URL specified during
+`repo init`, and that often fetches the latest revisions of all projects in
+the manifest.  This flow works well for tracking and developing with the
+latest code, but often it's desirable to sync to other points.  For example,
+to get a local build matching a specific release or build to reproduce bugs
+reported by other people.
+
+Repo's sync subcommand has support for fetching manifests from a server over
+an XML-RPC connection.  The local configuration and network API are defined by
+repo, but individual projects have to host their own server for the client to
+communicate with.
+
+This process is called "smart syncing" -- instead of blindly fetching the latest
+revision of all projects and getting an unknown state to develop against, the
+client passes a request to the server and is given a matching manifest that
+typically specifies specific commits for every project to fetch a known source
+state.
+
+[TOC]
+
+## Manifest Configuration
+
+The manifest specifies the server to communicate with via the
+the [`<manifest-server>` element](manifest-format.md#Element-manifest_server)
+element.  This is how the client knows what service to talk to.
+
+```xml
+  <manifest-server url="https://example.com/your/manifest/server/url" />
+```
+
+If the URL starts with `persistent-`, then the
+[`git-remote-persistent-https` helper](https://github.com/git/git/blob/HEAD/contrib/persistent-https/README)
+is used to communicate with the server.
+
+## Credentials
+
+Credentials may be specified directly in typical `username:password`
+[URI syntax](https://en.wikipedia.org/wiki/URI#Syntax) in the
+`<manifest-server>` element directly in the manifest.
+
+If they are not specified, `repo sync` has `--manifest-server-username=USERNAME`
+and `--manifest-server-password=PASSWORD` options.
+
+If those are not used, then repo will look up the host in your
+[`~/.netrc`](https://docs.python.org/3/library/netrc.html) database.
+
+When making the connection, cookies matching the host are automatically loaded
+from the cookiejar specified in
+[Git's `http.cookiefile` setting](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpcookieFile).
+
+## Manifest Server
+
+Unfortunately, there are no public reference implementations.  Google has an
+internal one for Android, but it is written using Google's internal systems,
+so wouldn't be that helpful as a reference.
+
+That said, the XML-RPC API is pretty simple, so any standard XML-RPC server
+example would do.  Google's internal server uses Python's
+[xmlrpc.server.SimpleXMLRPCDispatcher](https://docs.python.org/3/library/xmlrpc.server.html).
+
+## Network API
+
+The manifest server should implement the following RPC methods.
+
+### GetApprovedManifest
+
+> `GetApprovedManifest(branch: str, target: Optional[str]) -> str`
+
+The meaning of `branch` and `target` is not strictly defined.  The server may
+interpret them however it wants.  The recommended interpretation is that the
+`branch` matches the manifest branch, and `target` is an identifier for your
+project that matches something users would build.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+### GetManifest
+
+> `GetManifest(tag: str) -> str`
+
+The meaning of `tag` is not strictly defined.  Projects are encouraged to use
+a system where the tag matches a unique source state.
+
+See the client section below for how repo typically generates these values.
+
+The server will return a manifest or an error.  If it's an error, repo will
+show the output directly to the user to provide a limited feedback channel.
+
+If the user's request is ambiguous and could match multiple manifests, the
+server has to decide whether to pick one automatically (and silently such that
+the user won't know there were multiple matches), or return an error and force
+the user to be more specific.
+
+## Client Options
+
+Once repo has successfully downloaded the manifest from the server, it saves a
+copy into `.repo/manifests/smart_sync_override.xml` so users can examine it.
+The next time `repo sync` is run, this file is automatically replaced or removed
+based on the current set of options.
+
+### --smart-sync
+
+Repo will call `GetApprovedManifest(branch[, target])`.
+
+The `branch` is determined by the current manifest branch as specified by
+`--manifest-branch=BRANCH` when running `repo init`.
+
+The `target` is defined by environment variables in the order below.  If none
+of them match, then `target` is omitted.  These variables were decided as they
+match the settings Android build environments automatically setup.
+
+1.  `${SYNC_TARGET}`: If defined, the value is used directly.
+2.  `${TARGET_PRODUCT}-${TARGET_RELEASE}-${TARGET_BUILD_VARIANT}`: If these
+    variables are all defined, then they are merged with `-` and used.
+3.  `${TARGET_PRODUCT}-${TARGET_BUILD_VARIANT}`: If these variables are all
+    defined, then they are merged with `-` and used.
+
+### --smart-tag=TAG
+
+Repo will call `GetManifest(TAG)`.

docs/windows.md

@@ -0,0 +1,169 @@
+# Microsoft Windows Details
+
+Repo is primarily developed on Linux with a lot of users on macOS.
+Windows is, unfortunately, not a common platform.
+There is support in repo for Windows, but there might be some rough edges.
+
+Keep in mind that Windows in general is "best effort" and "community supported".
+That means we don't actively test or verify behavior, but rely heavily on users
+to report problems back to us, and to contribute fixes as needed.
+
+[TOC]
+
+## Windows
+
+We only support Windows 10 or newer.
+This is largely due to symlinks not being available in older versions, but it's
+also due to most developers not using Windows.
+
+We will never add code specific to older versions of Windows.
+It might work, but it most likely won't, so please don't bother asking.
+
+## Git worktrees
+
+*** note
+**Warning**: Repo's support for Git worktrees is new & experimental.
+Please report any bugs and be sure to maintain backups!
+***
+
+The Repo 2.4 release introduced support for [Git worktrees][git-worktree].
+You don't have to worry about or understand this particular feature, so don't
+worry if this section of the Git manual is particularly impenetrable.
+
+The salient point is that Git worktrees allow Repo to create repo client
+checkouts that do not require symlinks at all under Windows.
+This means users no longer need Administrator access to sync code.
+
+Simply use `--worktree` when running `repo init` to opt in.
+
+This does not effect specific Git repositories that use symlinks themselves.
+
+[git-worktree]: https://git-scm.com/docs/git-worktree
+
+## Symlinks by default
+
+*** note
+**NB**: This section applies to the default Repo behavior which does not use
+Git worktrees (see the previous section for more info).
+***
+
+Repo will use symlinks heavily internally.
+On *NIX platforms, this isn't an issue, but Windows makes it a bit difficult.
+
+There are some documents out there for how to do this, but usually the easiest
+answer is to run your shell as an Administrator and invoke repo/git in that.
+
+This isn't a great solution, but Windows doesn't make this easy, so here we are.
+
+### Launch Git Bash
+
+If you install Git Bash (see below), you can launch that with appropriate
+permissions so that all programs "just work".
+
+* Open the Start Menu (i.e. press the ⊞ key).
+* Find/search for "Git Bash".
+* Right click it and select "Run as administrator".
+
+*** note
+**NB**: This environment is only needed when running `repo`, or any specific `git`
+command that might involve symlinks (e.g. `pull` or `checkout`).
+You do not need to run all your commands in here such as your editor.
+***
+
+### Symlinks with GNU tools
+
+If you want to use `ln -s` inside of the default Git/bash shell, you might need
+to export this environment variable:
+```sh
+$ export MSYS="winsymlinks:nativestrict"
+```
+
+Otherwise `ln -s` will copy files and not actually create a symlink.
+This also helps `tar` unpack symlinks, so that's nice.
+
+### References
+
+* https://github.com/git-for-windows/git/wiki/Symbolic-Links
+* https://blogs.windows.com/windowsdeveloper/2016/12/02/symlinks-windows-10/
+
+## Python
+
+Python 3.6 or newer is required.
+Python 2 is known to be broken when running under Windows.
+See our [Python Support](./python-support.md) document for more details.
+
+You can grab the latest Windows installer here:<br>
+https://www.python.org/downloads/release/python-3
+
+## Git
+
+You should install the most recent version of Git for Windows:<br>
+https://git-scm.com/download/win
+
+When installing, make sure to turn on "Enable symbolic links" when prompted.
+
+If you've already installed Git for Windows, you can simply download the latest
+installer from above and run it again.
+It should safely upgrade things in situ for you.
+This is useful if you want to switch the symbolic link option after the fact.
+
+## Shell
+
+We don't have a specific requirement for shell environments when running repo.
+Most developers use MinTTY/bash that's included with the Git for Windows install
+(so see above for installing Git).
+
+Command & Powershell & the Windows Terminal probably work.
+Who knows!
+
+## FAQ
+
+### repo upload always complains about allowing hooks or using --no-verify!
+
+When using `repo upload` in projects that have custom repohooks, you might get
+an error like the following:
+```sh
+$ repo upload
+ERROR: You must allow the pre-upload hook or use --no-verify.
+```
+
+This can be confusing as you never get prompted.
+[MinTTY has a bug][mintty] that breaks isatty checking inside of repo which
+causes repo to never interactively prompt the user which means the upload check
+always fails.
+
+You can workaround this by manually granting consent when uploading.
+Simply add the `--verify` option whenever uploading:
+```sh
+$ repo upload --verify
+```
+
+You will have to specify this flag every time you upload.
+
+[mintty]: https://github.com/mintty/mintty/issues/56
+
+### repohooks always fail with an close_fds error.
+
+When using the [reference repohooks project][repohooks] included in AOSP,
+you might see errors like this when running `repo upload`:
+```sh
+$ repo upload
+ERROR: Traceback (most recent call last):
+  ...
+  File "C:\...\lib\subprocess.py", line 351, in __init__
+    raise ValueError("close_fds is not supported on Windows "
+ValueError: close_fds is not supported on Windows platforms if you redirect stdin/stderr/stdout
+
+Failed to run main() for pre-upload hook; see traceback above.
+```
+
+This error shows up when using Python 2.
+You should upgrade to Python 3 instead (see above).
+
+If you already have Python 3 installed, make sure it's the default version.
+Running `python --version` should say `Python 3`, not `Python 2`.
+If you didn't install the Python versions, or don't have permission to change
+the default version, you can probably workaround this by changing `$PATH` in
+your shell so the Python 3 version is found first.
+
+[repohooks]: https://android.googlesource.com/platform/tools/repohooks

editor.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,72 +13,107 @@
 # limitations under the License.
 
 import os
-import sys
+import re
 import subprocess
+import sys
 import tempfile
 
 from error import EditorError
+import platform_utils
 
-class Editor(object):
-  """Manages the user's preferred text editor."""
 
-  _editor = None
-  globalConfig = None
+class Editor:
+    """Manages the user's preferred text editor."""
 
-  @classmethod
-  def _GetEditor(cls):
-    if cls._editor is None:
-      cls._editor = cls._SelectEditor()
-    return cls._editor
+    _editor = None
+    globalConfig = None
 
-  @classmethod
-  def _SelectEditor(cls):
-    e = os.getenv('GIT_EDITOR')
-    if e:
-      return e
+    @classmethod
+    def _GetEditor(cls):
+        if cls._editor is None:
+            cls._editor = cls._SelectEditor()
+        return cls._editor
 
-    e = cls.globalConfig.GetString('core.editor')
-    if e:
-      return e
+    @classmethod
+    def _SelectEditor(cls):
+        e = os.getenv("GIT_EDITOR")
+        if e:
+            return e
 
-    e = os.getenv('VISUAL')
-    if e:
-      return e
+        if cls.globalConfig:
+            e = cls.globalConfig.GetString("core.editor")
+            if e:
+                return e
 
-    e = os.getenv('EDITOR')
-    if e:
-      return e
+        e = os.getenv("VISUAL")
+        if e:
+            return e
 
-    if os.getenv('TERM') == 'dumb':
-      print >>sys.stderr,\
-"""No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
+        e = os.getenv("EDITOR")
+        if e:
+            return e
+
+        if os.getenv("TERM") == "dumb":
+            print(
+                """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
 Tried to fall back to vi but terminal is dumb.  Please configure at
-least one of these before using this command."""
-      sys.exit(1)
+least one of these before using this command.""",  # noqa: E501
+                file=sys.stderr,
+            )
+            sys.exit(1)
 
-    return 'vi'
+        return "vi"
 
-  @classmethod
-  def EditString(cls, data):
-    """Opens an editor to edit the given content.
+    @classmethod
+    def EditString(cls, data):
+        """Opens an editor to edit the given content.
 
-       Args:
-         data        : the text to edit
-  
-      Returns:
-        new value of edited text; None if editing did not succeed
-    """
-    editor = cls._GetEditor().split()
-    fd, path = tempfile.mkstemp()
-    try:
-      os.write(fd, data)
-      os.close(fd)
-      fd = None
+        Args:
+            data: The text to edit.
 
-      if subprocess.Popen(editor + [path]).wait() != 0:
-        raise EditorError()
-      return open(path).read()
-    finally:
-      if fd:
-        os.close(fd)
-      os.remove(path)
+        Returns:
+            New value of edited text.
+
+        Raises:
+            EditorError: The editor failed to run.
+        """
+        editor = cls._GetEditor()
+        if editor == ":":
+            return data
+
+        fd, path = tempfile.mkstemp()
+        try:
+            os.write(fd, data.encode("utf-8"))
+            os.close(fd)
+            fd = None
+
+            if platform_utils.isWindows():
+                # Split on spaces, respecting quoted strings
+                import shlex
+
+                args = shlex.split(editor)
+                shell = False
+            elif re.compile("^.*[$ \t'].*$").match(editor):
+                args = [editor + ' "$@"', "sh"]
+                shell = True
+            else:
+                args = [editor]
+                shell = False
+            args.append(path)
+
+            try:
+                rc = subprocess.Popen(args, shell=shell).wait()
+            except OSError as e:
+                raise EditorError(f"editor failed, {str(e)}: {editor} {path}")
+            if rc != 0:
+                raise EditorError(
+                    "editor failed with exit status %d: %s %s"
+                    % (rc, editor, path)
+                )
+
+            with open(path, mode="rb") as fd2:
+                return fd2.read().decode("utf-8")
+        finally:
+            if fd:
+                os.close(fd)
+            platform_utils.remove(path)

error.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,60 +12,178 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-class ManifestParseError(Exception):
-  """Failed to parse the manifest file.
-  """
+from typing import List
 
-class ManifestInvalidRevisionError(Exception):
-  """The revision value in a project is incorrect.
-  """
 
-class EditorError(Exception):
-  """Unspecified error from the user's text editor.
-  """
+class BaseRepoError(Exception):
+    """All repo specific exceptions derive from BaseRepoError."""
 
-class GitError(Exception):
-  """Unspecified internal error from git.
-  """
-  def __init__(self, command):
-    self.command = command
 
-  def __str__(self):
-    return self.command
+class RepoError(BaseRepoError):
+    """Exceptions thrown inside repo that can be handled."""
 
-class ImportError(Exception):
-  """An import from a non-Git format cannot be performed.
-  """
-  def __init__(self, reason):
-    self.reason = reason
+    def __init__(self, *args, project: str = None) -> None:
+        super().__init__(*args)
+        self.project = project
 
-  def __str__(self):
-    return self.reason
 
-class UploadError(Exception):
-  """A bundle upload to Gerrit did not succeed.
-  """
-  def __init__(self, reason):
-    self.reason = reason
+class RepoExitError(BaseRepoError):
+    """Exception thrown that result in termination of repo program.
+    - Should only be handled in main.py
+    """
 
-  def __str__(self):
-    return self.reason
+    def __init__(
+        self,
+        *args,
+        exit_code: int = 1,
+        aggregate_errors: List[Exception] = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.exit_code = exit_code
+        self.aggregate_errors = aggregate_errors
 
-class NoSuchProjectError(Exception):
-  """A specified project does not exist in the work tree.
-  """
-  def __init__(self, name=None):
-    self.name = name
 
-  def __str__(self):
-    if self.Name is None:
-      return 'in current directory'
-    return self.name
+class RepoUnhandledExceptionError(RepoExitError):
+    """Exception that maintains error as reason for program exit."""
 
-class RepoChangedException(Exception):
-  """Thrown if 'repo sync' results in repo updating its internal
-     repo or manifest repositories.  In this special case we must
-     use exec to re-execute repo with the new code and manifest.
-  """
-  def __init__(self, extra_args=[]):
-    self.extra_args = extra_args
+    def __init__(
+        self,
+        error: BaseException,
+        **kwargs,
+    ) -> None:
+        super().__init__(error, **kwargs)
+        self.error = error
+
+
+class SilentRepoExitError(RepoExitError):
+    """RepoExitError that should no include CLI logging of issue/issues."""
+
+
+class ManifestParseError(RepoExitError):
+    """Failed to parse the manifest file."""
+
+
+class ManifestInvalidRevisionError(ManifestParseError):
+    """The revision value in a project is incorrect."""
+
+
+class ManifestInvalidPathError(ManifestParseError):
+    """A path used in <copyfile> or <linkfile> is incorrect."""
+
+
+class NoManifestException(RepoExitError):
+    """The required manifest does not exist."""
+
+    def __init__(self, path, reason, **kwargs):
+        super().__init__(path, reason, **kwargs)
+        self.path = path
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason
+
+
+class EditorError(RepoError):
+    """Unspecified error from the user's text editor."""
+
+    def __init__(self, reason, **kwargs):
+        super().__init__(reason, **kwargs)
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason
+
+
+class GitError(RepoError):
+    """Unspecified git related error."""
+
+    def __init__(self, message, command_args=None, **kwargs):
+        super().__init__(message, **kwargs)
+        self.message = message
+        self.command_args = command_args
+
+    def __str__(self):
+        return self.message
+
+
+class GitAuthError(RepoExitError):
+    """Cannot talk to remote due to auth issue."""
+
+
+class UploadError(RepoError):
+    """A bundle upload to Gerrit did not succeed."""
+
+    def __init__(self, reason, **kwargs):
+        super().__init__(reason, **kwargs)
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason
+
+
+class DownloadError(RepoExitError):
+    """Cannot download a repository."""
+
+    def __init__(self, reason, **kwargs):
+        super().__init__(reason, **kwargs)
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason
+
+
+class InvalidArgumentsError(RepoExitError):
+    """Invalid command Arguments."""
+
+
+class SyncError(RepoExitError):
+    """Cannot sync repo."""
+
+
+class UpdateManifestError(RepoExitError):
+    """Cannot update manifest."""
+
+
+class NoSuchProjectError(RepoExitError):
+    """A specified project does not exist in the work tree."""
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(**kwargs)
+        self.name = name
+
+    def __str__(self):
+        if self.name is None:
+            return "in current directory"
+        return self.name
+
+
+class InvalidProjectGroupsError(RepoExitError):
+    """A specified project is not suitable for the specified groups"""
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(**kwargs)
+        self.name = name
+
+    def __str__(self):
+        if self.name is None:
+            return "in current directory"
+        return self.name
+
+
+class RepoChangedException(BaseRepoError):
+    """Thrown if 'repo sync' results in repo updating its internal
+    repo or manifest repositories.  In this special case we must
+    use exec to re-execute repo with the new code and manifest.
+    """
+
+    def __init__(self, extra_args=None):
+        super().__init__(extra_args)
+        self.extra_args = extra_args or []
+
+
+class HookError(RepoError):
+    """Thrown if a 'repo-hook' could not be run.
+
+    The common case is that the file wasn't present when we tried to run it.
+    """

event_log.py

@@ -0,0 +1,192 @@
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import multiprocessing
+
+
+TASK_COMMAND = "command"
+TASK_SYNC_NETWORK = "sync-network"
+TASK_SYNC_LOCAL = "sync-local"
+
+
+class EventLog:
+    """Event log that records events that occurred during a repo invocation.
+
+    Events are written to the log as a consecutive JSON entries, one per line.
+    Each entry contains the following keys:
+    - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
+          The ID is only unique for the invocation of the repo command.
+    - name: Name of the object being operated upon.
+    - task_name: The task that was performed.
+    - start: Timestamp of when the operation started.
+    - finish: Timestamp of when the operation finished.
+    - success: Boolean indicating if the operation was successful.
+    - try_count: A counter indicating the try count of this task.
+
+    Optionally:
+    - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
+        events.
+
+    Valid task_names include:
+    - command: The invocation of a subcommand.
+    - sync-network: The network component of a sync command.
+    - sync-local: The local component of a sync command.
+
+    Specific tasks may include additional informational properties.
+    """
+
+    def __init__(self):
+        """Initializes the event log."""
+        self._log = []
+        self._parent = None
+
+    def Add(
+        self,
+        name,
+        task_name,
+        start,
+        finish=None,
+        success=None,
+        try_count=1,
+        kind="RepoOp",
+    ):
+        """Add an event to the log.
+
+        Args:
+            name: Name of the object being operated upon.
+            task_name: A sub-task that was performed for name.
+            start: Timestamp of when the operation started.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
+            try_count: A counter indicating the try count of this task.
+            kind: The kind of the object for the unique identifier.
+
+        Returns:
+            A dictionary of the event added to the log.
+        """
+        event = {
+            "id": (kind, _NextEventId()),
+            "name": name,
+            "task_name": task_name,
+            "start_time": start,
+            "try": try_count,
+        }
+
+        if self._parent:
+            event["parent"] = self._parent["id"]
+
+        if success is not None or finish is not None:
+            self.FinishEvent(event, finish, success)
+
+        self._log.append(event)
+        return event
+
+    def AddSync(self, project, task_name, start, finish, success):
+        """Add a event to the log for a sync command.
+
+        Args:
+            project: Project being synced.
+            task_name: A sub-task that was performed for name.
+                One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
+            start: Timestamp of when the operation started.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
+
+        Returns:
+            A dictionary of the event added to the log.
+        """
+        event = self.Add(project.relpath, task_name, start, finish, success)
+        if event is not None:
+            event["project"] = project.name
+            if project.revisionExpr:
+                event["revision"] = project.revisionExpr
+            if project.remote.url:
+                event["project_url"] = project.remote.url
+            if project.remote.fetchUrl:
+                event["remote_url"] = project.remote.fetchUrl
+            try:
+                event["git_hash"] = project.GetCommitRevisionId()
+            except Exception:
+                pass
+        return event
+
+    def GetStatusString(self, success):
+        """Converst a boolean success to a status string.
+
+        Args:
+            success: Boolean indicating if the operation was successful.
+
+        Returns:
+            status string.
+        """
+        return "pass" if success else "fail"
+
+    def FinishEvent(self, event, finish, success):
+        """Finishes an incomplete event.
+
+        Args:
+            event: An event that has been added to the log.
+            finish: Timestamp of when the operation finished.
+            success: Boolean indicating if the operation was successful.
+
+        Returns:
+            A dictionary of the event added to the log.
+        """
+        event["status"] = self.GetStatusString(success)
+        event["finish_time"] = finish
+        return event
+
+    def SetParent(self, event):
+        """Set a parent event for all new entities.
+
+        Args:
+            event: The event to use as a parent.
+        """
+        self._parent = event
+
+    def Write(self, filename):
+        """Writes the log out to a file.
+
+        Args:
+            filename: The file to write the log to.
+        """
+        with open(filename, "w+") as f:
+            for e in self._log:
+                json.dump(e, f, sort_keys=True)
+                f.write("\n")
+
+
+# An integer id that is unique across this invocation of the program, to be set
+# by the first Add event. We can't set it here since it results in leaked
+# resources (see: https://issues.gerritcodereview.com/353656374).
+_EVENT_ID = None
+
+
+def _NextEventId():
+    """Helper function for grabbing the next unique id.
+
+    Returns:
+        A unique, to this invocation of the program, integer id.
+    """
+    global _EVENT_ID
+    if _EVENT_ID is None:
+        # There is a small chance of race condition - two parallel processes
+        # setting up _EVENT_ID. However, we expect TASK_COMMAND to happen before
+        # mp kicks in.
+        _EVENT_ID = multiprocessing.Value("i", 1)
+    with _EVENT_ID.get_lock():
+        val = _EVENT_ID.value
+        _EVENT_ID.value += 1
+    return val

fetch.py

@@ -0,0 +1,57 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module contains functions used to fetch files from various sources."""
+
+import subprocess
+import sys
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+from error import RepoExitError
+
+
+class FetchFileError(RepoExitError):
+    """Exit error when fetch_file fails."""
+
+
+def fetch_file(url, verbose=False):
+    """Fetch a file from the specified source using the appropriate protocol.
+
+    Returns:
+        The contents of the file as bytes.
+    """
+    scheme = urlparse(url).scheme
+    if scheme == "gs":
+        cmd = ["gsutil", "cat", url]
+        errors = []
+        try:
+            result = subprocess.run(
+                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
+            )
+            if result.stderr and verbose:
+                print(
+                    'warning: non-fatal error running "gsutil": %s'
+                    % result.stderr,
+                    file=sys.stderr,
+                )
+            return result.stdout
+        except subprocess.CalledProcessError as e:
+            errors.append(e)
+            print(
+                'fatal: error running "gsutil": %s' % e.stderr, file=sys.stderr
+            )
+        raise FetchFileError(aggregate_errors=errors)
+    with urlopen(url) as f:
+        return f.read()

git_command.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,152 +12,639 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+import json
 import os
-import sys
+import re
 import subprocess
-from error import GitError
+import sys
+from typing import Any, Optional
 
-GIT = 'git'
-MIN_GIT_VERSION = (1, 5, 4)
-GIT_DIR = 'GIT_DIR'
-REPO_TRACE = 'REPO_TRACE'
+from error import GitError
+from error import RepoExitError
+from git_refs import HEAD
+from git_trace2_event_log_base import BaseEventLog
+import platform_utils
+from repo_logging import RepoLogger
+from repo_trace import IsTrace
+from repo_trace import REPO_TRACE
+from repo_trace import Trace
+from wrapper import Wrapper
+
+
+GIT = "git"
+GIT_DIR = "GIT_DIR"
 
 LAST_GITDIR = None
 LAST_CWD = None
-try:
-  TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  TRACE = False
+DEFAULT_GIT_FAIL_MESSAGE = "git command failure"
+ERROR_EVENT_LOGGING_PREFIX = "RepoGitCommandError"
+# Common line length limit
+GIT_ERROR_STDOUT_LINES = 1
+GIT_ERROR_STDERR_LINES = 10
+INVALID_GIT_EXIT_CODE = 126
+
+logger = RepoLogger(__file__)
 
 
-class _GitCall(object):
-  def version(self):
-    p = GitCommand(None, ['--version'], capture_stdout=True)
-    if p.Wait() == 0:
-      return p.stdout
-    return None
+class _GitCall:
+    @functools.lru_cache(maxsize=None)
+    def version_tuple(self):
+        ret = Wrapper().ParseGitVersion()
+        if ret is None:
+            msg = "fatal: unable to detect git version"
+            logger.error(msg)
+            raise GitRequireError(msg)
+        return ret
+
+    def __getattr__(self, name):
+        name = name.replace("_", "-")
+
+        def fun(*cmdv):
+            command = [name]
+            command.extend(cmdv)
+            return GitCommand(None, command, add_event_log=False).Wait() == 0
+
+        return fun
+
 
-  def __getattr__(self, name):
-    name = name.replace('_','-')
-    def fun(*cmdv):
-      command = [name]
-      command.extend(cmdv)
-      return GitCommand(None, command).Wait() == 0
-    return fun
 git = _GitCall()
 
-class GitCommand(object):
-  def __init__(self,
-               project,
-               cmdv,
-               bare = False,
-               provide_stdin = False,
-               capture_stdout = False,
-               capture_stderr = False,
-               disable_editor = False,
-               cwd = None,
-               gitdir = None):
-    env = dict(os.environ)
 
-    for e in [REPO_TRACE,
-              GIT_DIR,
-              'GIT_ALTERNATE_OBJECT_DIRECTORIES',
-              'GIT_OBJECT_DIRECTORY',
-              'GIT_WORK_TREE',
-              'GIT_GRAFT_FILE',
-              'GIT_INDEX_FILE']:
-      if e in env:
-        del env[e]
+def RepoSourceVersion():
+    """Return the version of the repo.git tree."""
+    ver = getattr(RepoSourceVersion, "version", None)
+
+    # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+    # to initialize version info we provide.
+    if ver is None:
+        env = GitCommand._GetBasicEnv()
+
+        proj = os.path.dirname(os.path.abspath(__file__))
+        env[GIT_DIR] = os.path.join(proj, ".git")
+        result = subprocess.run(
+            [GIT, "describe", HEAD],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            encoding="utf-8",
+            env=env,
+            check=False,
+        )
+        if result.returncode == 0:
+            ver = result.stdout.strip()
+            if ver.startswith("v"):
+                ver = ver[1:]
+        else:
+            ver = "unknown"
+        setattr(RepoSourceVersion, "version", ver)
+
+    return ver
+
+
+@functools.lru_cache(maxsize=None)
+def GetEventTargetPath():
+    """Get the 'trace2.eventtarget' path from git configuration.
+
+    Returns:
+        path: git config's 'trace2.eventtarget' path if it exists, or None
+    """
+    path = None
+    cmd = ["config", "--get", "trace2.eventtarget"]
+    # TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
+    # system git config variables.
+    p = GitCommand(
+        None,
+        cmd,
+        capture_stdout=True,
+        capture_stderr=True,
+        bare=True,
+        add_event_log=False,
+    )
+    retval = p.Wait()
+    if retval == 0:
+        # Strip trailing carriage-return in path.
+        path = p.stdout.rstrip("\n")
+        if path == "":
+            return None
+    elif retval != 1:
+        # `git config --get` is documented to produce an exit status of `1`
+        # if the requested variable is not present in the configuration.
+        # Report any other return value as an error.
+        logger.error(
+            "repo: error: 'git config --get' call failed with return code: "
+            "%r, stderr: %r",
+            retval,
+            p.stderr,
+        )
+    return path
+
+
+class UserAgent:
+    """Mange User-Agent settings when talking to external services
+
+    We follow the style as documented here:
+    https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+    """
+
+    _os = None
+    _repo_ua = None
+    _git_ua = None
+
+    @property
+    def os(self):
+        """The operating system name."""
+        if self._os is None:
+            os_name = sys.platform
+            if os_name.lower().startswith("linux"):
+                os_name = "Linux"
+            elif os_name == "win32":
+                os_name = "Win32"
+            elif os_name == "cygwin":
+                os_name = "Cygwin"
+            elif os_name == "darwin":
+                os_name = "Darwin"
+            self._os = os_name
+
+        return self._os
+
+    @property
+    def repo(self):
+        """The UA when connecting directly from repo."""
+        if self._repo_ua is None:
+            py_version = sys.version_info
+            self._repo_ua = "git-repo/%s (%s) git/%s Python/%d.%d.%d" % (
+                RepoSourceVersion(),
+                self.os,
+                git.version_tuple().full,
+                py_version.major,
+                py_version.minor,
+                py_version.micro,
+            )
+
+        return self._repo_ua
+
+    @property
+    def git(self):
+        """The UA when running git."""
+        if self._git_ua is None:
+            self._git_ua = (
+                f"git/{git.version_tuple().full} ({self.os}) "
+                f"git-repo/{RepoSourceVersion()}"
+            )
+        return self._git_ua
+
+
+user_agent = UserAgent()
+
+
+def git_require(min_version, fail=False, msg=""):
+    git_version = git.version_tuple()
+    if min_version <= git_version:
+        return True
+    if fail:
+        need = ".".join(map(str, min_version))
+        if msg:
+            msg = " for " + msg
+        error_msg = f"fatal: git {need} or later required{msg}"
+        logger.error(error_msg)
+        raise GitRequireError(error_msg)
+    return False
+
+
+def _build_env(
+    _kwargs_only=(),
+    bare: Optional[bool] = False,
+    disable_editor: Optional[bool] = False,
+    ssh_proxy: Optional[Any] = None,
+    gitdir: Optional[str] = None,
+    objdir: Optional[str] = None,
+):
+    """Constucts an env dict for command execution."""
+
+    assert _kwargs_only == (), "_build_env only accepts keyword arguments."
+
+    env = GitCommand._GetBasicEnv()
 
     if disable_editor:
-      env['GIT_EDITOR'] = ':'
+        env["GIT_EDITOR"] = ":"
+    if ssh_proxy:
+        env["REPO_SSH_SOCK"] = ssh_proxy.sock()
+        env["GIT_SSH"] = ssh_proxy.proxy
+        env["GIT_SSH_VARIANT"] = "ssh"
+    if "http_proxy" in env and "darwin" == sys.platform:
+        s = f"'http.proxy={env['http_proxy']}'"
+        p = env.get("GIT_CONFIG_PARAMETERS")
+        if p is not None:
+            s = p + " " + s
+        env["GIT_CONFIG_PARAMETERS"] = s
+    if "GIT_ALLOW_PROTOCOL" not in env:
+        env["GIT_ALLOW_PROTOCOL"] = (
+            "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc"
+        )
+    env["GIT_HTTP_USER_AGENT"] = user_agent.git
 
-    if project:
-      if not cwd:
-        cwd = project.worktree
-      if not gitdir:
-        gitdir = project.gitdir
+    if objdir:
+        # Set to the place we want to save the objects.
+        env["GIT_OBJECT_DIRECTORY"] = objdir
 
-    command = [GIT]
-    if bare:
-      if gitdir:
+        alt_objects = os.path.join(gitdir, "objects") if gitdir else None
+        if alt_objects and os.path.realpath(alt_objects) != os.path.realpath(
+            objdir
+        ):
+            # Allow git to search the original place in case of local or unique
+            # refs that git will attempt to resolve even if we aren't fetching
+            # them.
+            env["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = alt_objects
+    if bare and gitdir is not None:
         env[GIT_DIR] = gitdir
-      cwd = None
-    command.extend(cmdv)
 
-    if provide_stdin:
-      stdin = subprocess.PIPE
-    else:
-      stdin = None
+    return env
 
-    if capture_stdout:
-      stdout = subprocess.PIPE
-    else:
-      stdout = None
 
-    if capture_stderr:
-      stderr = subprocess.PIPE
-    else:
-      stderr = None
+class GitCommand:
+    """Wrapper around a single git invocation."""
 
-    if TRACE:
-      global LAST_CWD
-      global LAST_GITDIR
+    def __init__(
+        self,
+        project,
+        cmdv,
+        bare=False,
+        input=None,
+        capture_stdout=False,
+        capture_stderr=False,
+        merge_output=False,
+        disable_editor=False,
+        ssh_proxy=None,
+        cwd=None,
+        gitdir=None,
+        objdir=None,
+        verify_command=False,
+        add_event_log=True,
+        log_as_error=True,
+    ):
+        if project:
+            if not cwd:
+                cwd = project.worktree
+            if not gitdir:
+                gitdir = project.gitdir
 
-      dbg = ''
+        self.project = project
+        self.cmdv = cmdv
+        self.verify_command = verify_command
+        self.stdout, self.stderr = None, None
 
-      if cwd and LAST_CWD != cwd:
-        if LAST_GITDIR or LAST_CWD:
-          dbg += '\n'
-        dbg += ': cd %s\n' % cwd
-        LAST_CWD = cwd
+        # Git on Windows wants its paths only using / for reliability.
+        if platform_utils.isWindows():
+            if objdir:
+                objdir = objdir.replace("\\", "/")
+            if gitdir:
+                gitdir = gitdir.replace("\\", "/")
 
-      if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
-        if LAST_GITDIR or LAST_CWD:
-          dbg += '\n'
-        dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
-        LAST_GITDIR = env[GIT_DIR]
+        env = _build_env(
+            disable_editor=disable_editor,
+            ssh_proxy=ssh_proxy,
+            objdir=objdir,
+            gitdir=gitdir,
+            bare=bare,
+        )
 
-      dbg += ': '
-      dbg += ' '.join(command)
-      if stdin == subprocess.PIPE:
-        dbg += ' 0<|'
-      if stdout == subprocess.PIPE:
-        dbg += ' 1>|'
-      if stderr == subprocess.PIPE:
-        dbg += ' 2>|'
-      print >>sys.stderr, dbg
+        command = [GIT]
+        if bare:
+            cwd = None
+        command_name = cmdv[0]
+        command.append(command_name)
 
-    try:
-      p = subprocess.Popen(command,
-                           cwd = cwd,
-                           env = env,
-                           stdin = stdin,
-                           stdout = stdout,
-                           stderr = stderr)
-    except Exception, e:
-      raise GitError('%s: %s' % (command[1], e))
+        if command_name in ("fetch", "clone"):
+            env["GIT_TERMINAL_PROMPT"] = "0"
+            # Need to use the --progress flag for fetch/clone so output will be
+            # displayed as by default git only does progress output if stderr is
+            # a TTY.
+            if sys.stderr.isatty():
+                if "--progress" not in cmdv and "--quiet" not in cmdv:
+                    command.append("--progress")
+        command.extend(cmdv[1:])
 
-    self.process = p
-    self.stdin = p.stdin
+        event_log = (
+            BaseEventLog(env=env, add_init_count=True)
+            if add_event_log
+            else None
+        )
 
-  def Wait(self):
-    p = self.process
+        try:
+            self._RunCommand(
+                command,
+                env,
+                capture_stdout=capture_stdout,
+                capture_stderr=capture_stderr,
+                merge_output=merge_output,
+                ssh_proxy=ssh_proxy,
+                cwd=cwd,
+                input=input,
+            )
+            self.VerifyCommand()
+        except GitCommandError as e:
+            if event_log is not None:
+                error_info = json.dumps(
+                    {
+                        "ErrorType": type(e).__name__,
+                        "Project": e.project,
+                        "CommandName": command_name,
+                        "Message": str(e),
+                        "ReturnCode": (
+                            str(e.git_rc) if e.git_rc is not None else None
+                        ),
+                        "IsError": log_as_error,
+                    }
+                )
+                event_log.ErrorEvent(
+                    f"{ERROR_EVENT_LOGGING_PREFIX}:{error_info}"
+                )
+                event_log.Write(GetEventTargetPath())
+            if isinstance(e, GitPopenCommandError):
+                raise
 
-    if p.stdin:
-      p.stdin.close()
-      self.stdin = None
+    def _RunCommand(
+        self,
+        command,
+        env,
+        capture_stdout=False,
+        capture_stderr=False,
+        merge_output=False,
+        ssh_proxy=None,
+        cwd=None,
+        input=None,
+    ):
+        # Set subprocess.PIPE for streams that need to be captured.
+        stdin = subprocess.PIPE if input else None
+        stdout = subprocess.PIPE if capture_stdout else None
+        stderr = (
+            subprocess.STDOUT
+            if merge_output
+            else (subprocess.PIPE if capture_stderr else None)
+        )
 
-    if p.stdout:
-      self.stdout = p.stdout.read()
-      p.stdout.close()
-    else:
-      p.stdout = None
+        # tee_stderr acts like a tee command for stderr, in that, it captures
+        # stderr from the subprocess and streams it back to sys.stderr, while
+        # keeping a copy in-memory.
+        # This allows us to store stderr logs from the subprocess into
+        # GitCommandError.
+        # Certain git operations, such as `git push`, writes diagnostic logs,
+        # such as, progress bar for pushing, into stderr. To ensure we don't
+        # break git's UX, we need to write to sys.stderr as we read from the
+        # subprocess. Setting encoding or errors makes subprocess return
+        # io.TextIOWrapper, which is line buffered. To avoid line-buffering
+        # while tee-ing stderr, we unset these kwargs. See GitCommand._Tee
+        # for tee-ing between the streams.
+        # We tee stderr iff the caller doesn't want to capture any stream to
+        # not disrupt the existing flow.
+        # See go/tee-repo-stderr for more context.
+        tee_stderr = False
+        kwargs = {"encoding": "utf-8", "errors": "backslashreplace"}
+        if not (stdin or stdout or stderr):
+            tee_stderr = True
+            # stderr will be written back to sys.stderr even though it is
+            # piped here.
+            stderr = subprocess.PIPE
+            kwargs = {}
 
-    if p.stderr:
-      self.stderr = p.stderr.read()
-      p.stderr.close()
-    else:
-      p.stderr = None
+        dbg = ""
+        if IsTrace():
+            global LAST_CWD
+            global LAST_GITDIR
 
-    return self.process.wait()
+            if cwd and LAST_CWD != cwd:
+                if LAST_GITDIR or LAST_CWD:
+                    dbg += "\n"
+                dbg += ": cd %s\n" % cwd
+                LAST_CWD = cwd
+
+            if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
+                if LAST_GITDIR or LAST_CWD:
+                    dbg += "\n"
+                dbg += ": export GIT_DIR=%s\n" % env[GIT_DIR]
+                LAST_GITDIR = env[GIT_DIR]
+
+            if "GIT_OBJECT_DIRECTORY" in env:
+                dbg += (
+                    ": export GIT_OBJECT_DIRECTORY=%s\n"
+                    % env["GIT_OBJECT_DIRECTORY"]
+                )
+            if "GIT_ALTERNATE_OBJECT_DIRECTORIES" in env:
+                dbg += ": export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n" % (
+                    env["GIT_ALTERNATE_OBJECT_DIRECTORIES"]
+                )
+
+            dbg += ": "
+            dbg += " ".join(command)
+            if stdin == subprocess.PIPE:
+                dbg += " 0<|"
+            if stdout == subprocess.PIPE:
+                dbg += " 1>|"
+            if stderr == subprocess.PIPE:
+                dbg += " 2>|"
+            elif stderr == subprocess.STDOUT:
+                dbg += " 2>&1"
+
+        with Trace(
+            "git command %s %s with debug: %s", LAST_GITDIR, command, dbg
+        ):
+            try:
+                p = subprocess.Popen(
+                    command,
+                    cwd=cwd,
+                    env=env,
+                    stdin=stdin,
+                    stdout=stdout,
+                    stderr=stderr,
+                    **kwargs,
+                )
+            except Exception as e:
+                raise GitPopenCommandError(
+                    message=f"{command[1]}: {e}",
+                    project=self.project.name if self.project else None,
+                    command_args=self.cmdv,
+                )
+
+            if ssh_proxy:
+                ssh_proxy.add_client(p)
+
+            self.process = p
+
+            try:
+                if tee_stderr:
+                    # tee_stderr streams stderr to sys.stderr while capturing
+                    # a copy within self.stderr. tee_stderr is only enabled
+                    # when the caller wants to pipe no stream.
+                    self.stderr = self._Tee(p.stderr, sys.stderr)
+                else:
+                    self.stdout, self.stderr = p.communicate(input=input)
+            finally:
+                if ssh_proxy:
+                    ssh_proxy.remove_client(p)
+            self.rc = p.wait()
+
+    @staticmethod
+    def _Tee(in_stream, out_stream):
+        """Writes text from in_stream to out_stream while recording in buffer.
+
+        Args:
+            in_stream: I/O stream to be read from.
+            out_stream: I/O stream to write to.
+
+        Returns:
+            A str containing everything read from the in_stream.
+        """
+        buffer = ""
+        read_size = 1024 if sys.version_info < (3, 7) else -1
+        chunk = in_stream.read1(read_size)
+        while chunk:
+            # Convert to str.
+            if not hasattr(chunk, "encode"):
+                chunk = chunk.decode("utf-8", "backslashreplace")
+
+            buffer += chunk
+            out_stream.write(chunk)
+            out_stream.flush()
+
+            chunk = in_stream.read1(read_size)
+
+        return buffer
+
+    @staticmethod
+    def _GetBasicEnv():
+        """Return a basic env for running git under.
+
+        This is guaranteed to be side-effect free.
+        """
+        env = os.environ.copy()
+        for key in (
+            REPO_TRACE,
+            GIT_DIR,
+            "GIT_ALTERNATE_OBJECT_DIRECTORIES",
+            "GIT_OBJECT_DIRECTORY",
+            "GIT_WORK_TREE",
+            "GIT_GRAFT_FILE",
+            "GIT_INDEX_FILE",
+        ):
+            env.pop(key, None)
+        return env
+
+    def VerifyCommand(self):
+        if self.rc == 0:
+            return None
+        stdout = (
+            "\n".join(self.stdout.split("\n")[:GIT_ERROR_STDOUT_LINES])
+            if self.stdout
+            else None
+        )
+        stderr = (
+            "\n".join(self.stderr.split("\n")[:GIT_ERROR_STDERR_LINES])
+            if self.stderr
+            else None
+        )
+        project = self.project.name if self.project else None
+        raise GitCommandError(
+            project=project,
+            command_args=self.cmdv,
+            git_rc=self.rc,
+            git_stdout=stdout,
+            git_stderr=stderr,
+        )
+
+    def Wait(self):
+        if self.verify_command:
+            self.VerifyCommand()
+        return self.rc
+
+
+class GitRequireError(RepoExitError):
+    """Error raised when git version is unavailable or invalid."""
+
+    def __init__(self, message, exit_code: int = INVALID_GIT_EXIT_CODE):
+        super().__init__(message, exit_code=exit_code)
+
+
+class GitCommandError(GitError):
+    """
+    Error raised from a failed git command.
+    Note that GitError can refer to any Git related error (e.g. branch not
+    specified for project.py 'UploadForReview'), while GitCommandError is
+    raised exclusively from non-zero exit codes returned from git commands.
+    """
+
+    # Tuples with error formats and suggestions for those errors.
+    _ERROR_TO_SUGGESTION = [
+        (
+            re.compile("couldn't find remote ref .*"),
+            "Check if the provided ref exists in the remote.",
+        ),
+        (
+            re.compile("unable to access '.*': .*"),
+            (
+                "Please make sure you have the correct access rights and the "
+                "repository exists."
+            ),
+        ),
+        (
+            re.compile("'.*' does not appear to be a git repository"),
+            "Are you running this repo command outside of a repo workspace?",
+        ),
+        (
+            re.compile("not a git repository"),
+            "Are you running this repo command outside of a repo workspace?",
+        ),
+    ]
+
+    def __init__(
+        self,
+        message: str = DEFAULT_GIT_FAIL_MESSAGE,
+        git_rc: int = None,
+        git_stdout: str = None,
+        git_stderr: str = None,
+        **kwargs,
+    ):
+        super().__init__(
+            message,
+            **kwargs,
+        )
+        self.git_rc = git_rc
+        self.git_stdout = git_stdout
+        self.git_stderr = git_stderr
+
+    @property
+    @functools.lru_cache(maxsize=None)
+    def suggestion(self):
+        """Returns helpful next steps for the given stderr."""
+        if not self.git_stderr:
+            return self.git_stderr
+
+        for err, suggestion in self._ERROR_TO_SUGGESTION:
+            if err.search(self.git_stderr):
+                return suggestion
+
+        return None
+
+    def __str__(self):
+        args = "[]" if not self.command_args else " ".join(self.command_args)
+        error_type = type(self).__name__
+        string = f"{error_type}: '{args}' on {self.project} failed"
+
+        if self.message != DEFAULT_GIT_FAIL_MESSAGE:
+            string += f": {self.message}"
+
+        if self.git_stdout:
+            string += f"\nstdout: {self.git_stdout}"
+
+        if self.git_stderr:
+            string += f"\nstderr: {self.git_stderr}"
+
+        if self.suggestion:
+            string += f"\nsuggestion: {self.suggestion}"
+
+        return string
+
+
+class GitPopenCommandError(GitError):
+    """
+    Error raised when subprocess.Popen fails for a GitCommand
+    """

git_refs.py

@@ -0,0 +1,165 @@
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import platform_utils
+from repo_trace import Trace
+
+
+HEAD = "HEAD"
+R_CHANGES = "refs/changes/"
+R_HEADS = "refs/heads/"
+R_TAGS = "refs/tags/"
+R_PUB = "refs/published/"
+R_WORKTREE = "refs/worktree/"
+R_WORKTREE_M = R_WORKTREE + "m/"
+R_M = "refs/remotes/m/"
+
+
+class GitRefs:
+    def __init__(self, gitdir):
+        self._gitdir = gitdir
+        self._phyref = None
+        self._symref = None
+        self._mtime = {}
+
+    @property
+    def all(self):
+        self._EnsureLoaded()
+        return self._phyref
+
+    def get(self, name):
+        try:
+            return self.all[name]
+        except KeyError:
+            return ""
+
+    def deleted(self, name):
+        if self._phyref is not None:
+            if name in self._phyref:
+                del self._phyref[name]
+
+            if name in self._symref:
+                del self._symref[name]
+
+            if name in self._mtime:
+                del self._mtime[name]
+
+    def symref(self, name):
+        try:
+            self._EnsureLoaded()
+            return self._symref[name]
+        except KeyError:
+            return ""
+
+    def _EnsureLoaded(self):
+        if self._phyref is None or self._NeedUpdate():
+            self._LoadAll()
+
+    def _NeedUpdate(self):
+        with Trace(": scan refs %s", self._gitdir):
+            for name, mtime in self._mtime.items():
+                try:
+                    if mtime != os.path.getmtime(
+                        os.path.join(self._gitdir, name)
+                    ):
+                        return True
+                except OSError:
+                    return True
+            return False
+
+    def _LoadAll(self):
+        with Trace(": load refs %s", self._gitdir):
+            self._phyref = {}
+            self._symref = {}
+            self._mtime = {}
+
+            self._ReadPackedRefs()
+            self._ReadLoose("refs/")
+            self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+            scan = self._symref
+            attempts = 0
+            while scan and attempts < 5:
+                scan_next = {}
+                for name, dest in scan.items():
+                    if dest in self._phyref:
+                        self._phyref[name] = self._phyref[dest]
+                    else:
+                        scan_next[name] = dest
+                scan = scan_next
+                attempts += 1
+
+    def _ReadPackedRefs(self):
+        path = os.path.join(self._gitdir, "packed-refs")
+        try:
+            fd = open(path)
+            mtime = os.path.getmtime(path)
+        except OSError:
+            return
+        try:
+            for line in fd:
+                line = str(line)
+                if line[0] == "#":
+                    continue
+                if line[0] == "^":
+                    continue
+
+                line = line[:-1]
+                p = line.split(" ")
+                ref_id = p[0]
+                name = p[1]
+
+                self._phyref[name] = ref_id
+        finally:
+            fd.close()
+        self._mtime["packed-refs"] = mtime
+
+    def _ReadLoose(self, prefix):
+        base = os.path.join(self._gitdir, prefix)
+        for name in platform_utils.listdir(base):
+            p = os.path.join(base, name)
+            # We don't implement the full ref validation algorithm, just the
+            # simple rules that would show up in local filesystems.
+            # https://git-scm.com/docs/git-check-ref-format
+            if name.startswith(".") or name.endswith(".lock"):
+                pass
+            elif platform_utils.isdir(p):
+                self._mtime[prefix] = os.path.getmtime(base)
+                self._ReadLoose(prefix + name + "/")
+            else:
+                self._ReadLoose1(p, prefix + name)
+
+    def _ReadLoose1(self, path, name):
+        try:
+            with open(path) as fd:
+                mtime = os.path.getmtime(path)
+                ref_id = fd.readline()
+        except (OSError, UnicodeError):
+            return
+
+        try:
+            ref_id = ref_id.decode()
+        except AttributeError:
+            pass
+        if not ref_id:
+            return
+        ref_id = ref_id[:-1]
+
+        if ref_id.startswith("ref: "):
+            self._symref[name] = ref_id[5:]
+        else:
+            self._phyref[name] = ref_id
+        self._mtime[name] = mtime

git_ssh

@@ -1,5 +1,6 @@
+#!/bin/sh
 #
-# Copyright (C) 2008 The Android Open Source Project
+# Copyright (C) 2009 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-class Remote(object):
-  def __init__(self, name,
-               fetch=None,
-               review=None,
-               projectName=None):
-    self.name = name
-    self.fetchUrl = fetch
-    self.reviewUrl = review
-    self.projectName = projectName
-    self.requiredCommits = []
+exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@"

git_superproject.py

@@ -0,0 +1,591 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provide functionality to get projects and their commit ids from Superproject.
+
+For more information on superproject, check out:
+https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
+
+Examples:
+  superproject = Superproject(manifest, name, remote, revision)
+  UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
+"""
+
+import functools
+import hashlib
+import os
+import sys
+import time
+from typing import NamedTuple
+import urllib.parse
+
+from git_command import git_require
+from git_command import GitCommand
+from git_config import RepoConfig
+from git_refs import GitRefs
+
+
+_SUPERPROJECT_GIT_NAME = "superproject.git"
+_SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml"
+
+
+class SyncResult(NamedTuple):
+    """Return the status of sync and whether caller should exit."""
+
+    # Whether the superproject sync was successful.
+    success: bool
+    # Whether the caller should exit.
+    fatal: bool
+
+
+class CommitIdsResult(NamedTuple):
+    """Return the commit ids and whether caller should exit."""
+
+    # A dictionary with the projects/commit ids on success, otherwise None.
+    commit_ids: dict
+    # Whether the caller should exit.
+    fatal: bool
+
+
+class UpdateProjectsResult(NamedTuple):
+    """Return the overriding manifest file and whether caller should exit."""
+
+    # Path name of the overriding manifest file if successful, otherwise None.
+    manifest_path: str
+    # Whether the caller should exit.
+    fatal: bool
+
+
+class Superproject:
+    """Get commit ids from superproject.
+
+    Initializes a bare local copy of a superproject for the manifest. This
+    allows lookup of commit ids for all projects. It contains
+    _project_commit_ids which is a dictionary with project/commit id entries.
+    """
+
+    def __init__(
+        self,
+        manifest,
+        name,
+        remote,
+        revision,
+        superproject_dir="exp-superproject",
+    ):
+        """Initializes superproject.
+
+        Args:
+            manifest: A Manifest object that is to be written to a file.
+            name: The unique name of the superproject
+            remote: The RemoteSpec for the remote.
+            revision: The name of the git branch to track.
+            superproject_dir: Relative path under |manifest.subdir| to checkout
+                superproject.
+        """
+        self._project_commit_ids = None
+        self._manifest = manifest
+        self.name = name
+        self.remote = remote
+        self.revision = self._branch = revision
+        self._repodir = manifest.repodir
+        self._superproject_dir = superproject_dir
+        self._superproject_path = manifest.SubmanifestInfoDir(
+            manifest.path_prefix, superproject_dir
+        )
+        self._manifest_path = os.path.join(
+            self._superproject_path, _SUPERPROJECT_MANIFEST_NAME
+        )
+        git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-"
+        self._remote_url = remote.url
+        self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
+        self._work_git = os.path.join(
+            self._superproject_path, self._work_git_name
+        )
+
+        # The following are command arguemnts, rather than superproject
+        # attributes, and were included here originally.  They should eventually
+        # become arguments that are passed down from the public methods, instead
+        # of being treated as attributes.
+        self._git_event_log = None
+        self._quiet = False
+        self._print_messages = False
+
+    def SetQuiet(self, value):
+        """Set the _quiet attribute."""
+        self._quiet = value
+
+    def SetPrintMessages(self, value):
+        """Set the _print_messages attribute."""
+        self._print_messages = value
+
+    @property
+    def commit_id(self):
+        """Returns the commit ID of the superproject checkout."""
+        cmd = ["rev-parse", self.revision]
+        p = GitCommand(
+            None,  # project
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval != 0:
+            self._LogWarning(
+                "git rev-parse call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                p.stdwerr,
+            )
+            return None
+        return p.stdout
+
+    @property
+    def project_commit_ids(self):
+        """Returns a dictionary of projects and their commit ids."""
+        return self._project_commit_ids
+
+    @property
+    def manifest_path(self):
+        """Returns the manifest path if the path exists or None."""
+        return (
+            self._manifest_path if os.path.exists(self._manifest_path) else None
+        )
+
+    @property
+    def repo_id(self):
+        """Returns the repo ID for the superproject.
+
+        For example, if the superproject points to:
+            https://android-review.googlesource.com/platform/superproject/
+        Then the repo_id would be:
+            android/platform/superproject
+        """
+        if review_url := self.remote.review:
+            parsed_url = urllib.parse.urlparse(review_url)
+            if netloc := parsed_url.netloc:
+                parts = netloc.split("-review", 1)
+                host = parts[0]
+                rev = GitRefs(self._work_git).get("HEAD")
+                return f"{host}/{self.name}@{rev}"
+        return None
+
+    def _LogMessage(self, fmt, *inputs):
+        """Logs message to stderr and _git_event_log."""
+        message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}"
+        if self._print_messages:
+            print(message, file=sys.stderr)
+        self._git_event_log.ErrorEvent(message, fmt)
+
+    def _LogMessagePrefix(self):
+        """Returns the prefix string to be logged in each log message"""
+        return (
+            f"repo superproject branch: {self._branch} url: {self._remote_url}"
+        )
+
+    def _LogError(self, fmt, *inputs):
+        """Logs error message to stderr and _git_event_log."""
+        self._LogMessage(f"error: {fmt}", *inputs)
+
+    def _LogWarning(self, fmt, *inputs):
+        """Logs warning message to stderr and _git_event_log."""
+        self._LogMessage(f"warning: {fmt}", *inputs)
+
+    def _Init(self):
+        """Sets up a local Git repository to get a copy of a superproject.
+
+        Returns:
+            True if initialization is successful, or False.
+        """
+        if not os.path.exists(self._superproject_path):
+            os.mkdir(self._superproject_path)
+        if not self._quiet and not os.path.exists(self._work_git):
+            print(
+                "%s: Performing initial setup for superproject; this might "
+                "take several minutes." % self._work_git
+            )
+        cmd = ["init", "--bare", self._work_git_name]
+        p = GitCommand(
+            None,
+            cmd,
+            cwd=self._superproject_path,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval:
+            self._LogWarning(
+                "git init call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return False
+        return True
+
+    def _Fetch(self):
+        """Fetches a superproject for the manifest based on |_remote_url|.
+
+        This runs git fetch which stores a local copy the superproject.
+
+        Returns:
+            True if fetch is successful, or False.
+        """
+        if not os.path.exists(self._work_git):
+            self._LogWarning("git fetch missing directory: {}", self._work_git)
+            return False
+        if not git_require((2, 28, 0)):
+            self._LogWarning(
+                "superproject requires a git version 2.28 or later"
+            )
+            return False
+        cmd = [
+            "fetch",
+            self._remote_url,
+            "--depth",
+            "1",
+            "--force",
+            "--no-tags",
+            "--filter",
+            "blob:none",
+        ]
+
+        # Check if there is a local ref that we can pass to --negotiation-tip.
+        # If this is the first fetch, it does not exist yet.
+        # We use --negotiation-tip to speed up the fetch. Superproject branches
+        # do not share commits. So this lets git know it only needs to send
+        # commits reachable from the specified local refs.
+        rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}")
+        if rev_commit:
+            cmd.extend(["--negotiation-tip", rev_commit])
+
+        if self._branch:
+            cmd += [self._branch + ":" + self._branch]
+        p = GitCommand(
+            None,
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval:
+            self._LogWarning(
+                "git fetch call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return False
+        return True
+
+    def _LsTree(self):
+        """Gets the commit ids for all projects.
+
+        Works only in git repositories.
+
+        Returns:
+            data: data returned from 'git ls-tree ...'. None on error.
+        """
+        if not os.path.exists(self._work_git):
+            self._LogWarning(
+                "git ls-tree missing directory: {}", self._work_git
+            )
+            return None
+        data = None
+        branch = "HEAD" if not self._branch else self._branch
+        cmd = ["ls-tree", "-z", "-r", branch]
+
+        p = GitCommand(
+            None,
+            cmd,
+            gitdir=self._work_git,
+            bare=True,
+            capture_stdout=True,
+            capture_stderr=True,
+        )
+        retval = p.Wait()
+        if retval == 0:
+            data = p.stdout
+        else:
+            self._LogWarning(
+                "git ls-tree call failed, command: git {}, "
+                "return code: {}, stderr: {}",
+                cmd,
+                retval,
+                p.stderr,
+            )
+            return None
+        return data
+
+    def Sync(self, git_event_log):
+        """Gets a local copy of a superproject for the manifest.
+
+        Args:
+            git_event_log: an EventLog, for git tracing.
+
+        Returns:
+            SyncResult
+        """
+        self._git_event_log = git_event_log
+        if not self._manifest.superproject:
+            self._LogWarning(
+                "superproject tag is not defined in manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return SyncResult(False, False)
+
+        should_exit = True
+        if not self._remote_url:
+            self._LogWarning(
+                "superproject URL is not defined in manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return SyncResult(False, should_exit)
+
+        if not self._Init():
+            return SyncResult(False, should_exit)
+        if not self._Fetch():
+            return SyncResult(False, should_exit)
+        if not self._quiet:
+            print(
+                "%s: Initial setup for superproject completed." % self._work_git
+            )
+        return SyncResult(True, False)
+
+    def _GetAllProjectsCommitIds(self):
+        """Get commit ids for all projects from superproject and save them.
+
+        Commit ids are saved in _project_commit_ids.
+
+        Returns:
+            CommitIdsResult
+        """
+        sync_result = self.Sync(self._git_event_log)
+        if not sync_result.success:
+            return CommitIdsResult(None, sync_result.fatal)
+
+        data = self._LsTree()
+        if not data:
+            self._LogWarning(
+                "git ls-tree failed to return data for manifest: {}",
+                self._manifest.manifestFile,
+            )
+            return CommitIdsResult(None, True)
+
+        # Parse lines like the following to select lines starting with '160000'
+        # and build a dictionary with project path (last element) and its commit
+        # id (3rd element).
+        #
+        # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
+        # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00  # noqa: E501
+        commit_ids = {}
+        for line in data.split("\x00"):
+            ls_data = line.split(None, 3)
+            if not ls_data:
+                break
+            if ls_data[0] == "160000":
+                commit_ids[ls_data[3]] = ls_data[2]
+
+        self._project_commit_ids = commit_ids
+        return CommitIdsResult(commit_ids, False)
+
+    def _WriteManifestFile(self):
+        """Writes manifest to a file.
+
+        Returns:
+            manifest_path: Path name of the file into which manifest is written
+                instead of None.
+        """
+        if not os.path.exists(self._superproject_path):
+            self._LogWarning(
+                "missing superproject directory: {}", self._superproject_path
+            )
+            return None
+        manifest_str = self._manifest.ToXml(
+            groups=self._manifest.GetGroupsStr(), omit_local=True
+        ).toxml()
+        manifest_path = self._manifest_path
+        try:
+            with open(manifest_path, "w", encoding="utf-8") as fp:
+                fp.write(manifest_str)
+        except OSError as e:
+            self._LogError("cannot write manifest to : {} {}", manifest_path, e)
+            return None
+        return manifest_path
+
+    def _SkipUpdatingProjectRevisionId(self, project):
+        """Checks if a project's revision id needs to be updated or not.
+
+        Revision id for projects from local manifest will not be updated.
+
+        Args:
+            project: project whose revision id is being updated.
+
+        Returns:
+            True if a project's revision id should not be updated, or False,
+        """
+        path = project.relpath
+        if not path:
+            return True
+        # Skip the project with revisionId.
+        if project.revisionId:
+            return True
+        # Skip the project if it comes from the local manifest.
+        return project.manifest.IsFromLocalManifest(project)
+
+    def UpdateProjectsRevisionId(self, projects, git_event_log):
+        """Update revisionId of every project in projects with the commit id.
+
+        Args:
+            projects: a list of projects whose revisionId needs to be updated.
+            git_event_log: an EventLog, for git tracing.
+
+        Returns:
+            UpdateProjectsResult
+        """
+        self._git_event_log = git_event_log
+        commit_ids_result = self._GetAllProjectsCommitIds()
+        commit_ids = commit_ids_result.commit_ids
+        if not commit_ids:
+            return UpdateProjectsResult(None, commit_ids_result.fatal)
+
+        projects_missing_commit_ids = []
+        for project in projects:
+            if self._SkipUpdatingProjectRevisionId(project):
+                continue
+            path = project.relpath
+            commit_id = commit_ids.get(path)
+            if not commit_id:
+                projects_missing_commit_ids.append(path)
+
+        # If superproject doesn't have a commit id for a project, then report an
+        # error event and continue as if do not use superproject is specified.
+        if projects_missing_commit_ids:
+            self._LogWarning(
+                "please file a bug using {} to report missing "
+                "commit_ids for: {}",
+                self._manifest.contactinfo.bugurl,
+                projects_missing_commit_ids,
+            )
+            return UpdateProjectsResult(None, False)
+
+        for project in projects:
+            if not self._SkipUpdatingProjectRevisionId(project):
+                project.SetRevisionId(commit_ids.get(project.relpath))
+
+        manifest_path = self._WriteManifestFile()
+        return UpdateProjectsResult(manifest_path, False)
+
+
+@functools.lru_cache(maxsize=None)
+def _UseSuperprojectFromConfiguration():
+    """Returns the user choice of whether to use superproject."""
+    user_cfg = RepoConfig.ForUser()
+    time_now = int(time.time())
+
+    user_value = user_cfg.GetBoolean("repo.superprojectChoice")
+    if user_value is not None:
+        user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire")
+        if (
+            user_expiration is None
+            or user_expiration <= 0
+            or user_expiration >= time_now
+        ):
+            # TODO(b/190688390) - Remove prompt when we are comfortable with the
+            # new default value.
+            if user_value:
+                print(
+                    (
+                        "You are currently enrolled in Git submodules "
+                        "experiment (go/android-submodules-quickstart).  Use "
+                        "--no-use-superproject to override.\n"
+                    ),
+                    file=sys.stderr,
+                )
+            else:
+                print(
+                    (
+                        "You are not currently enrolled in Git submodules "
+                        "experiment (go/android-submodules-quickstart).  Use "
+                        "--use-superproject to override.\n"
+                    ),
+                    file=sys.stderr,
+                )
+            return user_value
+
+    # We don't have an unexpired choice, ask for one.
+    system_cfg = RepoConfig.ForSystem()
+    system_value = system_cfg.GetBoolean("repo.superprojectChoice")
+    if system_value:
+        # The system configuration is proposing that we should enable the
+        # use of superproject. Treat the user as enrolled for two weeks.
+        #
+        # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+        # default value.
+        userchoice = True
+        time_choiceexpire = time_now + (86400 * 14)
+        user_cfg.SetString(
+            "repo.superprojectChoiceExpire", str(time_choiceexpire)
+        )
+        user_cfg.SetBoolean("repo.superprojectChoice", userchoice)
+        print(
+            "You are automatically enrolled in Git submodules experiment "
+            "(go/android-submodules-quickstart) for another two weeks.\n",
+            file=sys.stderr,
+        )
+        return True
+
+    # For all other cases, we would not use superproject by default.
+    return False
+
+
+def PrintMessages(use_superproject, manifest):
+    """Returns a boolean if error/warning messages are to be printed.
+
+    Args:
+        use_superproject: option value from optparse.
+        manifest: manifest to use.
+    """
+    return use_superproject is not None or bool(manifest.superproject)
+
+
+def UseSuperproject(use_superproject, manifest):
+    """Returns a boolean if use-superproject option is enabled.
+
+    Args:
+        use_superproject: option value from optparse.
+        manifest: manifest to use.
+
+    Returns:
+        Whether the superproject should be used.
+    """
+
+    if not manifest.superproject:
+        # This (sub) manifest does not have a superproject definition.
+        return False
+    elif use_superproject is not None:
+        return use_superproject
+    else:
+        client_value = manifest.manifestProject.use_superproject
+        if client_value is not None:
+            return client_value
+        elif manifest.superproject:
+            return _UseSuperprojectFromConfiguration()
+        else:
+            return False

git_trace2_event_log.py

@@ -0,0 +1,32 @@
+from git_command import GetEventTargetPath
+from git_command import RepoSourceVersion
+from git_trace2_event_log_base import BaseEventLog
+
+
+class EventLog(BaseEventLog):
+    """Event log that records events that occurred during a repo invocation.
+
+    Events are written to the log as a consecutive JSON entries, one per line.
+    Entries follow the git trace2 EVENT format.
+
+    Each entry contains the following common keys:
+    - event: The event name
+    - sid: session-id - Unique string to allow process instance to be
+          identified.
+    - thread: The thread name.
+    - time: is the UTC time of the event.
+
+    Valid 'event' names and event specific fields are documented here:
+    https://git-scm.com/docs/api-trace2#_event_format
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(repo_source_version=RepoSourceVersion(), **kwargs)
+
+    def Write(self, path=None, **kwargs):
+        if path is None:
+            path = self._GetEventTargetPath()
+        return super().Write(path=path, **kwargs)
+
+    def _GetEventTargetPath(self):
+        return GetEventTargetPath()

git_trace2_event_log_base.py

@@ -0,0 +1,356 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provide event logging in the git trace2 EVENT format.
+
+The git trace2 EVENT format is defined at:
+https://www.kernel.org/pub/software/scm/git/docs/technical/api-trace2.html#_event_format
+https://git-scm.com/docs/api-trace2#_the_event_format_target
+
+  Usage:
+
+  git_trace_log = EventLog()
+  git_trace_log.StartEvent()
+  ...
+  git_trace_log.ExitEvent()
+  git_trace_log.Write()
+"""
+
+
+import datetime
+import errno
+import json
+import os
+import socket
+import sys
+import tempfile
+import threading
+
+
+# Timeout when sending events via socket (applies to connect, send)
+SOCK_TIMEOUT = 0.5  # in seconds
+# BaseEventLog __init__ Counter that is consistent within the same process
+p_init_count = 0
+
+
+class BaseEventLog:
+    """Event log that records events that occurred during a repo invocation.
+
+    Events are written to the log as a consecutive JSON entries, one per line.
+    Entries follow the git trace2 EVENT format.
+
+    Each entry contains the following common keys:
+    - event: The event name
+    - sid: session-id - Unique string to allow process instance to be
+          identified.
+    - thread: The thread name.
+    - time: is the UTC time of the event.
+
+    Valid 'event' names and event specific fields are documented here:
+    https://git-scm.com/docs/api-trace2#_event_format
+    """
+
+    def __init__(
+        self, env=None, repo_source_version=None, add_init_count=False
+    ):
+        """Initializes the event log."""
+        global p_init_count
+        p_init_count += 1
+        self._log = []
+        # Try to get session-id (sid) from environment (setup in repo launcher).
+        KEY = "GIT_TRACE2_PARENT_SID"
+        if env is None:
+            env = os.environ
+
+        self.start = datetime.datetime.now(datetime.timezone.utc)
+
+        # Save both our sid component and the complete sid.
+        # We use our sid component (self._sid) as the unique filename prefix and
+        # the full sid (self._full_sid) in the log itself.
+        self._sid = (
+            f"repo-{self.start.strftime('%Y%m%dT%H%M%SZ')}-P{os.getpid():08x}"
+        )
+
+        if add_init_count:
+            self._sid = f"{self._sid}-{p_init_count}"
+
+        parent_sid = env.get(KEY)
+        # Append our sid component to the parent sid (if it exists).
+        if parent_sid is not None:
+            self._full_sid = parent_sid + "/" + self._sid
+        else:
+            self._full_sid = self._sid
+
+        # Set/update the environment variable.
+        # Environment handling across systems is messy.
+        try:
+            env[KEY] = self._full_sid
+        except UnicodeEncodeError:
+            env[KEY] = self._full_sid.encode()
+
+        if repo_source_version is not None:
+            # Add a version event to front of the log.
+            self._AddVersionEvent(repo_source_version)
+
+    @property
+    def full_sid(self):
+        return self._full_sid
+
+    def _AddVersionEvent(self, repo_source_version):
+        """Adds a 'version' event at the beginning of current log."""
+        version_event = self._CreateEventDict("version")
+        version_event["evt"] = "2"
+        version_event["exe"] = repo_source_version
+        self._log.insert(0, version_event)
+
+    def _CreateEventDict(self, event_name):
+        """Returns a dictionary with common keys/values for git trace2 events.
+
+        Args:
+            event_name: The event name.
+
+        Returns:
+            Dictionary with the common event fields populated.
+        """
+        return {
+            "event": event_name,
+            "sid": self._full_sid,
+            "thread": threading.current_thread().name,
+            "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        }
+
+    def StartEvent(self, argv):
+        """Append a 'start' event to the current log."""
+        start_event = self._CreateEventDict("start")
+        start_event["argv"] = argv
+        self._log.append(start_event)
+
+    def ExitEvent(self, result):
+        """Append an 'exit' event to the current log.
+
+        Args:
+            result: Exit code of the event
+        """
+        exit_event = self._CreateEventDict("exit")
+
+        # Consider 'None' success (consistent with event_log result handling).
+        if result is None:
+            result = 0
+        exit_event["code"] = result
+        time_delta = datetime.datetime.now(datetime.timezone.utc) - self.start
+        exit_event["t_abs"] = time_delta.total_seconds()
+        self._log.append(exit_event)
+
+    def CommandEvent(self, name, subcommands):
+        """Append a 'command' event to the current log.
+
+        Args:
+            name: Name of the primary command (ex: repo, git)
+            subcommands: List of the sub-commands (ex: version, init, sync)
+        """
+        command_event = self._CreateEventDict("cmd_name")
+        name = f"{name}-"
+        name += "-".join(subcommands)
+        command_event["name"] = name
+        command_event["hierarchy"] = name
+        self._log.append(command_event)
+
+    def LogConfigEvents(self, config, event_dict_name):
+        """Append a |event_dict_name| event for each config key in |config|.
+
+        Args:
+            config: Configuration dictionary.
+            event_dict_name: Name of the event dictionary for items to be logged
+                under.
+        """
+        for param, value in config.items():
+            event = self._CreateEventDict(event_dict_name)
+            event["param"] = param
+            event["value"] = value
+            self._log.append(event)
+
+    def DefParamRepoEvents(self, config):
+        """Append 'def_param' events for repo config keys to the current log.
+
+        This appends one event for each repo.* config key.
+
+        Args:
+            config: Repo configuration dictionary
+        """
+        # Only output the repo.* config parameters.
+        repo_config = {k: v for k, v in config.items() if k.startswith("repo.")}
+        self.LogConfigEvents(repo_config, "def_param")
+
+    def GetDataEventName(self, value):
+        """Returns 'data-json' if the value is an array else returns 'data'."""
+        return "data-json" if value[0] == "[" and value[-1] == "]" else "data"
+
+    def LogDataConfigEvents(self, config, prefix):
+        """Append a 'data' event for each entry in |config| to the current log.
+
+        For each keyX and valueX of the config, "key" field of the event is
+        '|prefix|/keyX' and the "value" of the "key" field is valueX.
+
+        Args:
+            config: Configuration dictionary.
+            prefix: Prefix for each key that is logged.
+        """
+        for key, value in config.items():
+            event = self._CreateEventDict(self.GetDataEventName(value))
+            event["key"] = f"{prefix}/{key}"
+            event["value"] = value
+            self._log.append(event)
+
+    def ErrorEvent(self, msg, fmt=None):
+        """Append a 'error' event to the current log."""
+        error_event = self._CreateEventDict("error")
+        if fmt is None:
+            fmt = msg
+        error_event["msg"] = f"RepoErrorEvent:{msg}"
+        error_event["fmt"] = f"RepoErrorEvent:{fmt}"
+        self._log.append(error_event)
+
+    def _WriteLog(self, write_fn):
+        """Writes the log out using a provided writer function.
+
+        Generate compact JSON output for each item in the log, and write it
+        using write_fn.
+
+        Args:
+            write_fn: A function that accepts byts and writes them to a
+                destination.
+        """
+
+        for e in self._log:
+            # Dump in compact encoding mode.
+            # See 'Compact encoding' in Python docs:
+            # https://docs.python.org/3/library/json.html#module-json
+            write_fn(
+                json.dumps(e, indent=None, separators=(",", ":")).encode(
+                    "utf-8"
+                )
+                + b"\n"
+            )
+
+    def Write(self, path=None):
+        """Writes the log out to a file or socket.
+
+        Log is only written if 'path' or 'git config --get trace2.eventtarget'
+        provide a valid path (or socket) to write logs to.
+
+        Logging filename format follows the git trace2 style of being a unique
+        (exclusive writable) file.
+
+        Args:
+            path: Path to where logs should be written. The path may have a
+                prefix of the form "af_unix:[{stream|dgram}:]", in which case
+                the path is treated as a Unix domain socket. See
+                https://git-scm.com/docs/api-trace2#_enabling_a_target for
+                details.
+
+        Returns:
+            log_path: Path to the log file or socket if log is written,
+                otherwise None
+        """
+        log_path = None
+        # If no logging path is specified, exit.
+        if path is None:
+            return None
+
+        path_is_socket = False
+        socket_type = None
+        if isinstance(path, str):
+            parts = path.split(":", 1)
+            if parts[0] == "af_unix" and len(parts) == 2:
+                path_is_socket = True
+                path = parts[1]
+                parts = path.split(":", 1)
+                if parts[0] == "stream" and len(parts) == 2:
+                    socket_type = socket.SOCK_STREAM
+                    path = parts[1]
+                elif parts[0] == "dgram" and len(parts) == 2:
+                    socket_type = socket.SOCK_DGRAM
+                    path = parts[1]
+            else:
+                # Get absolute path.
+                path = os.path.abspath(os.path.expanduser(path))
+        else:
+            raise TypeError("path: str required but got %s." % type(path))
+
+        # Git trace2 requires a directory to write log to.
+
+        # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
+        if not (path_is_socket or os.path.isdir(path)):
+            return None
+
+        if path_is_socket:
+            if socket_type == socket.SOCK_STREAM or socket_type is None:
+                try:
+                    with socket.socket(
+                        socket.AF_UNIX, socket.SOCK_STREAM
+                    ) as sock:
+                        sock.settimeout(SOCK_TIMEOUT)
+                        sock.connect(path)
+                        self._WriteLog(sock.sendall)
+                    return f"af_unix:stream:{path}"
+                except OSError as err:
+                    # If we tried to connect to a DGRAM socket using STREAM,
+                    # ignore the attempt and continue to DGRAM below. Otherwise,
+                    # issue a warning.
+                    if err.errno != errno.EPROTOTYPE:
+                        print(
+                            f"repo: warning: git trace2 logging failed: {err}",
+                            file=sys.stderr,
+                        )
+                        return None
+            if socket_type == socket.SOCK_DGRAM or socket_type is None:
+                try:
+                    with socket.socket(
+                        socket.AF_UNIX, socket.SOCK_DGRAM
+                    ) as sock:
+                        self._WriteLog(lambda bs: sock.sendto(bs, path))
+                        return f"af_unix:dgram:{path}"
+                except OSError as err:
+                    print(
+                        f"repo: warning: git trace2 logging failed: {err}",
+                        file=sys.stderr,
+                    )
+                    return None
+            # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
+            # (SOCK_DGRAM).
+            print(
+                "repo: warning: git trace2 logging failed: could not write to "
+                "socket",
+                file=sys.stderr,
+            )
+            return None
+
+        # Path is an absolute path
+        # Use NamedTemporaryFile to generate a unique filename as required by
+        # git trace2.
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="xb", prefix=self._sid, dir=path, delete=False
+            ) as f:
+                # TODO(https://crbug.com/gerrit/13706): Support writing events
+                # as they occur.
+                self._WriteLog(f.write)
+                log_path = f.name
+        except FileExistsError as err:
+            print(
+                "repo: warning: git trace2 logging failed: %r" % err,
+                file=sys.stderr,
+            )
+            return None
+        return log_path

hooks.py

@@ -0,0 +1,503 @@
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+import traceback
+import urllib.parse
+
+from error import HookError
+from git_refs import HEAD
+
+
+class RepoHook:
+    """A RepoHook contains information about a script to run as a hook.
+
+    Hooks are used to run a python script before running an upload (for
+    instance, to run presubmit checks).  Eventually, we may have hooks for other
+    actions.
+
+    This shouldn't be confused with files in the 'repo/hooks' directory.  Those
+    files are copied into each '.git/hooks' folder for each project.  Repo-level
+    hooks are associated instead with repo actions.
+
+    Hooks are always python.  When a hook is run, we will load the hook into the
+    interpreter and execute its main() function.
+
+    Combinations of hook option flags:
+    - no-verify=False, verify=False (DEFAULT):
+      If stdout is a tty, can prompt about running hooks if needed.
+      If user denies running hooks, the action is cancelled. If stdout is
+      not a tty and we would need to prompt about hooks, action is
+      cancelled.
+    - no-verify=False, verify=True:
+      Always run hooks with no prompt.
+    - no-verify=True, verify=False:
+      Never run hooks, but run action anyway (AKA bypass hooks).
+    - no-verify=True, verify=True:
+      Invalid
+    """
+
+    def __init__(
+        self,
+        hook_type,
+        hooks_project,
+        repo_topdir,
+        manifest_url,
+        bypass_hooks=False,
+        allow_all_hooks=False,
+        ignore_hooks=False,
+        abort_if_user_denies=False,
+    ):
+        """RepoHook constructor.
+
+        Params:
+            hook_type: A string representing the type of hook. This is also used
+                to figure out the name of the file containing the hook. For
+                example: 'pre-upload'.
+            hooks_project: The project containing the repo hooks.
+                If you have a manifest, this is manifest.repo_hooks_project.
+                OK if this is None, which will make the hook a no-op.
+            repo_topdir: The top directory of the repo client checkout.
+                This is the one containing the .repo directory. Scripts will
+                run with CWD as this directory.
+                If you have a manifest, this is manifest.topdir.
+            manifest_url: The URL to the manifest git repo.
+            bypass_hooks: If True, then 'Do not run the hook'.
+            allow_all_hooks: If True, then 'Run the hook without prompting'.
+            ignore_hooks: If True, then 'Do not abort action if hooks fail'.
+            abort_if_user_denies: If True, we'll abort running the hook if the
+                user doesn't allow us to run the hook.
+        """
+        self._hook_type = hook_type
+        self._hooks_project = hooks_project
+        self._repo_topdir = repo_topdir
+        self._manifest_url = manifest_url
+        self._bypass_hooks = bypass_hooks
+        self._allow_all_hooks = allow_all_hooks
+        self._ignore_hooks = ignore_hooks
+        self._abort_if_user_denies = abort_if_user_denies
+
+        # Store the full path to the script for convenience.
+        if self._hooks_project:
+            self._script_fullpath = os.path.join(
+                self._hooks_project.worktree, self._hook_type + ".py"
+            )
+        else:
+            self._script_fullpath = None
+
+    def _GetHash(self):
+        """Return a hash of the contents of the hooks directory.
+
+        We'll just use git to do this.  This hash has the property that if
+        anything changes in the directory we will return a different has.
+
+        SECURITY CONSIDERATION:
+            This hash only represents the contents of files in the hook
+            directory, not any other files imported or called by hooks.  Changes
+            to imported files can change the script behavior without affecting
+            the hash.
+
+        Returns:
+            A string representing the hash.  This will always be ASCII so that
+            it can be printed to the user easily.
+        """
+        assert self._hooks_project, "Must have hooks to calculate their hash."
+
+        # We will use the work_git object rather than just calling
+        # GetRevisionId(). That gives us a hash of the latest checked in version
+        # of the files that the user will actually be executing.  Specifically,
+        # GetRevisionId() doesn't appear to change even if a user checks out a
+        # different version of the hooks repo (via git checkout) nor if a user
+        # commits their own revs.
+        #
+        # NOTE: Local (non-committed) changes will not be factored into this
+        # hash. I think this is OK, since we're really only worried about
+        # warning the user about upstream changes.
+        return self._hooks_project.work_git.rev_parse(HEAD)
+
+    def _GetMustVerb(self):
+        """Return 'must' if the hook is required; 'should' if not."""
+        if self._abort_if_user_denies:
+            return "must"
+        else:
+            return "should"
+
+    def _CheckForHookApproval(self):
+        """Check to see whether this hook has been approved.
+
+        We'll accept approval of manifest URLs if they're using secure
+        transports. This way the user can say they trust the manifest hoster.
+        For insecure hosts, we fall back to checking the hash of the hooks repo.
+
+        Note that we ask permission for each individual hook even though we use
+        the hash of all hooks when detecting changes.  We'd like the user to be
+        able to approve / deny each hook individually.  We only use the hash of
+        all hooks because there is no other easy way to detect changes to local
+        imports.
+
+        Returns:
+            True if this hook is approved to run; False otherwise.
+
+        Raises:
+            HookError: Raised if the user doesn't approve and
+                abort_if_user_denies was passed to the consturctor.
+        """
+        if self._ManifestUrlHasSecureScheme():
+            return self._CheckForHookApprovalManifest()
+        else:
+            return self._CheckForHookApprovalHash()
+
+    def _CheckForHookApprovalHelper(
+        self, subkey, new_val, main_prompt, changed_prompt
+    ):
+        """Check for approval for a particular attribute and hook.
+
+        Args:
+            subkey: The git config key under [repo.hooks.<hook_type>] to store
+                the last approved string.
+            new_val: The new value to compare against the last approved one.
+            main_prompt: Message to display to the user to ask for approval.
+            changed_prompt: Message explaining why we're re-asking for approval.
+
+        Returns:
+            True if this hook is approved to run; False otherwise.
+
+        Raises:
+            HookError: Raised if the user doesn't approve and
+                abort_if_user_denies was passed to the consturctor.
+        """
+        hooks_config = self._hooks_project.config
+        git_approval_key = f"repo.hooks.{self._hook_type}.{subkey}"
+
+        # Get the last value that the user approved for this hook; may be None.
+        old_val = hooks_config.GetString(git_approval_key)
+
+        if old_val is not None:
+            # User previously approved hook and asked not to be prompted again.
+            if new_val == old_val:
+                # Approval matched.  We're done.
+                return True
+            else:
+                # Give the user a reason why we're prompting, since they last
+                # told us to "never ask again".
+                prompt = f"WARNING: {changed_prompt}\n\n"
+        else:
+            prompt = ""
+
+        # Prompt the user if we're not on a tty; on a tty we'll assume "no".
+        if sys.stdout.isatty():
+            prompt += main_prompt + " (yes/always/NO)? "
+            response = input(prompt).lower()
+            print()
+
+            # User is doing a one-time approval.
+            if response in ("y", "yes"):
+                return True
+            elif response == "always":
+                hooks_config.SetString(git_approval_key, new_val)
+                return True
+
+        # For anything else, we'll assume no approval.
+        if self._abort_if_user_denies:
+            raise HookError(
+                "You must allow the %s hook or use --no-verify."
+                % self._hook_type
+            )
+
+        return False
+
+    def _ManifestUrlHasSecureScheme(self):
+        """Check if the URI for the manifest is a secure transport."""
+        secure_schemes = (
+            "file",
+            "https",
+            "ssh",
+            "persistent-https",
+            "sso",
+            "rpc",
+        )
+        parse_results = urllib.parse.urlparse(self._manifest_url)
+        return parse_results.scheme in secure_schemes
+
+    def _CheckForHookApprovalManifest(self):
+        """Check whether the user has approved this manifest host.
+
+        Returns:
+            True if this hook is approved to run; False otherwise.
+        """
+        return self._CheckForHookApprovalHelper(
+            "approvedmanifest",
+            self._manifest_url,
+            f"Run hook scripts from {self._manifest_url}",
+            f"Manifest URL has changed since {self._hook_type} was allowed.",
+        )
+
+    def _CheckForHookApprovalHash(self):
+        """Check whether the user has approved the hooks repo.
+
+        Returns:
+            True if this hook is approved to run; False otherwise.
+        """
+        prompt = (
+            "Repo %s run the script:\n"
+            "  %s\n"
+            "\n"
+            "Do you want to allow this script to run"
+        )
+        return self._CheckForHookApprovalHelper(
+            "approvedhash",
+            self._GetHash(),
+            prompt % (self._GetMustVerb(), self._script_fullpath),
+            f"Scripts have changed since {self._hook_type} was allowed.",
+        )
+
+    @staticmethod
+    def _ExtractInterpFromShebang(data):
+        """Extract the interpreter used in the shebang.
+
+        Try to locate the interpreter the script is using (ignoring `env`).
+
+        Args:
+            data: The file content of the script.
+
+        Returns:
+            The basename of the main script interpreter, or None if a shebang is
+            not used or could not be parsed out.
+        """
+        firstline = data.splitlines()[:1]
+        if not firstline:
+            return None
+
+        # The format here can be tricky.
+        shebang = firstline[0].strip()
+        m = re.match(r"^#!\s*([^\s]+)(?:\s+([^\s]+))?", shebang)
+        if not m:
+            return None
+
+        # If the using `env`, find the target program.
+        interp = m.group(1)
+        if os.path.basename(interp) == "env":
+            interp = m.group(2)
+
+        return interp
+
+    def _ExecuteHookViaImport(self, data, context, **kwargs):
+        """Execute the hook code in |data| directly.
+
+        Args:
+            data: The code of the hook to execute.
+            context: Basic Python context to execute the hook inside.
+            kwargs: Arbitrary arguments to pass to the hook script.
+
+        Raises:
+            HookError: When the hooks failed for any reason.
+        """
+        # Exec, storing global context in the context dict.  We catch exceptions
+        # and convert to a HookError w/ just the failing traceback.
+        try:
+            exec(compile(data, self._script_fullpath, "exec"), context)
+        except Exception:
+            raise HookError(
+                "%s\nFailed to import %s hook; see traceback above."
+                % (traceback.format_exc(), self._hook_type)
+            )
+
+        # Running the script should have defined a main() function.
+        if "main" not in context:
+            raise HookError('Missing main() in: "%s"' % self._script_fullpath)
+
+        # Call the main function in the hook.  If the hook should cause the
+        # build to fail, it will raise an Exception.  We'll catch that convert
+        # to a HookError w/ just the failing traceback.
+        try:
+            context["main"](**kwargs)
+        except Exception:
+            raise HookError(
+                "%s\nFailed to run main() for %s hook; see traceback "
+                "above." % (traceback.format_exc(), self._hook_type)
+            )
+
+    def _ExecuteHook(self, **kwargs):
+        """Actually execute the given hook.
+
+        This will run the hook's 'main' function in our python interpreter.
+
+        Args:
+            kwargs: Keyword arguments to pass to the hook.  These are often
+                specific to the hook type.  For instance, pre-upload hooks will
+                contain a project_list.
+        """
+        # Keep sys.path and CWD stashed away so that we can always restore them
+        # upon function exit.
+        orig_path = os.getcwd()
+        orig_syspath = sys.path
+
+        try:
+            # Always run hooks with CWD as topdir.
+            os.chdir(self._repo_topdir)
+
+            # Put the hook dir as the first item of sys.path so hooks can do
+            # relative imports.  We want to replace the repo dir as [0] so
+            # hooks can't import repo files.
+            sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
+
+            # Initial global context for the hook to run within.
+            context = {"__file__": self._script_fullpath}
+
+            # Add 'hook_should_take_kwargs' to the arguments to be passed to
+            # main. We don't actually want hooks to define their main with this
+            # argument--it's there to remind them that their hook should always
+            # take **kwargs.
+            # For instance, a pre-upload hook should be defined like:
+            #   def main(project_list, **kwargs):
+            #
+            # This allows us to later expand the API without breaking old hooks.
+            kwargs = kwargs.copy()
+            kwargs["hook_should_take_kwargs"] = True
+
+            # See what version of python the hook has been written against.
+            data = open(self._script_fullpath).read()
+            interp = self._ExtractInterpFromShebang(data)
+            if interp:
+                prog = os.path.basename(interp)
+                if prog.startswith("python2"):
+                    raise HookError("Python 2 is not supported")
+
+            # Run the hook by importing directly.
+            self._ExecuteHookViaImport(data, context, **kwargs)
+        finally:
+            # Restore sys.path and CWD.
+            sys.path = orig_syspath
+            os.chdir(orig_path)
+
+    def _CheckHook(self):
+        # Bail with a nice error if we can't find the hook.
+        if not os.path.isfile(self._script_fullpath):
+            raise HookError(
+                "Couldn't find repo hook: %s" % self._script_fullpath
+            )
+
+    def Run(self, **kwargs):
+        """Run the hook.
+
+        If the hook doesn't exist (because there is no hooks project or because
+        this particular hook is not enabled), this is a no-op.
+
+        Args:
+            user_allows_all_hooks: If True, we will never prompt about running
+                the hook--we'll just assume it's OK to run it.
+            kwargs: Keyword arguments to pass to the hook.  These are often
+                specific to the hook type.  For instance, pre-upload hooks will
+                contain a project_list.
+
+        Returns:
+            True: On success or ignore hooks by user-request
+            False: The hook failed. The caller should respond with aborting the
+                action. Some examples in which False is returned:
+                * Finding the hook failed while it was enabled, or
+                * the user declined to run a required hook (from
+                    _CheckForHookApproval)
+                In all these cases the user did not pass the proper arguments to
+                ignore the result through the option combinations as listed in
+                AddHookOptionGroup().
+        """
+        # Do not do anything in case bypass_hooks is set, or
+        # no-op if there is no hooks project or if hook is disabled.
+        if (
+            self._bypass_hooks
+            or not self._hooks_project
+            or self._hook_type not in self._hooks_project.enabled_repo_hooks
+        ):
+            return True
+
+        passed = True
+        try:
+            self._CheckHook()
+
+            # Make sure the user is OK with running the hook.
+            if self._allow_all_hooks or self._CheckForHookApproval():
+                # Run the hook with the same version of python we're using.
+                self._ExecuteHook(**kwargs)
+        except SystemExit as e:
+            passed = False
+            print(
+                "ERROR: %s hooks exited with exit code: %s"
+                % (self._hook_type, str(e)),
+                file=sys.stderr,
+            )
+        except HookError as e:
+            passed = False
+            print("ERROR: %s" % str(e), file=sys.stderr)
+
+        if not passed and self._ignore_hooks:
+            print(
+                "\nWARNING: %s hooks failed, but continuing anyways."
+                % self._hook_type,
+                file=sys.stderr,
+            )
+            passed = True
+
+        return passed
+
+    @classmethod
+    def FromSubcmd(cls, manifest, opt, *args, **kwargs):
+        """Method to construct the repo hook class
+
+        Args:
+            manifest: The current active manifest for this command from which we
+                extract a couple of fields.
+            opt: Contains the commandline options for the action of this hook.
+                It should contain the options added by AddHookOptionGroup() in
+                which we are interested in RepoHook execution.
+        """
+        for key in ("bypass_hooks", "allow_all_hooks", "ignore_hooks"):
+            kwargs.setdefault(key, getattr(opt, key))
+        kwargs.update(
+            {
+                "hooks_project": manifest.repo_hooks_project,
+                "repo_topdir": manifest.topdir,
+                "manifest_url": manifest.manifestProject.GetRemote(
+                    "origin"
+                ).url,
+            }
+        )
+        return cls(*args, **kwargs)
+
+    @staticmethod
+    def AddOptionGroup(parser, name):
+        """Help options relating to the various hooks."""
+
+        # Note that verify and no-verify are NOT opposites of each other, which
+        # is why they store to different locations. We are using them to match
+        # 'git commit' syntax.
+        group = parser.add_option_group(name + " hooks")
+        group.add_option(
+            "--no-verify",
+            dest="bypass_hooks",
+            action="store_true",
+            help="Do not run the %s hook." % name,
+        )
+        group.add_option(
+            "--verify",
+            dest="allow_all_hooks",
+            action="store_true",
+            help="Run the %s hook without prompting." % name,
+        )
+        group.add_option(
+            "--ignore-hooks",
+            action="store_true",
+            help="Do not abort if %s hooks fail." % name,
+        )

hooks/commit-msg

@@ -0,0 +1,113 @@
+#!/bin/sh
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://gerrit.googlesource.com/gerrit/
+# This is synced from commit: 62f5bbea67f6dafa6e22a601a0c298214c510caf
+# DO NOT EDIT THIS FILE
+#
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u
+
+# avoid [[ which is not POSIX sh.
+if test "$#" != 1 ; then
+  echo "$0 requires an argument."
+  exit 1
+fi
+
+if test ! -f "$1" ; then
+  echo "file does not exist: $1"
+  exit 1
+fi
+
+# Do not create a change id if requested
+case "$(git config --get gerrit.createChangeId)" in
+  false)
+    exit 0
+    ;;
+  always)
+    ;;
+  *)
+    # Do not create a change id for squash/fixup commits.
+    if head -n1 "$1" | LC_ALL=C grep -q '^[a-z][a-z]*! '; then
+      exit 0
+    fi
+    ;;
+esac
+
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+  refhash="$(git rev-parse HEAD)"
+else
+  refhash="$(git hash-object -t tree /dev/null)"
+fi
+
+random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin)
+dest="$1.tmp.${random}"
+
+trap 'rm -f "$dest" "$dest-2"' EXIT
+
+if ! cat "$1" | sed -e '/>8/q' | git stripspace --strip-comments > "${dest}" ; then
+   echo "cannot strip comments from $1"
+   exit 1
+fi
+
+if test ! -s "${dest}" ; then
+  echo "file is empty: $1"
+  exit 1
+fi
+
+reviewurl="$(git config --get gerrit.reviewUrl)"
+if test -n "${reviewurl}" ; then
+  token="Link"
+  value="${reviewurl%/}/id/I$random"
+  pattern=".*/id/I[0-9a-f]\{40\}"
+else
+  token="Change-Id"
+  value="I$random"
+  pattern=".*"
+fi
+
+if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then
+  exit 0
+fi
+
+# There must be a Signed-off-by trailer for the code below to work. Insert a
+# sentinel at the end to make sure there is one.
+# Avoid the --in-place option which only appeared in Git 2.8
+if ! git interpret-trailers \
+         --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then
+  echo "cannot insert Signed-off-by sentinel line in $1"
+  exit 1
+fi
+
+# Make sure the trailer appears before any Signed-off-by trailers by inserting
+# it as if it was a Signed-off-by trailer and then use sed to remove the
+# Signed-off-by prefix and the Signed-off-by sentinel line.
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --where option which only appeared in Git 2.15
+if ! git -c trailer.where=before interpret-trailers \
+         --trailer "Signed-off-by: $token: $value" < "$dest-2" |
+     sed -e "s/^Signed-off-by: \($token: \)/\1/" \
+         -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
+  echo "cannot insert $token line in $1"
+  exit 1
+fi
+
+if ! mv "${dest}" "$1" ; then
+  echo "cannot mv ${dest} to $1"
+  exit 1
+fi

hooks/pre-auto-gc

@@ -1,25 +1,27 @@
 #!/bin/sh
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://github.com/git/git
+# This is synced from commit: 00e10ef10e161a913893b8cb33aa080d4ca5baa6
+# DO NOT EDIT THIS FILE
 #
 # An example hook script to verify if you are on battery, in case you
 # are running Linux or OS X. Called by git-gc --auto with no arguments.
 # The hook should exit with non-zero status after issuing an appropriate
 # message if it wants to stop the auto repacking.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This hook is stored in the contrib/hooks directory. Your distribution
+# may have put this somewhere else. If you want to use this hook, you
+# should make this script executable then link to it in the repository
+# you would like to use it in.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# For example, if the hook is stored in
+# /usr/share/git-core/contrib/hooks/pre-auto-gc-battery:
+#
+# cd /path/to/your/repository.git
+# ln -sf /usr/share/git-core/contrib/hooks/pre-auto-gc-battery \
+#	hooks/pre-auto-gc
 
-if test -x /sbin/on_ac_power && /sbin/on_ac_power
+if test -x /sbin/on_ac_power && (/sbin/on_ac_power;test $? -ne 1)
 then
 	exit 0
 elif test "$(cat /sys/class/power_supply/AC/online 2>/dev/null)" = 1
@@ -35,7 +37,7 @@ elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
 then
 	exit 0
 elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
-	grep -q "Currently drawing from 'AC Power'"
+	grep -q "drawing from 'AC Power'"
 then
 	exit 0
 fi

man/repo-abandon.1

@@ -0,0 +1,49 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo abandon" "Repo Manual"
+.SH NAME
+repo \- repo abandon - manual page for repo abandon
+.SH SYNOPSIS
+.B repo
+\fI\,abandon \/\fR[\fI\,--all | <branchname>\/\fR] [\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Permanently abandon a development branch
+.PP
+This subcommand permanently abandons a development branch by
+deleting it (and all its history) from your local repository.
+.PP
+It is equivalent to "git branch \fB\-D\fR <branchname>".
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-\-all\fR
+delete all branches in all projects
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help abandon` to view the detailed manual.

man/repo-branch.1

@@ -0,0 +1 @@
+.so man1/repo-branches.1

man/repo-branches.1

@@ -0,0 +1,72 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo branches" "Repo Manual"
+.SH NAME
+repo \- repo branches - manual page for repo branches
+.SH SYNOPSIS
+.B repo
+\fI\,branches \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+View current topic branches
+.PP
+Summarizes the currently available topic branches.
+.PP
+# Branch Display
+.PP
+The branch display output by this command is organized into four
+columns of information; for example:
+.TP
+*P nocolor
+| in repo
+.TP
+repo2
+|
+.PP
+The first column contains a * if the branch is the currently
+checked out branch in any of the specified projects, or a blank
+if no project has the branch checked out.
+.PP
+The second column contains either blank, p or P, depending upon
+the upload status of the branch.
+.IP
+(blank): branch not yet published by repo upload
+.IP
+P: all commits were published by repo upload
+p: only some commits were published by repo upload
+.PP
+The third column contains the branch name.
+.PP
+The fourth column (after the | separator) lists the projects that
+the branch appears in, or does not appear in.  If no project list
+is shown, then the branch appears in all projects.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help branches` to view the detailed manual.

man/repo-checkout.1

@@ -0,0 +1,49 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo checkout" "Repo Manual"
+.SH NAME
+repo \- repo checkout - manual page for repo checkout
+.SH SYNOPSIS
+.B repo
+\fI\,checkout <branchname> \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Checkout a branch for development
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help checkout` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo checkout' command checks out an existing branch that was previously
+created by 'repo start'.
+.PP
+The command is equivalent to:
+.IP
+repo forall [<project>...] \fB\-c\fR git checkout <branchname>

man/repo-cherry-pick.1

@@ -0,0 +1,41 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo cherry-pick" "Repo Manual"
+.SH NAME
+repo \- repo cherry-pick - manual page for repo cherry-pick
+.SH SYNOPSIS
+.B repo
+\fI\,cherry-pick <sha1>\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Cherry\-pick a change.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help cherry\-pick` to view the detailed manual.
+.SH DETAILS
+.PP
+\&'repo cherry\-pick' cherry\-picks a change from one branch to another. The change
+id will be updated, and a reference to the old change id will be added.

man/repo-diff.1

@@ -0,0 +1,48 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo diff" "Repo Manual"
+.SH NAME
+repo \- repo diff - manual page for repo diff
+.SH SYNOPSIS
+.B repo
+\fI\,diff \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Show changes between commit and working tree
+.PP
+The \fB\-u\fR option causes 'repo diff' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-u\fR, \fB\-\-absolute\fR
+paths are relative to the repository root
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help diff` to view the detailed manual.

man/repo-diffmanifests.1

@@ -0,0 +1,74 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo diffmanifests" "Repo Manual"
+.SH NAME
+repo \- repo diffmanifests - manual page for repo diffmanifests
+.SH SYNOPSIS
+.B repo
+\fI\,diffmanifests manifest1.xml \/\fR[\fI\,manifest2.xml\/\fR] [\fI\,options\/\fR]
+.SH DESCRIPTION
+Summary
+.PP
+Manifest diff utility
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-\-raw\fR
+display raw diff
+.TP
+\fB\-\-no\-color\fR
+does not display the diff in color
+.TP
+\fB\-\-pretty\-format=\fR<FORMAT>
+print the log using a custom git pretty format string
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help diffmanifests` to view the detailed manual.
+.SH DETAILS
+.PP
+The repo diffmanifests command shows differences between project revisions of
+manifest1 and manifest2. if manifest2 is not specified, current manifest.xml
+will be used instead. Both absolute and relative paths may be used for
+manifests. Relative paths start from project's ".repo/manifests" folder.
+.PP
+The \fB\-\-raw\fR option Displays the diff in a way that facilitates parsing, the
+project pattern will be <status> <path> <revision from> [<revision to>] and the
+commit pattern will be <status> <onelined log> with status values respectively :
+.IP
+A = Added project
+R = Removed project
+C = Changed project
+U = Project with unreachable revision(s) (revision(s) not found)
+.PP
+for project, and
+.IP
+A = Added commit
+R = Removed commit
+.PP
+for a commit.
+.PP
+Only changed projects may contain commits, and commit status always starts with
+a space, and are part of last printed project. Unreachable revisions may occur
+if project is not up to date or if repo has not been initialized with all the
+groups, in which case some projects won't be synced and their revisions won't be
+found.

man/repo-download.1

@@ -0,0 +1,57 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo download" "Repo Manual"
+.SH NAME
+repo \- repo download - manual page for repo download
+.SH SYNOPSIS
+.B repo
+\fI\,download {\/\fR[\fI\,project\/\fR] \fI\,change\/\fR[\fI\,/patchset\/\fR]\fI\,}\/\fR...
+.SH DESCRIPTION
+Summary
+.PP
+Download and checkout a change
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-b\fR BRANCH, \fB\-\-branch\fR=\fI\,BRANCH\/\fR
+create a new branch first
+.TP
+\fB\-c\fR, \fB\-\-cherry\-pick\fR
+cherry\-pick instead of checkout
+.TP
+\fB\-x\fR, \fB\-\-record\-origin\fR
+pass \fB\-x\fR when cherry\-picking
+.TP
+\fB\-r\fR, \fB\-\-revert\fR
+revert instead of checkout
+.TP
+\fB\-f\fR, \fB\-\-ff\-only\fR
+force fast\-forward merge
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help download` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo download' command downloads a change from the review system and makes
+it available in your project's local working directory. If no project is
+specified try to use current directory as a project.

man/repo-forall.1

@@ -0,0 +1,146 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo forall" "Repo Manual"
+.SH NAME
+repo \- repo forall - manual page for repo forall
+.SH SYNOPSIS
+.B repo
+\fI\,forall \/\fR[\fI\,<project>\/\fR...] \fI\,-c <command> \/\fR[\fI\,<arg>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Run a shell command in each project
+.PP
+repo forall \fB\-r\fR str1 [str2] ... \fB\-c\fR <command> [<arg>...]
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-r\fR, \fB\-\-regex\fR
+execute the command only on projects matching regex or
+wildcard expression
+.TP
+\fB\-i\fR, \fB\-\-inverse\-regex\fR
+execute the command only on projects not matching
+regex or wildcard expression
+.TP
+\fB\-g\fR GROUPS, \fB\-\-groups\fR=\fI\,GROUPS\/\fR
+execute the command only on projects matching the
+specified groups
+.TP
+\fB\-c\fR, \fB\-\-command\fR
+command (and arguments) to execute
+.TP
+\fB\-e\fR, \fB\-\-abort\-on\-errors\fR
+abort if a command exits unsuccessfully
+.TP
+\fB\-\-ignore\-missing\fR
+silently skip & do not exit non\-zero due missing
+checkouts
+.TP
+\fB\-\-interactive\fR
+force interactive usage
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.TP
+\fB\-p\fR
+show project headers before output
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help forall` to view the detailed manual.
+.SH DETAILS
+.PP
+Executes the same shell command in each project.
+.PP
+The \fB\-r\fR option allows running the command only on projects matching regex or
+wildcard expression.
+.PP
+By default, projects are processed non\-interactively in parallel. If you want to
+run interactive commands, make sure to pass \fB\-\-interactive\fR to force \fB\-\-jobs\fR 1.
+While the processing order of projects is not guaranteed, the order of project
+output is stable.
+.PP
+Output Formatting
+.PP
+The \fB\-p\fR option causes 'repo forall' to bind pipes to the command's stdin, stdout
+and stderr streams, and pipe all output into a continuous stream that is
+displayed in a single pager session. Project headings are inserted before the
+output of each command is displayed. If the command produces no output in a
+project, no heading is displayed.
+.PP
+The formatting convention used by \fB\-p\fR is very suitable for some types of
+searching, e.g. `repo forall \fB\-p\fR \fB\-c\fR git log \fB\-SFoo\fR` will print all commits that
+add or remove references to Foo.
+.PP
+The \fB\-v\fR option causes 'repo forall' to display stderr messages if a command
+produces output only on stderr. Normally the \fB\-p\fR option causes command output to
+be suppressed until the command produces at least one byte of output on stdout.
+.PP
+Environment
+.PP
+pwd is the project's working directory. If the current client is a mirror
+client, then pwd is the Git repository.
+.PP
+REPO_PROJECT is set to the unique name of the project.
+.PP
+REPO_PATH is the path relative the the root of the client.
+.PP
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+.PP
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+.PP
+REPO_REMOTE is the name of the remote system from the manifest.
+.PP
+REPO_LREV is the name of the revision from the manifest, translated to a local
+tracking branch. If you need to pass the manifest revision to a locally executed
+git command, use REPO_LREV.
+.PP
+REPO_RREV is the name of the revision from the manifest, exactly as written in
+the manifest.
+.PP
+REPO_COUNT is the total number of projects being iterated.
+.PP
+REPO_I is the current (1\-based) iteration count. Can be used in conjunction with
+REPO_COUNT to add a simple progress indicator to your command.
+.PP
+REPO__* are any extra environment variables, specified by the "annotation"
+element under any project element. This can be useful for differentiating trees
+based on user\-specific criteria, or simply annotating tree details.
+.PP
+shell positional arguments ($1, $2, .., $#) are set to any arguments following
+<command>.
+.PP
+Example: to list projects:
+.IP
+repo forall \fB\-c\fR 'echo $REPO_PROJECT'
+.PP
+Notice that $REPO_PROJECT is quoted to ensure it is expanded in the context of
+running <command> instead of in the calling shell.
+.PP
+Unless \fB\-p\fR is used, stdin, stdout, stderr are inherited from the terminal and are
+not redirected.
+.PP
+If \fB\-e\fR is used, when a command exits unsuccessfully, 'repo forall' will abort
+without iterating through the remaining projects.

man/repo-gc.1

@@ -0,0 +1,47 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "April 2025" "repo gc" "Repo Manual"
+.SH NAME
+repo \- repo gc - manual page for repo gc
+.SH SYNOPSIS
+.B repo
+\fI\,gc\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Cleaning up internal repo and Git state.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-n\fR, \fB\-\-dry\-run\fR
+do everything except actually delete
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
+.TP
+\fB\-\-repack\fR
+repack all projects that use partial clone with
+filter=blob:none
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help gc` to view the detailed manual.

man/repo-grep.1

@@ -0,0 +1,132 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo grep" "Repo Manual"
+.SH NAME
+repo \- repo grep - manual page for repo grep
+.SH SYNOPSIS
+.B repo
+\fI\,grep {pattern | -e pattern} \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Print lines matching a pattern
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.SS Logging options:
+.TP
+\fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.SS Sources:
+.TP
+\fB\-\-cached\fR
+Search the index, instead of the work tree
+.TP
+\fB\-r\fR TREEish, \fB\-\-revision\fR=\fI\,TREEish\/\fR
+Search TREEish, instead of the work tree
+.SS Pattern:
+.TP
+\fB\-e\fR PATTERN
+Pattern to search for
+.TP
+\fB\-i\fR, \fB\-\-ignore\-case\fR
+Ignore case differences
+.TP
+\fB\-a\fR, \fB\-\-text\fR
+Process binary files as if they were text
+.TP
+\fB\-I\fR
+Don't match the pattern in binary files
+.TP
+\fB\-w\fR, \fB\-\-word\-regexp\fR
+Match the pattern only at word boundaries
+.TP
+\fB\-v\fR, \fB\-\-invert\-match\fR
+Select non\-matching lines
+.TP
+\fB\-G\fR, \fB\-\-basic\-regexp\fR
+Use POSIX basic regexp for patterns (default)
+.TP
+\fB\-E\fR, \fB\-\-extended\-regexp\fR
+Use POSIX extended regexp for patterns
+.TP
+\fB\-F\fR, \fB\-\-fixed\-strings\fR
+Use fixed strings (not regexp) for pattern
+.SS Pattern Grouping:
+.TP
+\fB\-\-all\-match\fR
+Limit match to lines that have all patterns
+.TP
+\fB\-\-and\fR, \fB\-\-or\fR, \fB\-\-not\fR
+Boolean operators to combine patterns
+.TP
+\-(, \-)
+Boolean operator grouping
+.SS Output:
+.TP
+\fB\-n\fR
+Prefix the line number to matching lines
+.TP
+\fB\-C\fR CONTEXT
+Show CONTEXT lines around match
+.TP
+\fB\-B\fR CONTEXT
+Show CONTEXT lines before match
+.TP
+\fB\-A\fR CONTEXT
+Show CONTEXT lines after match
+.TP
+\fB\-l\fR, \fB\-\-name\-only\fR, \fB\-\-files\-with\-matches\fR
+Show only file names containing matching lines
+.TP
+\fB\-L\fR, \fB\-\-files\-without\-match\fR
+Show only file names not containing matching lines
+.PP
+Run `repo help grep` to view the detailed manual.
+.SH DETAILS
+.PP
+Search for the specified patterns in all project files.
+.PP
+Boolean Options
+.PP
+The following options can appear as often as necessary to express the pattern to
+locate:
+.HP
+\fB\-e\fR PATTERN
+.HP
+\fB\-\-and\fR, \fB\-\-or\fR, \fB\-\-not\fR, \-(, \-)
+.PP
+Further, the \fB\-r\fR/\-\-revision option may be specified multiple times in order to
+scan multiple trees. If the same file matches in more than one tree, only the
+first result is reported, prefixed by the revision name it was found under.
+.PP
+Examples
+.PP
+Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
+.IP
+repo grep \fB\-e\fR '#define' \fB\-\-and\fR \-\e( \fB\-e\fR MAX_PATH \fB\-e\fR PATH_MAX \e)
+.PP
+Look for a line that has 'NODE' or 'Unexpected' in files that contain a line
+that matches both expressions:
+.IP
+repo grep \fB\-\-all\-match\fR \fB\-e\fR NODE \fB\-e\fR Unexpected

man/repo-help.1

@@ -0,0 +1,46 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo help" "Repo Manual"
+.SH NAME
+repo \- repo help - manual page for repo help
+.SH SYNOPSIS
+.B repo
+\fI\,help \/\fR[\fI\,--all|command\/\fR]
+.SH DESCRIPTION
+Summary
+.PP
+Display detailed help on a command
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-a\fR, \fB\-\-all\fR
+show the complete list of commands
+.TP
+\fB\-\-help\-all\fR
+show the \fB\-\-help\fR of all commands
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help help` to view the detailed manual.
+.SH DETAILS
+.PP
+Displays detailed usage information about a command.

man/repo-info.1

@@ -0,0 +1,53 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo info" "Repo Manual"
+.SH NAME
+repo \- repo info - manual page for repo info
+.SH SYNOPSIS
+.B repo
+\fI\,info \/\fR[\fI\,-dl\/\fR] [\fI\,-o \/\fR[\fI\,-c\/\fR]] [\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Get info on the manifest branch, current branch or unmerged branches
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-d\fR, \fB\-\-diff\fR
+show full info and commit diff including remote
+branches
+.TP
+\fB\-o\fR, \fB\-\-overview\fR
+show overview of all local commits
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+consider only checked out branches
+.TP
+\fB\-\-no\-current\-branch\fR
+consider all local branches
+.TP
+\fB\-l\fR, \fB\-\-local\-only\fR
+disable all remote operations
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help info` to view the detailed manual.

man/repo-init.1

@@ -0,0 +1,204 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "September 2024" "repo init" "Repo Manual"
+.SH NAME
+repo \- repo init - manual page for repo init
+.SH SYNOPSIS
+.B repo
+\fI\,init \/\fR[\fI\,options\/\fR] [\fI\,manifest url\/\fR]
+.SH DESCRIPTION
+Summary
+.PP
+Initialize a repo client checkout in the current directory
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Manifest options:
+.TP
+\fB\-u\fR URL, \fB\-\-manifest\-url\fR=\fI\,URL\/\fR
+manifest repository location
+.TP
+\fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
+manifest branch or revision (use HEAD for default)
+.TP
+\fB\-\-manifest\-upstream\-branch\fR=\fI\,BRANCH\/\fR
+when a commit is provided to \fB\-\-manifest\-branch\fR, this
+is the name of the git ref in which the commit can be
+found
+.TP
+\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
+initial manifest file
+.TP
+\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
+restrict manifest projects to ones with specified
+group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
+.TP
+\fB\-p\fR PLATFORM, \fB\-\-platform\fR=\fI\,PLATFORM\/\fR
+restrict manifest projects to ones with a specified
+platform group [auto|all|none|linux|darwin|...]
+.TP
+\fB\-\-submodules\fR
+sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
+.SS Manifest (only) checkout options:
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+fetch only current manifest branch from server
+(default)
+.TP
+\fB\-\-no\-current\-branch\fR
+fetch all manifest branches from server
+.TP
+\fB\-\-tags\fR
+fetch tags in the manifest
+.TP
+\fB\-\-no\-tags\fR
+don't fetch tags in the manifest
+.SS Checkout modes:
+.TP
+\fB\-\-mirror\fR
+create a replica of the remote repositories rather
+than a client working directory
+.TP
+\fB\-\-archive\fR
+checkout an archive instead of a git repository for
+each project. See git archive.
+.TP
+\fB\-\-worktree\fR
+use git\-worktree to manage projects
+.SS Project checkout optimizations:
+.TP
+\fB\-\-reference\fR=\fI\,DIR\/\fR
+location of mirror directory
+.TP
+\fB\-\-dissociate\fR
+dissociate from reference mirrors after clone
+.TP
+\fB\-\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone with given depth; see git clone
+.TP
+\fB\-\-partial\-clone\fR
+perform partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
+.TP
+\fB\-\-no\-partial\-clone\fR
+disable use of partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
+.TP
+\fB\-\-partial\-clone\-exclude\fR=\fI\,PARTIAL_CLONE_EXCLUDE\/\fR
+exclude the specified projects (a comma\-delimited
+project names) from partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
+.TP
+\fB\-\-clone\-filter\fR=\fI\,CLONE_FILTER\/\fR
+filter for use with \fB\-\-partial\-clone\fR [default:
+blob:none]
+.TP
+\fB\-\-use\-superproject\fR
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
+.TP
+\fB\-\-no\-use\-superproject\fR
+disable use of manifest superprojects
+.TP
+\fB\-\-clone\-bundle\fR
+enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
+not \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-no\-clone\-bundle\fR
+disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
+\fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
+.SS repo Version options:
+.TP
+\fB\-\-repo\-url\fR=\fI\,URL\/\fR
+repo repository location ($REPO_URL)
+.TP
+\fB\-\-repo\-rev\fR=\fI\,REV\/\fR
+repo branch or revision ($REPO_REV)
+.TP
+\fB\-\-no\-repo\-verify\fR
+do not verify repo source code
+.SS Other options:
+.TP
+\fB\-\-config\-name\fR
+Always prompt for name/e\-mail
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help init` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo init' command is run once to install and initialize repo. The latest
+repo source code and manifest collection is downloaded from the server and is
+installed in the .repo/ directory in the current working directory.
+.PP
+When creating a new checkout, the manifest URL is the only required setting. It
+may be specified using the \fB\-\-manifest\-url\fR option, or as the first optional
+argument.
+.PP
+The optional \fB\-b\fR argument can be used to select the manifest branch to checkout
+and use. If no branch is specified, the remote's default branch is used. This is
+equivalent to using \fB\-b\fR HEAD.
+.PP
+The optional \fB\-\-manifest\-upstream\-branch\fR argument can be used when a commit is
+provided to \fB\-\-manifest\-branch\fR (or \fB\-b\fR), to specify the name of the git ref in
+which the commit can be found.
+.PP
+The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
+used. If no manifest is specified, the manifest default.xml will be used.
+.PP
+If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded
+directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting
+up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be
+fully static and will not be re\-downloaded during subsesquent `repo init` and
+`repo sync` calls.
+.PP
+The \fB\-\-reference\fR option can be used to point to a directory that has the content
+of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as
+possible from the local reference directory when fetching from the server. This
+will make the sync go a lot faster by reducing data traffic on the network.
+.PP
+The \fB\-\-dissociate\fR option can be used to borrow the objects from the directory
+specified with the \fB\-\-reference\fR option only to reduce network transfer, and stop
+borrowing from them after a first clone is made by making necessary local copies
+of borrowed objects.
+.PP
+The \fB\-\-no\-clone\-bundle\fR option disables any attempt to use \fI\,$URL/clone.bundle\/\fP to
+bootstrap a new Git repository from a resumeable bundle file on a content
+delivery network. This may be necessary if there are problems with the local
+Python HTTP client or proxy configuration, but the Git binary works.
+.PP
+Switching Manifest Branches
+.PP
+To switch to another manifest branch, `repo init \fB\-b\fR otherbranch` may be used in
+an existing client. However, as this only updates the manifest, a subsequent
+`repo sync` (or `repo sync \fB\-d\fR`) is necessary to update the working directory
+files.

man/repo-list.1

@@ -0,0 +1,74 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo list" "Repo Manual"
+.SH NAME
+repo \- repo list - manual page for repo list
+.SH SYNOPSIS
+.B repo
+\fI\,list \/\fR[\fI\,-f\/\fR] [\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+List projects and their associated directories
+.PP
+repo list [\-f] \fB\-r\fR str1 [str2]...
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-r\fR, \fB\-\-regex\fR
+filter the project list based on regex or wildcard
+matching of strings
+.TP
+\fB\-g\fR GROUPS, \fB\-\-groups\fR=\fI\,GROUPS\/\fR
+filter the project list based on the groups the
+project is in
+.TP
+\fB\-a\fR, \fB\-\-all\fR
+show projects regardless of checkout state
+.TP
+\fB\-n\fR, \fB\-\-name\-only\fR
+display only the name of the repository
+.TP
+\fB\-p\fR, \fB\-\-path\-only\fR
+display only the path of the repository
+.TP
+\fB\-f\fR, \fB\-\-fullpath\fR
+display the full work tree path instead of the
+relative path
+.TP
+\fB\-\-relative\-to\fR=\fI\,PATH\/\fR
+display paths relative to this one (default: top of
+repo client checkout)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help list` to view the detailed manual.
+.SH DETAILS
+.PP
+List all projects; pass '.' to list the project for the cwd.
+.PP
+By default, only projects that currently exist in the checkout are shown. If you
+want to list all projects (using the specified filter settings), use the \fB\-\-all\fR
+option. If you want to show all projects regardless of the manifest groups, then
+also pass \fB\-\-groups\fR all.
+.PP
+This is similar to running: repo forall \fB\-c\fR 'echo "$REPO_PATH : $REPO_PROJECT"'.

man/repo-manifest.1

@@ -0,0 +1,671 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "April 2025" "repo manifest" "Repo Manual"
+.SH NAME
+repo \- repo manifest - manual page for repo manifest
+.SH SYNOPSIS
+.B repo
+\fI\,manifest \/\fR[\fI\,-o {-|NAME.xml}\/\fR] [\fI\,-m MANIFEST.xml\/\fR] [\fI\,-r\/\fR]
+.SH DESCRIPTION
+Summary
+.PP
+Manifest inspection utility
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-r\fR, \fB\-\-revision\-as\-HEAD\fR
+save revisions as current HEAD
+.TP
+\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
+temporary manifest to use for this sync
+.TP
+\fB\-\-suppress\-upstream\-revision\fR
+if in \fB\-r\fR mode, do not write the upstream field (only
+of use if the branch names for a sha1 manifest are
+sensitive)
+.TP
+\fB\-\-suppress\-dest\-branch\fR
+if in \fB\-r\fR mode, do not write the dest\-branch field
+(only of use if the branch names for a sha1 manifest
+are sensitive)
+.TP
+\fB\-\-format\fR=\fI\,FORMAT\/\fR
+output format: xml, json (default: xml)
+.TP
+\fB\-\-pretty\fR
+format output for humans to read
+.TP
+\fB\-\-no\-local\-manifests\fR
+ignore local manifests
+.TP
+\fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
+file to save the manifest to. (Filename prefix for
+multi\-tree.)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help manifest` to view the detailed manual.
+.SH DETAILS
+.PP
+With the \fB\-o\fR option, exports the current manifest for inspection. The manifest
+and (if present) local_manifests/ are combined together to produce a single
+manifest file. This file can be stored in a Git repository for use during future
+\&'repo init' invocations.
+.PP
+The \fB\-r\fR option can be used to generate a manifest file with project revisions set
+to the current commit hash. These are known as "revision locked manifests", as
+they don't follow a particular branch. In this case, the 'upstream' attribute is
+set to the ref we were on when the manifest was generated. The 'dest\-branch'
+attribute is set to indicate the remote ref to push changes to via 'repo
+upload'.
+.PP
+Multiple output formats are supported via \fB\-\-format\fR. The default output is XML,
+and formats are generally "condensed". Use \fB\-\-pretty\fR for more human\-readable
+variations.
+.PP
+repo Manifest Format
+.PP
+A repo manifest describes the structure of a repo client; that is the
+directories that are visible and where they should be obtained from with git.
+.PP
+The basic structure of a manifest is a bare Git repository holding a single
+`default.xml` XML file in the top level directory.
+.PP
+Manifests are inherently version controlled, since they are kept within a Git
+repository. Updates to manifests are automatically obtained by clients during
+`repo sync`.
+.PP
+[TOC]
+.PP
+XML File Format
+.PP
+A manifest XML file (e.g. `default.xml`) roughly conforms to the following DTD:
+.PP
+```xml <!DOCTYPE manifest [
+.TP
+<!ELEMENT manifest (notice?,
+remote*,
+default?,
+manifest\-server?,
+submanifest*?,
+remove\-project*,
+project*,
+extend\-project*,
+repo\-hooks?,
+superproject?,
+contactinfo?,
+include*)>
+.IP
+<!ELEMENT notice (#PCDATA)>
+.IP
+<!ELEMENT remote (annotation*)>
+<!ATTLIST remote name         ID    #REQUIRED>
+<!ATTLIST remote alias        CDATA #IMPLIED>
+<!ATTLIST remote fetch        CDATA #REQUIRED>
+<!ATTLIST remote pushurl      CDATA #IMPLIED>
+<!ATTLIST remote review       CDATA #IMPLIED>
+<!ATTLIST remote revision     CDATA #IMPLIED>
+.IP
+<!ELEMENT default EMPTY>
+<!ATTLIST default remote      IDREF #IMPLIED>
+<!ATTLIST default revision    CDATA #IMPLIED>
+<!ATTLIST default dest\-branch CDATA #IMPLIED>
+<!ATTLIST default upstream    CDATA #IMPLIED>
+<!ATTLIST default sync\-j      CDATA #IMPLIED>
+<!ATTLIST default sync\-c      CDATA #IMPLIED>
+<!ATTLIST default sync\-s      CDATA #IMPLIED>
+<!ATTLIST default sync\-tags   CDATA #IMPLIED>
+.IP
+<!ELEMENT manifest\-server EMPTY>
+<!ATTLIST manifest\-server url CDATA #REQUIRED>
+.IP
+<!ELEMENT submanifest EMPTY>
+<!ATTLIST submanifest name           ID #REQUIRED>
+<!ATTLIST submanifest remote         IDREF #IMPLIED>
+<!ATTLIST submanifest project        CDATA #IMPLIED>
+<!ATTLIST submanifest manifest\-name  CDATA #IMPLIED>
+<!ATTLIST submanifest revision       CDATA #IMPLIED>
+<!ATTLIST submanifest path           CDATA #IMPLIED>
+<!ATTLIST submanifest groups         CDATA #IMPLIED>
+<!ATTLIST submanifest default\-groups CDATA #IMPLIED>
+.TP
+<!ELEMENT project (annotation*,
+project*,
+copyfile*,
+linkfile*)>
+.TP
+<!ATTLIST project name
+CDATA #REQUIRED>
+.TP
+<!ATTLIST project path
+CDATA #IMPLIED>
+.TP
+<!ATTLIST project remote
+IDREF #IMPLIED>
+.TP
+<!ATTLIST project revision
+CDATA #IMPLIED>
+.IP
+<!ATTLIST project dest\-branch CDATA #IMPLIED>
+<!ATTLIST project groups      CDATA #IMPLIED>
+<!ATTLIST project sync\-c      CDATA #IMPLIED>
+<!ATTLIST project sync\-s      CDATA #IMPLIED>
+<!ATTLIST project sync\-tags   CDATA #IMPLIED>
+<!ATTLIST project upstream CDATA #IMPLIED>
+<!ATTLIST project clone\-depth CDATA #IMPLIED>
+<!ATTLIST project force\-path CDATA #IMPLIED>
+.IP
+<!ELEMENT annotation EMPTY>
+<!ATTLIST annotation name  CDATA #REQUIRED>
+<!ATTLIST annotation value CDATA #REQUIRED>
+<!ATTLIST annotation keep  CDATA "true">
+.IP
+<!ELEMENT copyfile EMPTY>
+<!ATTLIST copyfile src  CDATA #REQUIRED>
+<!ATTLIST copyfile dest CDATA #REQUIRED>
+.IP
+<!ELEMENT linkfile EMPTY>
+<!ATTLIST linkfile src CDATA #REQUIRED>
+<!ATTLIST linkfile dest CDATA #REQUIRED>
+.IP
+<!ELEMENT extend\-project EMPTY>
+<!ATTLIST extend\-project name CDATA #REQUIRED>
+<!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
+<!ATTLIST extend\-project groups CDATA #IMPLIED>
+<!ATTLIST extend\-project revision CDATA #IMPLIED>
+<!ATTLIST extend\-project remote CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
+<!ATTLIST extend\-project upstream CDATA #IMPLIED>
+<!ATTLIST extend\-project base\-rev CDATA #IMPLIED>
+.IP
+<!ELEMENT remove\-project EMPTY>
+<!ATTLIST remove\-project name     CDATA #IMPLIED>
+<!ATTLIST remove\-project path     CDATA #IMPLIED>
+<!ATTLIST remove\-project optional CDATA #IMPLIED>
+<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
+.IP
+<!ELEMENT repo\-hooks EMPTY>
+<!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
+<!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
+.IP
+<!ELEMENT superproject EMPTY>
+<!ATTLIST superproject name     CDATA #REQUIRED>
+<!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
+.IP
+<!ELEMENT contactinfo EMPTY>
+<!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
+.IP
+<!ELEMENT include EMPTY>
+<!ATTLIST include name     CDATA #REQUIRED>
+<!ATTLIST include groups   CDATA #IMPLIED>
+<!ATTLIST include revision CDATA #IMPLIED>
+.PP
+]>
+```
+.PP
+For compatibility purposes across repo releases, all unknown elements are
+silently ignored. However, repo reserves all possible names for itself for
+future use. If you want to use custom elements, the `x\-*` namespace is reserved
+for that purpose, and repo guarantees to never allocate any corresponding names.
+.PP
+A description of the elements and their attributes follows.
+.PP
+Element manifest
+.PP
+The root element of the file.
+.PP
+Element notice
+.PP
+Arbitrary text that is displayed to users whenever `repo sync` finishes. The
+content is simply passed through as it exists in the manifest.
+.PP
+Element remote
+.PP
+One or more remote elements may be specified. Each remote element specifies a
+Git URL shared by one or more projects and (optionally) the Gerrit review server
+those projects upload changes through.
+.PP
+Attribute `name`: A short name unique to this manifest file. The name specified
+here is used as the remote name in each project's .git/config, and is therefore
+automatically available to commands like `git fetch`, `git remote`, `git pull`
+and `git push`.
+.PP
+Attribute `alias`: The alias, if specified, is used to override `name` to be set
+as the remote name in each project's .git/config. Its value can be duplicated
+while attribute `name` has to be unique in the manifest file. This helps each
+project to be able to have same remote name which actually points to different
+remote url.
+.PP
+Attribute `fetch`: The Git URL prefix for all projects which use this remote.
+Each project's name is appended to this prefix to form the actual URL used to
+clone the project.
+.PP
+Attribute `pushurl`: The Git "push" URL prefix for all projects which use this
+remote. Each project's name is appended to this prefix to form the actual URL
+used to "git push" the project. This attribute is optional; if not specified
+then "git push" will use the same URL as the `fetch` attribute.
+.PP
+Attribute `review`: Hostname of the Gerrit server where reviews are uploaded to
+by `repo upload`. This attribute is optional; if not specified then `repo
+upload` will not function.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`).
+Remotes with their own revision will override the default revision.
+.PP
+Element default
+.PP
+At most one default element may be specified. Its remote and revision attributes
+are used when a project element does not specify its own remote or revision
+attribute.
+.PP
+Attribute `remote`: Name of a previously defined remote element. Project
+elements lacking a remote attribute of their own will use this remote.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`).
+Project elements lacking their own revision attribute will use this revision.
+.PP
+Attribute `dest\-branch`: Name of a Git branch (e.g. `main`). Project elements
+not setting their own `dest\-branch` will inherit this value. If this value is
+not set, projects will use `revision` by default instead.
+.PP
+Attribute `upstream`: Name of the Git ref in which a sha1 can be found. Used
+when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to sync the
+entire ref space. Project elements not setting their own `upstream` will inherit
+this value.
+.PP
+Attribute `sync\-j`: Number of parallel jobs to use when synching.
+.PP
+Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
+the `revision` attribute) rather than the whole ref space. Project elements
+lacking a sync\-c element of their own will use this value.
+.PP
+Attribute `sync\-s`: Set to true to also sync sub\-projects.
+.PP
+Attribute `sync\-tags`: Set to false to only sync the given Git branch (specified
+in the `revision` attribute) rather than the other ref tags.
+.PP
+Element manifest\-server
+.PP
+At most one manifest\-server may be specified. The url attribute is used to
+specify the URL of a manifest server, which is an XML RPC service.
+.PP
+See the [smart sync documentation](./smart\-sync.md) for more details.
+.PP
+Element submanifest
+.PP
+One or more submanifest elements may be specified. Each element describes a
+single manifest to be checked out as a child.
+.PP
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below. The same name can be
+used for submanifests with different parent (sub)manifests.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `project`: The manifest project name. The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with. The URL gets formed as:
+.IP
+${remote_fetch}/${project_name}.git
+.PP
+where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
+project's name attribute. The suffix ".git" is always appended as repo assumes
+the upstream is a forest of bare Git repositories. If the project has a parent
+element, its name will be prefixed by the parent's.
+.PP
+The project name must match the name Gerrit knows, if Gerrit is being used for
+code reviews.
+.PP
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components. It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote with
+the new settings needed.
+.PP
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+.PP
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+.PP
+Attribute `manifest\-name`: The manifest filename in the manifest project. If not
+supplied, `default.xml` is used.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash. If not supplied, `name` is
+used.
+.PP
+Attribute `path`: An optional path relative to the top directory of the repo
+client where the submanifest repo client top directory should be placed. If not
+supplied, `revision` is used.
+.PP
+`path` may not be an absolute path or use "." or ".." path components.
+.PP
+Attribute `groups`: List of additional groups to which all projects in the
+included submanifest belong. This appends and recurses, meaning all projects in
+submanifests carry all parent submanifest groups. Same syntax as the
+corresponding element of `project`.
+.PP
+Attribute `default\-groups`: The list of manifest groups to sync if no
+`\-\-groups=` parameter was specified at init. When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+.PP
+Element project
+.PP
+One or more project elements may be specified. Each element describes a single
+Git repository to be cloned into the repo client workspace. You may specify
+Git\-submodules by creating a nested project. Git\-submodules will be
+automatically recognized and inherit their parent's attributes, but those may be
+overridden by an explicitly specified project element.
+.PP
+Attribute `name`: A unique name for this project. The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with. The URL gets formed as:
+.IP
+${remote_fetch}/${project_name}.git
+.PP
+where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
+project's name attribute. The suffix ".git" is always appended as repo assumes
+the upstream is a forest of bare Git repositories. If the project has a parent
+element, its name will be prefixed by the parent's.
+.PP
+The project name must match the name Gerrit knows, if Gerrit is being used for
+code reviews.
+.PP
+"name" must not be empty, and may not be an absolute path or use "." or ".."
+path components. It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote with
+the new settings needed. These restrictions are not enforced for [Local
+Manifests].
+.PP
+Attribute `path`: An optional path relative to the top directory of the repo
+client where the Git working directory for this project should be placed. If not
+supplied the project "name" is used. If the project has a parent element, its
+path will be prefixed by the parent's.
+.PP
+"path" may not be an absolute path or use "." or ".." path components. These
+restrictions are not enforced for [Local Manifests].
+.PP
+If you want to place files into the root of the checkout (e.g. a README or
+Makefile or another build script), use the [copyfile] or [linkfile] elements
+instead.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this project. Names can be relative to refs/heads (e.g. just "main") or absolute
+(e.g. "refs/heads/main"). Tags and/or explicit SHA\-1s should work in theory, but
+have not been extensively tested. If not supplied the revision given by the
+remote element is used if applicable, else the default element is used.
+.PP
+Attribute `dest\-branch`: Name of a Git branch (e.g. `main`). When using `repo
+upload`, changes will be submitted for code review on this branch. If
+unspecified both here and in the default element, `revision` is used instead.
+.PP
+Attribute `groups`: List of groups to which this project belongs, whitespace or
+comma separated. All projects belong to the group "all", and each project
+automatically belongs to a group of its name:`name` and path:`path`. E.g. for
+`<project name="monkeys" path="barrel\-of"/>`, that project definition is
+implicitly in the following manifest groups: default, name:monkeys, and
+path:barrel\-of. If you place a project in the group "notdefault", it will not be
+automatically downloaded by repo. If the project has a parent element, the
+`name` and `path` here are the prefixed ones.
+.PP
+Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
+the `revision` attribute) rather than the whole ref space.
+.PP
+Attribute `sync\-s`: Set to true to also sync sub\-projects.
+.PP
+Attribute `upstream`: Name of the Git ref in which a sha1 can be found. Used
+when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to sync the
+entire ref space.
+.PP
+Attribute `clone\-depth`: Set the depth to use when fetching this project. If
+specified, this value will override any value given to repo init with the
+\fB\-\-depth\fR option on the command line.
+.PP
+Attribute `force\-path`: Set to true to force this project to create the local
+mirror repository according to its `path` attribute (if supplied) rather than
+the `name` attribute. This attribute only applies to the local mirrors syncing,
+it will be ignored when syncing the projects in a client working directory.
+.PP
+Element extend\-project
+.PP
+Modify the attributes of the named project.
+.PP
+This element is mostly useful in a local manifest file, to modify the attributes
+of an existing project without completely replacing the existing project
+definition. This makes the local manifest more robust against changes to the
+original manifest.
+.PP
+Attribute `path`: If specified, limit the change to projects checked out at the
+specified path, rather than all projects with the given name.
+.PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
+Attribute `groups`: List of additional groups to which this project belongs.
+Same syntax as the corresponding element of `project`.
+.PP
+Attribute `revision`: If specified, overrides the revision of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `remote`: If specified, overrides the remote of the original project.
+Same syntax as the corresponding element of `project`.
+.PP
+Attribute `dest\-branch`: If specified, overrides the dest\-branch of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `upstream`: If specified, overrides the upstream of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+extended. Manifest parse will fail and give a list of mismatch extends if the
+revisions being extended have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
+Element annotation
+.PP
+Zero or more annotation elements may be specified as children of a project or
+remote element. Each element describes a name\-value pair. For projects, this
+name\-value pair will be exported into each project's environment during a
+\&'forall' command, prefixed with `REPO__`. In addition, there is an optional
+attribute "keep" which accepts the case insensitive values "true" (default) or
+"false". This attribute determines whether or not the annotation will be kept
+when exported with the manifest subcommand.
+.PP
+Element copyfile
+.PP
+Zero or more copyfile elements may be specified as children of a project
+element. Each element describes a src\-dest pair of files; the "src" file will be
+copied to the "dest" place during `repo sync` command.
+.PP
+"src" is project relative, "dest" is relative to the top of the tree. Copying
+from paths outside of the project or to paths outside of the repo client is not
+allowed.
+.PP
+"src" and "dest" must be files. Directories or symlinks are not allowed.
+Intermediate paths must not be symlinks either.
+.PP
+Parent directories of "dest" will be automatically created if missing.
+.PP
+Element linkfile
+.PP
+It's just like copyfile and runs at the same time as copyfile but instead of
+copying it creates a symlink.
+.PP
+The symlink is created at "dest" (relative to the top of the tree) and points to
+the path specified by "src" which is a path in the project.
+.PP
+Parent directories of "dest" will be automatically created if missing.
+.PP
+The symlink target may be a file or directory, but it may not point outside of
+the repo client.
+.PP
+Element remove\-project
+.PP
+Deletes a project from the internal manifest table, possibly allowing a
+subsequent project element in the same manifest file to replace the project with
+a different source.
+.PP
+This element is mostly useful in a local manifest file, where the user can
+remove a project, and possibly replace it with their own definition.
+.PP
+The project `name` or project `path` can be used to specify the remove target
+meaning one of them is required. If only name is specified, all projects with
+that name are removed.
+.PP
+If both name and path are specified, only projects with the same name and path
+are removed, meaning projects with the same name but in other locations are
+kept.
+.PP
+If only path is specified, a matching project is removed regardless of its name.
+Logic otherwise behaves like both are specified.
+.PP
+Attribute `optional`: Set to true to ignore remove\-project elements with no
+matching `project` element.
+.PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+removed. Manifest parse will fail and give a list of mismatch removes if the
+revisions being removed have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
+Element repo\-hooks
+.PP
+NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.
+.PP
+Only one repo\-hooks element may be specified at a time. Attempting to redefine
+it will fail to parse.
+.PP
+Attribute `in\-project`: The project where the hooks are defined. The value must
+match the `name` attribute (**not** the `path` attribute) of a previously
+defined `project` element.
+.PP
+Attribute `enabled\-list`: List of hooks to use, whitespace or comma separated.
+.PP
+Element superproject
+.PP
+*** *Note*: This is currently a WIP. ***
+.PP
+NB: See the [git superprojects documentation](
+https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects) for background
+information.
+.PP
+This element is used to specify the URL of the superproject. It has "name" and
+"remote" as atrributes. Only "name" is required while the others have reasonable
+defaults. At most one superproject may be specified. Attempting to redefine it
+will fail to parse.
+.PP
+Attribute `name`: A unique name for the superproject. This attribute has the
+same meaning as project's name attribute. See the [element
+project](#element\-project) for more information.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
+Element contactinfo
+.PP
+*** *Note*: This is currently a WIP. ***
+.PP
+This element is used to let manifest authors self\-register contact info. It has
+"bugurl" as a required atrribute. This element can be repeated, and any later
+entries will clobber earlier ones. This would allow manifest authors who extend
+manifests to specify their own contact info.
+.PP
+Attribute `bugurl`: The URL to file a bug against the manifest owner.
+.PP
+Element include
+.PP
+This element provides the capability of including another manifest file into the
+originating manifest. Normal rules apply for the target manifest to include \- it
+must be a usable manifest on its own.
+.PP
+Attribute `name`: the manifest to include, specified relative to the manifest
+repository's root.
+.PP
+"name" may not be an absolute path or use "." or ".." path components. These
+restrictions are not enforced for [Local Manifests].
+.PP
+Attribute `groups`: List of additional groups to which all projects in the
+included manifest belong. This appends and recurses, meaning all projects in
+included manifests carry all parent include groups. Same syntax as the
+corresponding element of `project`.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`)
+default to which all projects in the included manifest belong.
+.PP
+Local Manifests
+.PP
+Additional remotes and projects may be added through local manifest files stored
+in `$TOP_DIR/.repo/local_manifests/*.xml`.
+.PP
+For example:
+.IP
+\f(CW$ ls .repo/local_manifests\fR
+.IP
+local_manifest.xml
+another_local_manifest.xml
+.IP
+\f(CW$ cat .repo/local_manifests/local_manifest.xml\fR
+.IP
+<?xml version="1.0" encoding="UTF\-8"?>
+<manifest>
+.IP
+<project path="manifest"
+.IP
+name="tools/manifest" />
+.IP
+<project path="platform\-manifest"
+.IP
+name="platform/manifest" />
+.IP
+</manifest>
+.PP
+Users may add projects to the local manifest(s) prior to a `repo sync`
+invocation, instructing repo to automatically download and manage these extra
+projects.
+.PP
+Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will be loaded
+in alphabetical order.
+.PP
+Projects from local manifest files are added into local::<local manifest
+filename> group.
+.PP
+The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
+.SS [copyfile]: #Element\-copyfile [linkfile]: #Element\-linkfile [Local Manifests]:
+.PP
+#local\-manifests

man/repo-overview.1

@@ -0,0 +1,52 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo overview" "Repo Manual"
+.SH NAME
+repo \- repo overview - manual page for repo overview
+.SH SYNOPSIS
+.B repo
+\fI\,overview \/\fR[\fI\,--current-branch\/\fR] [\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Display overview of unmerged project branches
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+consider only checked out branches
+.TP
+\fB\-\-no\-current\-branch\fR
+consider all local branches
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help overview` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo overview' command is used to display an overview of the projects
+branches, and list any local commits that have not yet been merged into the
+project.
+.PP
+The \fB\-c\fR/\-\-current\-branch option can be used to restrict the output to only
+branches currently checked out in each project. By default, all branches are
+displayed.

man/repo-prune.1

@@ -0,0 +1,41 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo prune" "Repo Manual"
+.SH NAME
+repo \- repo prune - manual page for repo prune
+.SH SYNOPSIS
+.B repo
+\fI\,prune \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Prune (delete) already merged topics
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help prune` to view the detailed manual.

man/repo-rebase.1

@@ -0,0 +1,68 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo rebase" "Repo Manual"
+.SH NAME
+repo \- repo rebase - manual page for repo rebase
+.SH SYNOPSIS
+.B repo
+\fI\,rebase {\/\fR[\fI\,<project>\/\fR...] \fI\,| -i <project>\/\fR...\fI\,}\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Rebase local branches on upstream branch
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-\-fail\-fast\fR
+stop rebasing after first error is hit
+.TP
+\fB\-f\fR, \fB\-\-force\-rebase\fR
+pass \fB\-\-force\-rebase\fR to git rebase
+.TP
+\fB\-\-no\-ff\fR
+pass \fB\-\-no\-ff\fR to git rebase
+.TP
+\fB\-\-autosquash\fR
+pass \fB\-\-autosquash\fR to git rebase
+.TP
+\fB\-\-whitespace\fR=\fI\,WS\/\fR
+pass \fB\-\-whitespace\fR to git rebase
+.TP
+\fB\-\-auto\-stash\fR
+stash local modifications before starting
+.TP
+\fB\-m\fR, \fB\-\-onto\-manifest\fR
+rebase onto the manifest version instead of upstream
+HEAD (this helps to make sure the local tree stays
+consistent if you previously synced to a manifest)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.TP
+\fB\-i\fR, \fB\-\-interactive\fR
+interactive rebase (single project only)
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help rebase` to view the detailed manual.
+.SH DETAILS
+.PP
+\&'repo rebase' uses git rebase to move local changes in the current topic branch
+to the HEAD of the upstream history, useful when you have made commits in a
+topic branch but need to incorporate new upstream changes "underneath" them.

man/repo-selfupdate.1

@@ -0,0 +1,48 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo selfupdate" "Repo Manual"
+.SH NAME
+repo \- repo selfupdate - manual page for repo selfupdate
+.SH SYNOPSIS
+.B repo
+\fI\,selfupdate\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Update repo to the latest version
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.SS repo Version options:
+.TP
+\fB\-\-no\-repo\-verify\fR
+do not verify repo source code
+.PP
+Run `repo help selfupdate` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo selfupdate' command upgrades repo to the latest version, if a newer
+version is available.
+.PP
+Normally this is done automatically by 'repo sync' and does not need to be
+performed by an end\-user.

man/repo-smartsync.1

@@ -0,0 +1,152 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "September 2024" "repo smartsync" "Repo Manual"
+.SH NAME
+repo \- repo smartsync - manual page for repo smartsync
+.SH SYNOPSIS
+.B repo
+\fI\,smartsync \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Update working tree to the latest known good revision
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
+number of network jobs to run in parallel (defaults to
+\fB\-\-jobs\fR or 1)
+.TP
+\fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
+number of local checkout jobs to run in parallel
+(defaults to \fB\-\-jobs\fR or 8)
+.TP
+\fB\-f\fR, \fB\-\-force\-broken\fR
+obsolete option (to be deleted in the future)
+.TP
+\fB\-\-fail\-fast\fR
+stop syncing after first error is hit
+.TP
+\fB\-\-force\-sync\fR
+overwrite an existing git directory if it needs to
+point to a different object directory. WARNING: this
+may cause loss of data
+.TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
+\fB\-\-force\-remove\-dirty\fR
+force remove projects with uncommitted modifications
+if projects no longer exist in the manifest. WARNING:
+this may cause loss of data
+.TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
+\fB\-l\fR, \fB\-\-local\-only\fR
+only update working tree, don't fetch
+.TP
+\fB\-\-no\-manifest\-update\fR, \fB\-\-nmu\fR
+use the existing manifest checkout as\-is. (do not
+update to the latest revision)
+.TP
+\fB\-n\fR, \fB\-\-network\-only\fR
+fetch only, don't update working tree
+.TP
+\fB\-d\fR, \fB\-\-detach\fR
+detach projects back to manifest revision
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+fetch only current branch from server
+.TP
+\fB\-\-no\-current\-branch\fR
+fetch all branches from server
+.TP
+\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
+temporary manifest to use for this sync
+.TP
+\fB\-\-clone\-bundle\fR
+enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
+.TP
+\fB\-\-no\-clone\-bundle\fR
+disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
+.TP
+\fB\-u\fR MANIFEST_SERVER_USERNAME, \fB\-\-manifest\-server\-username\fR=\fI\,MANIFEST_SERVER_USERNAME\/\fR
+username to authenticate with the manifest server
+.TP
+\fB\-p\fR MANIFEST_SERVER_PASSWORD, \fB\-\-manifest\-server\-password\fR=\fI\,MANIFEST_SERVER_PASSWORD\/\fR
+password to authenticate with the manifest server
+.TP
+\fB\-\-fetch\-submodules\fR
+fetch submodules from server
+.TP
+\fB\-\-use\-superproject\fR
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
+.TP
+\fB\-\-no\-use\-superproject\fR
+disable use of manifest superprojects
+.TP
+\fB\-\-tags\fR
+fetch tags
+.TP
+\fB\-\-no\-tags\fR
+don't fetch tags (default)
+.TP
+\fB\-\-optimized\-fetch\fR
+only fetch projects fixed to sha1 if revision does not
+exist locally
+.TP
+\fB\-\-retry\-fetches\fR=\fI\,RETRY_FETCHES\/\fR
+number of times to retry fetches on transient errors
+.TP
+\fB\-\-prune\fR
+delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.SS repo Version options:
+.TP
+\fB\-\-no\-repo\-verify\fR
+do not verify repo source code
+.PP
+Run `repo help smartsync` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo smartsync' command is a shortcut for sync \fB\-s\fR.

man/repo-stage.1

@@ -0,0 +1,43 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo stage" "Repo Manual"
+.SH NAME
+repo \- repo stage - manual page for repo stage
+.SH SYNOPSIS
+.B repo
+\fI\,stage -i \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Stage file(s) for commit
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.TP
+\fB\-i\fR, \fB\-\-interactive\fR
+use interactive staging
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help stage` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo stage' command stages files to prepare the next commit.

man/repo-start.1

@@ -0,0 +1,54 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo start" "Repo Manual"
+.SH NAME
+repo \- repo start - manual page for repo start
+.SH SYNOPSIS
+.B repo
+\fI\,start <newbranchname> \/\fR[\fI\,--all | <project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Start a new branch for development
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-\-all\fR
+begin branch in all projects
+.TP
+\fB\-r\fR REVISION, \fB\-\-rev\fR=\fI\,REVISION\/\fR, \fB\-\-revision\fR=\fI\,REVISION\/\fR
+point branch at this revision instead of upstream
+.TP
+\fB\-\-head\fR, \fB\-\-HEAD\fR
+abbreviation for \fB\-\-rev\fR HEAD
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help start` to view the detailed manual.
+.SH DETAILS
+.PP
+\&'repo start' begins a new branch of development, starting from the revision
+specified in the manifest.

man/repo-status.1

@@ -0,0 +1,111 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo status" "Repo Manual"
+.SH NAME
+repo \- repo status - manual page for repo status
+.SH SYNOPSIS
+.B repo
+\fI\,status \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Show the working tree status
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-o\fR, \fB\-\-orphans\fR
+include objects in working directory outside of repo
+projects
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help status` to view the detailed manual.
+.SH DETAILS
+.PP
+\&'repo status' compares the working tree to the staging area (aka index), and the
+most recent commit on this branch (HEAD), in each project specified. A summary
+is displayed, one line per file where there is a difference between these three
+states.
+.PP
+The \fB\-j\fR/\-\-jobs option can be used to run multiple status queries in parallel.
+.PP
+The \fB\-o\fR/\-\-orphans option can be used to show objects that are in the working
+directory, but not associated with a repo project. This includes unmanaged
+top\-level files and directories, but also includes deeper items. For example, if
+dir/subdir/proj1 and dir/subdir/proj2 are repo projects, dir/subdir/proj3 will
+be shown if it is not known to repo.
+.PP
+Status Display
+.PP
+The status display is organized into three columns of information, for example
+if the file 'subcmds/status.py' is modified in the project 'repo' on branch
+\&'devwork':
+.TP
+project repo/
+branch devwork
+.TP
+\fB\-m\fR
+subcmds/status.py
+.PP
+The first column explains how the staging area (index) differs from the last
+commit (HEAD). Its values are always displayed in upper case and have the
+following meanings:
+.TP
+\-:
+no difference
+.TP
+A:
+added         (not in HEAD,     in index                     )
+.TP
+M:
+modified      (    in HEAD,     in index, different content  )
+.TP
+D:
+deleted       (    in HEAD, not in index                     )
+.TP
+R:
+renamed       (not in HEAD,     in index, path changed       )
+.TP
+C:
+copied        (not in HEAD,     in index, copied from another)
+.TP
+T:
+mode changed  (    in HEAD,     in index, same content       )
+.TP
+U:
+unmerged; conflict resolution required
+.PP
+The second column explains how the working directory differs from the index. Its
+values are always displayed in lower case and have the following meanings:
+.TP
+\-:
+new / unknown (not in index,     in work tree                )
+.TP
+m:
+modified      (    in index,     in work tree, modified      )
+.TP
+d:
+deleted       (    in index, not in work tree                )

man/repo-sync.1

@@ -0,0 +1,251 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "September 2024" "repo sync" "Repo Manual"
+.SH NAME
+repo \- repo sync - manual page for repo sync
+.SH SYNOPSIS
+.B repo
+\fI\,sync \/\fR[\fI\,<project>\/\fR...]
+.SH DESCRIPTION
+Summary
+.PP
+Update working tree to the latest revision
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
+number of network jobs to run in parallel (defaults to
+\fB\-\-jobs\fR or 1)
+.TP
+\fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
+number of local checkout jobs to run in parallel
+(defaults to \fB\-\-jobs\fR or 8)
+.TP
+\fB\-f\fR, \fB\-\-force\-broken\fR
+obsolete option (to be deleted in the future)
+.TP
+\fB\-\-fail\-fast\fR
+stop syncing after first error is hit
+.TP
+\fB\-\-force\-sync\fR
+overwrite an existing git directory if it needs to
+point to a different object directory. WARNING: this
+may cause loss of data
+.TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
+\fB\-\-force\-remove\-dirty\fR
+force remove projects with uncommitted modifications
+if projects no longer exist in the manifest. WARNING:
+this may cause loss of data
+.TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
+\fB\-l\fR, \fB\-\-local\-only\fR
+only update working tree, don't fetch
+.TP
+\fB\-\-no\-manifest\-update\fR, \fB\-\-nmu\fR
+use the existing manifest checkout as\-is. (do not
+update to the latest revision)
+.TP
+\fB\-n\fR, \fB\-\-network\-only\fR
+fetch only, don't update working tree
+.TP
+\fB\-d\fR, \fB\-\-detach\fR
+detach projects back to manifest revision
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+fetch only current branch from server
+.TP
+\fB\-\-no\-current\-branch\fR
+fetch all branches from server
+.TP
+\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
+temporary manifest to use for this sync
+.TP
+\fB\-\-clone\-bundle\fR
+enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
+.TP
+\fB\-\-no\-clone\-bundle\fR
+disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
+.TP
+\fB\-u\fR MANIFEST_SERVER_USERNAME, \fB\-\-manifest\-server\-username\fR=\fI\,MANIFEST_SERVER_USERNAME\/\fR
+username to authenticate with the manifest server
+.TP
+\fB\-p\fR MANIFEST_SERVER_PASSWORD, \fB\-\-manifest\-server\-password\fR=\fI\,MANIFEST_SERVER_PASSWORD\/\fR
+password to authenticate with the manifest server
+.TP
+\fB\-\-fetch\-submodules\fR
+fetch submodules from server
+.TP
+\fB\-\-use\-superproject\fR
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
+.TP
+\fB\-\-no\-use\-superproject\fR
+disable use of manifest superprojects
+.TP
+\fB\-\-tags\fR
+fetch tags
+.TP
+\fB\-\-no\-tags\fR
+don't fetch tags (default)
+.TP
+\fB\-\-optimized\-fetch\fR
+only fetch projects fixed to sha1 if revision does not
+exist locally
+.TP
+\fB\-\-retry\-fetches\fR=\fI\,RETRY_FETCHES\/\fR
+number of times to retry fetches on transient errors
+.TP
+\fB\-\-prune\fR
+delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
+.TP
+\fB\-s\fR, \fB\-\-smart\-sync\fR
+smart sync using manifest from the latest known good
+build
+.TP
+\fB\-t\fR SMART_TAG, \fB\-\-smart\-tag\fR=\fI\,SMART_TAG\/\fR
+smart sync using manifest from a known tag
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.SS repo Version options:
+.TP
+\fB\-\-no\-repo\-verify\fR
+do not verify repo source code
+.PP
+Run `repo help sync` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo sync' command synchronizes local project directories with the remote
+repositories specified in the manifest. If a local project does not yet exist,
+it will clone a new local directory from the remote repository and set up
+tracking branches as specified in the manifest. If the local project already
+exists, 'repo sync' will update the remote branches and rebase any new local
+changes on top of the new remote changes.
+.PP
+\&'repo sync' will synchronize all projects listed at the command line. Projects
+can be specified either by name, or by a relative or absolute path to the
+project's local directory. If no projects are specified, 'repo sync' will
+synchronize all projects listed in the manifest.
+.PP
+The \fB\-d\fR/\-\-detach option can be used to switch specified projects back to the
+manifest revision. This option is especially helpful if the project is currently
+on a topic branch, but the manifest revision is temporarily needed.
+.PP
+The \fB\-s\fR/\-\-smart\-sync option can be used to sync to a known good build as
+specified by the manifest\-server element in the current manifest. The
+\fB\-t\fR/\-\-smart\-tag option is similar and allows you to specify a custom tag/label.
+.PP
+The \fB\-u\fR/\-\-manifest\-server\-username and \fB\-p\fR/\-\-manifest\-server\-password options can
+be used to specify a username and password to authenticate with the manifest
+server when using the \fB\-s\fR or \fB\-t\fR option.
+.PP
+If \fB\-u\fR and \fB\-p\fR are not specified when using the \fB\-s\fR or \fB\-t\fR option, 'repo sync' will
+attempt to read authentication credentials for the manifest server from the
+user's .netrc file.
+.PP
+\&'repo sync' will not use authentication credentials from \fB\-u\fR/\-p or .netrc if the
+manifest server specified in the manifest file already includes credentials.
+.PP
+By default, all projects will be synced. The \fB\-\-fail\-fast\fR option can be used to
+halt syncing as soon as possible when the first project fails to sync.
+.PP
+The \fB\-\-force\-sync\fR option can be used to overwrite existing git directories if
+they have previously been linked to a different object directory. WARNING: This
+may cause data to be lost since refs may be removed when overwriting.
+.PP
+The \fB\-\-force\-checkout\fR option can be used to force git to switch revs even if the
+index or the working tree differs from HEAD, and if there are untracked files.
+WARNING: This may cause data to be lost since uncommitted changes may be
+removed.
+.PP
+The \fB\-\-force\-remove\-dirty\fR option can be used to remove previously used projects
+with uncommitted changes. WARNING: This may cause data to be lost since
+uncommitted changes may be removed with projects that no longer exist in the
+manifest.
+.PP
+The \fB\-\-no\-clone\-bundle\fR option disables any attempt to use \fI\,$URL/clone.bundle\/\fP to
+bootstrap a new Git repository from a resumeable bundle file on a content
+delivery network. This may be necessary if there are problems with the local
+Python HTTP client or proxy configuration, but the Git binary works.
+.PP
+The \fB\-\-fetch\-submodules\fR option enables fetching Git submodules of a project from
+server.
+.PP
+The \fB\-c\fR/\-\-current\-branch option can be used to only fetch objects that are on the
+branch specified by a project's revision.
+.PP
+The \fB\-\-optimized\-fetch\fR option can be used to only fetch projects that are fixed
+to a sha1 revision if the sha1 revision does not already exist locally.
+.PP
+The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
+remote.
+.PP
+The \fB\-\-auto\-gc\fR option can be used to trigger garbage collection on all projects.
+By default, repo does not run garbage collection.
+.PP
+SSH Connections
+.PP
+If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,
+or user@host:path syntax) repo will automatically enable the SSH ControlMaster
+option when connecting to that host. This feature permits other projects in the
+same 'repo sync' session to reuse the same SSH tunnel, saving connection setup
+overheads.
+.PP
+To disable this behavior on UNIX platforms, set the GIT_SSH environment variable
+to 'ssh'. For example:
+.IP
+export GIT_SSH=ssh
+repo sync
+.PP
+Compatibility
+.PP
+This feature is automatically disabled on Windows, due to the lack of UNIX
+domain socket support.
+.PP
+This feature is not compatible with url.insteadof rewrites in the user's
+~/.gitconfig. 'repo sync' is currently not able to perform the rewrite early
+enough to establish the ControlMaster tunnel.
+.PP
+If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or later is
+required to fix a server side protocol bug.

man/repo-upload.1

@@ -0,0 +1,215 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "June 2024" "repo upload" "Repo Manual"
+.SH NAME
+repo \- repo upload - manual page for repo upload
+.SH SYNOPSIS
+.B repo
+\fI\,upload \/\fR[\fI\,--re --cc\/\fR] [\fI\,<project>\/\fR]...
+.SH DESCRIPTION
+Summary
+.PP
+Upload changes for code review
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
+number of jobs to run in parallel (default: based on
+number of CPU cores)
+.TP
+\fB\-t\fR, \fB\-\-topic\-branch\fR
+set the topic to the local branch name
+.TP
+\fB\-\-topic\fR=\fI\,TOPIC\/\fR
+set topic for the change
+.TP
+\fB\-\-hashtag\fR=\fI\,HASHTAGS\/\fR, \fB\-\-ht\fR=\fI\,HASHTAGS\/\fR
+add hashtags (comma delimited) to the review
+.TP
+\fB\-\-hashtag\-branch\fR, \fB\-\-htb\fR
+add local branch name as a hashtag
+.TP
+\fB\-l\fR LABELS, \fB\-\-label\fR=\fI\,LABELS\/\fR
+add a label when uploading
+.TP
+\fB\-\-pd\fR=\fI\,PATCHSET_DESCRIPTION\/\fR, \fB\-\-patchset\-description\fR=\fI\,PATCHSET_DESCRIPTION\/\fR
+description for patchset
+.TP
+\fB\-\-re\fR=\fI\,REVIEWERS\/\fR, \fB\-\-reviewers\fR=\fI\,REVIEWERS\/\fR
+request reviews from these people
+.TP
+\fB\-\-cc\fR=\fI\,CC\/\fR
+also send email to these email addresses
+.TP
+\fB\-\-br\fR=\fI\,BRANCH\/\fR, \fB\-\-branch\fR=\fI\,BRANCH\/\fR
+(local) branch to upload
+.TP
+\fB\-c\fR, \fB\-\-current\-branch\fR
+upload current git branch
+.TP
+\fB\-\-no\-current\-branch\fR
+upload all git branches
+.TP
+\fB\-\-ne\fR, \fB\-\-no\-emails\fR
+do not send e\-mails on upload
+.TP
+\fB\-p\fR, \fB\-\-private\fR
+upload as a private change (deprecated; use \fB\-\-wip\fR)
+.TP
+\fB\-w\fR, \fB\-\-wip\fR
+upload as a work\-in\-progress change
+.TP
+\fB\-r\fR, \fB\-\-ready\fR
+mark change as ready (clears work\-in\-progress setting)
+.TP
+\fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
+additional push options to transmit
+.TP
+\fB\-D\fR BRANCH, \fB\-\-destination\fR=\fI\,BRANCH\/\fR, \fB\-\-dest\fR=\fI\,BRANCH\/\fR
+submit for review on this target branch
+.TP
+\fB\-n\fR, \fB\-\-dry\-run\fR
+do everything except actually upload the CL
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
+.TP
+\fB\-\-ignore\-untracked\-files\fR
+ignore untracked files in the working copy
+.TP
+\fB\-\-no\-ignore\-untracked\-files\fR
+always ask about untracked files in the working copy
+.TP
+\fB\-\-no\-cert\-checks\fR
+disable verifying ssl certs (unsafe)
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.SS pre\-upload hooks:
+.TP
+\fB\-\-no\-verify\fR
+Do not run the pre\-upload hook.
+.TP
+\fB\-\-verify\fR
+Run the pre\-upload hook without prompting.
+.TP
+\fB\-\-ignore\-hooks\fR
+Do not abort if pre\-upload hooks fail.
+.PP
+Run `repo help upload` to view the detailed manual.
+.SH DETAILS
+.PP
+The 'repo upload' command is used to send changes to the Gerrit Code Review
+system. It searches for topic branches in local projects that have not yet been
+published for review. If multiple topic branches are found, 'repo upload' opens
+an editor to allow the user to select which branches to upload.
+.PP
+\&'repo upload' searches for uploadable changes in all projects listed at the
+command line. Projects can be specified either by name, or by a relative or
+absolute path to the project's local directory. If no projects are specified,
+\&'repo upload' will search for uploadable changes in all projects listed in the
+manifest.
+.PP
+If the \fB\-\-reviewers\fR or \fB\-\-cc\fR options are passed, those emails are added to the
+respective list of users, and emails are sent to any new users. Users passed as
+\fB\-\-reviewers\fR must already be registered with the code review system, or the
+upload will fail.
+.PP
+While most normal Gerrit options have dedicated command line options, direct
+access to the Gerit options is available via \fB\-\-push\-options\fR. This is useful when
+Gerrit has newer functionality that repo upload doesn't yet support, or doesn't
+have plans to support. See the Push Options documentation for more details:
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#push_options
+.PP
+Configuration
+.PP
+review.URL.autoupload:
+.PP
+To disable the "Upload ... (y/N)?" prompt, you can set a per\-project or global
+Git configuration option. If review.URL.autoupload is set to "true" then repo
+will assume you always answer "y" at the prompt, and will not prompt you
+further. If it is set to "false" then repo will assume you always answer "n",
+and will abort.
+.PP
+review.URL.autoreviewer:
+.PP
+To automatically append a user or mailing list to reviews, you can set a
+per\-project or global Git option to do so.
+.PP
+review.URL.autocopy:
+.PP
+To automatically copy a user or mailing list to all uploaded reviews, you can
+set a per\-project or global Git option to do so. Specifically,
+review.URL.autocopy can be set to a comma separated list of reviewers who you
+always want copied on all uploads with a non\-empty \fB\-\-re\fR argument.
+.PP
+review.URL.username:
+.PP
+Override the username used to connect to Gerrit Code Review. By default the
+local part of the email address is used.
+.PP
+The URL must match the review URL listed in the manifest XML file, or in the
+\&.git/config within the project. For example:
+.IP
+[remote "origin"]
+.IP
+url = git://git.example.com/project.git
+review = http://review.example.com/
+.IP
+[review "http://review.example.com/"]
+.IP
+autoupload = true
+autocopy = johndoe@company.com,my\-team\-alias@company.com
+.PP
+review.URL.uploadtopic:
+.PP
+To add a topic branch whenever uploading a commit, you can set a per\-project or
+global Git option to do so. If review.URL.uploadtopic is set to "true" then repo
+will assume you always want the equivalent of the \fB\-t\fR option to the repo command.
+If unset or set to "false" then repo will make use of only the command line
+option.
+.PP
+review.URL.uploadhashtags:
+.PP
+To add hashtags whenever uploading a commit, you can set a per\-project or global
+Git option to do so. The value of review.URL.uploadhashtags will be used as
+comma delimited hashtags like the \fB\-\-hashtag\fR option.
+.PP
+review.URL.uploadlabels:
+.PP
+To add labels whenever uploading a commit, you can set a per\-project or global
+Git option to do so. The value of review.URL.uploadlabels will be used as comma
+delimited labels like the \fB\-\-label\fR option.
+.PP
+review.URL.uploadnotify:
+.PP
+Control e\-mail notifications when uploading.
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#notify
+.PP
+review.URL.uploadwarningthreshold:
+.PP
+Repo will warn you if you are attempting to upload a large number of commits in
+one or more branches. By default, the threshold is five commits. This option
+allows you to override the warning threshold to a different value.
+.PP
+References
+.PP
+Gerrit Code Review: https://www.gerritcodereview.com/

man/repo-version.1

@@ -0,0 +1,37 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "July 2022" "repo version" "Repo Manual"
+.SH NAME
+repo \- repo version - manual page for repo version
+.SH SYNOPSIS
+.B repo
+\fI\,version\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Display the version of repo
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help version` to view the detailed manual.

man/repo.1

@@ -0,0 +1,137 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "April 2025" "repo" "Repo Manual"
+.SH NAME
+repo \- repository management tool built on top of git
+.SH SYNOPSIS
+.B repo
+[\fI\,-p|--paginate|--no-pager\/\fR] \fI\,COMMAND \/\fR[\fI\,ARGS\/\fR]
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-\-help\-all\fR
+show this help message with all subcommands and exit
+.TP
+\fB\-p\fR, \fB\-\-paginate\fR
+display command output in the pager
+.TP
+\fB\-\-no\-pager\fR
+disable the pager
+.TP
+\fB\-\-color\fR=\fI\,COLOR\/\fR
+control color usage: auto, always, never
+.TP
+\fB\-\-trace\fR
+trace git command execution (REPO_TRACE=1)
+.TP
+\fB\-\-trace\-to\-stderr\fR
+trace outputs go to stderr in addition to
+\&.repo/TRACE_FILE
+.TP
+\fB\-\-trace\-python\fR
+trace python command execution
+.TP
+\fB\-\-time\fR
+time repo command execution
+.TP
+\fB\-\-version\fR
+display this version of repo
+.TP
+\fB\-\-show\-toplevel\fR
+display the path of the top\-level directory of the
+repo client checkout
+.TP
+\fB\-\-event\-log\fR=\fI\,EVENT_LOG\/\fR
+filename of event log to append timeline to
+.TP
+\fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
+directory to write git trace2 event log to
+.TP
+\fB\-\-submanifest\-path\fR=\fI\,REL_PATH\/\fR
+submanifest path
+.SS "The complete list of recognized repo commands is:"
+.TP
+abandon
+Permanently abandon a development branch
+.TP
+branch
+View current topic branches
+.TP
+branches
+View current topic branches
+.TP
+checkout
+Checkout a branch for development
+.TP
+cherry\-pick
+Cherry\-pick a change.
+.TP
+diff
+Show changes between commit and working tree
+.TP
+diffmanifests
+Manifest diff utility
+.TP
+download
+Download and checkout a change
+.TP
+forall
+Run a shell command in each project
+.TP
+gc
+Cleaning up internal repo and Git state.
+.TP
+grep
+Print lines matching a pattern
+.TP
+help
+Display detailed help on a command
+.TP
+info
+Get info on the manifest branch, current branch or unmerged branches
+.TP
+init
+Initialize a repo client checkout in the current directory
+.TP
+list
+List projects and their associated directories
+.TP
+manifest
+Manifest inspection utility
+.TP
+overview
+Display overview of unmerged project branches
+.TP
+prune
+Prune (delete) already merged topics
+.TP
+rebase
+Rebase local branches on upstream branch
+.TP
+selfupdate
+Update repo to the latest version
+.TP
+smartsync
+Update working tree to the latest known good revision
+.TP
+stage
+Stage file(s) for commit
+.TP
+start
+Start a new branch for development
+.TP
+status
+Show the working tree status
+.TP
+sync
+Update working tree to the latest revision
+.TP
+upload
+Upload changes for code review
+.TP
+version
+Display the version of repo
+.PP
+See 'repo help <command>' for more information on a specific command.
+Bug reports: https://issues.gerritcodereview.com/issues/new?component=1370071

manifest.py

@@ -1,420 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-import xml.dom.minidom
-
-from git_config import GitConfig, IsId
-from project import Project, MetaProject, R_HEADS, HEAD
-from remote import Remote
-from error import ManifestParseError
-
-MANIFEST_FILE_NAME = 'manifest.xml'
-LOCAL_MANIFEST_NAME = 'local_manifest.xml'
-
-class _Default(object):
-  """Project defaults within the manifest."""
-
-  revision = None
-  remote = None
-
-
-class Manifest(object):
-  """manages the repo configuration file"""
-
-  def __init__(self, repodir):
-    self.repodir = os.path.abspath(repodir)
-    self.topdir = os.path.dirname(self.repodir)
-    self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
-    self.globalConfig = GitConfig.ForUser()
-
-    self.repoProject = MetaProject(self, 'repo',
-      gitdir   = os.path.join(repodir, 'repo/.git'),
-      worktree = os.path.join(repodir, 'repo'))
-
-    self.manifestProject = MetaProject(self, 'manifests',
-      gitdir   = os.path.join(repodir, 'manifests.git'),
-      worktree = os.path.join(repodir, 'manifests'))
-
-    self._Unload()
-
-  def Link(self, name):
-    """Update the repo metadata to use a different manifest.
-    """
-    path = os.path.join(self.manifestProject.worktree, name)
-    if not os.path.isfile(path):
-      raise ManifestParseError('manifest %s not found' % name)
-
-    old = self.manifestFile
-    try:
-      self.manifestFile = path
-      self._Unload()
-      self._Load()
-    finally:
-      self.manifestFile = old
-
-    try:
-      if os.path.exists(self.manifestFile):
-        os.remove(self.manifestFile)
-      os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError, e:
-      raise ManifestParseError('cannot link manifest %s' % name)
-
-  def _RemoteToXml(self, r, doc, root):
-    e = doc.createElement('remote')
-    root.appendChild(e)
-    e.setAttribute('name', r.name)
-    e.setAttribute('fetch', r.fetchUrl)
-    if r.reviewUrl is not None:
-      e.setAttribute('review', r.reviewUrl)
-    if r.projectName is not None:
-      e.setAttribute('project-name', r.projectName)
-
-  def Save(self, fd, peg_rev=False):
-    """Write the current manifest out to the given file descriptor.
-    """
-    doc = xml.dom.minidom.Document()
-    root = doc.createElement('manifest')
-    doc.appendChild(root)
-
-    d = self.default
-    sort_remotes = list(self.remotes.keys())
-    sort_remotes.sort()
-
-    for r in sort_remotes:
-      self._RemoteToXml(self.remotes[r], doc, root)
-    if self.remotes:
-      root.appendChild(doc.createTextNode(''))
-
-    have_default = False
-    e = doc.createElement('default')
-    if d.remote:
-      have_default = True
-      e.setAttribute('remote', d.remote.name)
-    if d.revision:
-      have_default = True
-      e.setAttribute('revision', d.revision)
-    if have_default:
-      root.appendChild(e)
-      root.appendChild(doc.createTextNode(''))
-
-    sort_projects = list(self.projects.keys())
-    sort_projects.sort()
-
-    for p in sort_projects:
-      p = self.projects[p]
-      e = doc.createElement('project')
-      root.appendChild(e)
-      e.setAttribute('name', p.name)
-      if p.relpath != p.name:
-        e.setAttribute('path', p.relpath)
-      if not d.remote or p.remote.name != d.remote.name:
-        e.setAttribute('remote', p.remote.name)
-      if peg_rev:
-        if self.IsMirror:
-          e.setAttribute('revision',
-                         p.bare_git.rev_parse(p.revision + '^0'))
-        else:
-          e.setAttribute('revision',
-                         p.work_git.rev_parse(HEAD + '^0'))
-      elif not d.revision or p.revision != d.revision:
-        e.setAttribute('revision', p.revision)
-
-      for r in p.extraRemotes:
-        self._RemoteToXml(p.extraRemotes[r], doc, e)
-      for c in p.copyfiles:
-        ce = doc.createElement('copyfile')
-        ce.setAttribute('src', c.src)
-        ce.setAttribute('dest', c.dest)
-        e.appendChild(ce)
-
-    doc.writexml(fd, '', '  ', '\n', 'UTF-8')
-
-  @property
-  def projects(self):
-    self._Load()
-    return self._projects
-
-  @property
-  def remotes(self):
-    self._Load()
-    return self._remotes
-
-  @property
-  def default(self):
-    self._Load()
-    return self._default
-
-  @property
-  def IsMirror(self):
-    return self.manifestProject.config.GetBoolean('repo.mirror')
-
-  def _Unload(self):
-    self._loaded = False
-    self._projects = {}
-    self._remotes = {}
-    self._default = None
-    self.branch = None
-
-  def _Load(self):
-    if not self._loaded:
-      m = self.manifestProject
-      b = m.GetBranch(m.CurrentBranch).merge
-      if b.startswith(R_HEADS):
-        b = b[len(R_HEADS):]
-      self.branch = b
-
-      self._ParseManifest(True)
-
-      local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
-      if os.path.exists(local):
-        try:
-          real = self.manifestFile
-          self.manifestFile = local
-          self._ParseManifest(False)
-        finally:
-          self.manifestFile = real
-
-      if self.IsMirror:
-        self._AddMetaProjectMirror(self.repoProject)
-        self._AddMetaProjectMirror(self.manifestProject)
-
-      self._loaded = True
-
-  def _ParseManifest(self, is_root_file):
-    root = xml.dom.minidom.parse(self.manifestFile)
-    if not root or not root.childNodes:
-      raise ManifestParseError, \
-            "no root node in %s" % \
-            self.manifestFile
-
-    config = root.childNodes[0]
-    if config.nodeName != 'manifest':
-      raise ManifestParseError, \
-            "no <manifest> in %s" % \
-            self.manifestFile
-
-    for node in config.childNodes:
-      if node.nodeName == 'remove-project':
-        name = self._reqatt(node, 'name')
-        try:
-          del self._projects[name]
-        except KeyError:
-          raise ManifestParseError, \
-                'project %s not found' % \
-                (name)
-
-    for node in config.childNodes:
-      if node.nodeName == 'remote':
-        remote = self._ParseRemote(node)
-        if self._remotes.get(remote.name):
-          raise ManifestParseError, \
-                'duplicate remote %s in %s' % \
-                (remote.name, self.manifestFile)
-        self._remotes[remote.name] = remote
-
-    for node in config.childNodes:
-      if node.nodeName == 'default':
-        if self._default is not None:
-          raise ManifestParseError, \
-                'duplicate default in %s' % \
-                (self.manifestFile)
-        self._default = self._ParseDefault(node)
-    if self._default is None:
-      self._default = _Default()
-
-    for node in config.childNodes:
-      if node.nodeName == 'project':
-        project = self._ParseProject(node)
-        if self._projects.get(project.name):
-          raise ManifestParseError, \
-                'duplicate project %s in %s' % \
-                (project.name, self.manifestFile)
-        self._projects[project.name] = project
-
-    for node in config.childNodes:
-      if node.nodeName == 'add-remote':
-        pn = self._reqatt(node, 'to-project')
-        project = self._projects.get(pn)
-        if not project:
-          raise ManifestParseError, \
-                'project %s not defined in %s' % \
-                (pn, self.manifestFile)
-        self._ParseProjectExtraRemote(project, node)
-
-  def _AddMetaProjectMirror(self, m):
-    name = None
-    m_url = m.GetRemote(m.remote.name).url
-    if m_url.endswith('/.git'):
-      raise ManifestParseError, 'refusing to mirror %s' % m_url
-
-    if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
-      if not url.endswith('/'):
-        url += '/'
-      if m_url.startswith(url):
-        remote = self._default.remote
-        name = m_url[len(url):]
-
-    if name is None:
-      s = m_url.rindex('/') + 1
-      remote = Remote('origin', fetch = m_url[:s])
-      name = m_url[s:]
-
-    if name.endswith('.git'):
-      name = name[:-4]
-
-    if name not in self._projects:
-      m.PreSync()
-      gitdir = os.path.join(self.topdir, '%s.git' % name)
-      project = Project(manifest = self,
-                        name = name,
-                        remote = remote,
-                        gitdir = gitdir,
-                        worktree = None,
-                        relpath = None,
-                        revision = m.revision)
-      self._projects[project.name] = project
-
-  def _ParseRemote(self, node):
-    """
-    reads a <remote> element from the manifest file
-    """
-    name = self._reqatt(node, 'name')
-    fetch = self._reqatt(node, 'fetch')
-    review = node.getAttribute('review')
-    if review == '':
-      review = None
-
-    projectName = node.getAttribute('project-name')
-    if projectName == '':
-      projectName = None
-
-    r = Remote(name=name,
-               fetch=fetch,
-               review=review,
-               projectName=projectName)
-
-    for n in node.childNodes:
-      if n.nodeName == 'require':
-        r.requiredCommits.append(self._reqatt(n, 'commit'))
-
-    return r
-
-  def _ParseDefault(self, node):
-    """
-    reads a <default> element from the manifest file
-    """
-    d = _Default()
-    d.remote = self._get_remote(node)
-    d.revision = node.getAttribute('revision')
-    if d.revision == '':
-      d.revision = None
-    return d
-
-  def _ParseProject(self, node):
-    """
-    reads a <project> element from the manifest file
-    """ 
-    name = self._reqatt(node, 'name')
-
-    remote = self._get_remote(node)
-    if remote is None:
-      remote = self._default.remote
-    if remote is None:
-      raise ManifestParseError, \
-            "no remote for project %s within %s" % \
-            (name, self.manifestFile)
-
-    revision = node.getAttribute('revision')
-    if not revision:
-      revision = self._default.revision
-    if not revision:
-      raise ManifestParseError, \
-            "no revision for project %s within %s" % \
-            (name, self.manifestFile)
-
-    path = node.getAttribute('path')
-    if not path:
-      path = name
-    if path.startswith('/'):
-      raise ManifestParseError, \
-            "project %s path cannot be absolute in %s" % \
-            (name, self.manifestFile)
-
-    if self.IsMirror:
-      relpath = None
-      worktree = None
-      gitdir = os.path.join(self.topdir, '%s.git' % name)
-    else:
-      worktree = os.path.join(self.topdir, path)
-      gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
-
-    project = Project(manifest = self,
-                      name = name,
-                      remote = remote,
-                      gitdir = gitdir,
-                      worktree = worktree,
-                      relpath = path,
-                      revision = revision)
-
-    for n in node.childNodes:
-      if n.nodeName == 'remote':
-        self._ParseProjectExtraRemote(project, n)
-      elif n.nodeName == 'copyfile':
-        self._ParseCopyFile(project, n)
-
-    return project
-
-  def _ParseProjectExtraRemote(self, project, n):
-    r = self._ParseRemote(n)
-    if project.extraRemotes.get(r.name) \
-       or project.remote.name == r.name:
-      raise ManifestParseError, \
-            'duplicate remote %s in project %s in %s' % \
-            (r.name, project.name, self.manifestFile)
-    project.extraRemotes[r.name] = r
-
-  def _ParseCopyFile(self, project, node):
-    src = self._reqatt(node, 'src')
-    dest = self._reqatt(node, 'dest')
-    if not self.IsMirror:
-      # src is project relative;
-      # dest is relative to the top of the tree
-      project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
-
-  def _get_remote(self, node):
-    name = node.getAttribute('remote')
-    if not name:
-      return None
-
-    v = self._remotes.get(name)
-    if not v:
-      raise ManifestParseError, \
-            "remote %s not defined in %s" % \
-            (name, self.manifestFile)
-    return v
-
-  def _reqatt(self, node, attname):
-    """
-    reads a required attribute from the node.
-    """
-    v = node.getAttribute(attname)
-    if not v:
-      raise ManifestParseError, \
-            "no %s in <%s> within %s" % \
-            (attname, node.nodeName, self.manifestFile)
-    return v

pager.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,70 +14,117 @@
 
 import os
 import select
+import subprocess
 import sys
 
+import platform_utils
+
+
 active = False
+pager_process = None
+old_stdout = None
+old_stderr = None
+
 
 def RunPager(globalConfig):
-  global active
+    if not os.isatty(0) or not os.isatty(1):
+        return
+    pager = _SelectPager(globalConfig)
+    if pager == "" or pager == "cat":
+        return
 
-  if not os.isatty(0):
-    return
-  pager = _SelectPager(globalConfig)
-  if pager == '' or pager == 'cat':
-    return
+    if platform_utils.isWindows():
+        _PipePager(pager)
+    else:
+        _ForkPager(pager)
 
-  # This process turns into the pager; a child it forks will
-  # do the real processing and output back to the pager. This
-  # is necessary to keep the pager in control of the tty.
-  #
-  try:
-    r, w = os.pipe()
-    pid = os.fork()
-    if not pid:
-      os.dup2(w, 1)
-      os.dup2(w, 2)
-      os.close(r)
-      os.close(w)
-      active = True
-      return
 
-    os.dup2(r, 0)
-    os.close(r)
-    os.close(w)
+def TerminatePager():
+    global pager_process
+    if pager_process:
+        sys.stdout.flush()
+        sys.stderr.flush()
+        pager_process.stdin.close()
+        pager_process.wait()
+        pager_process = None
+        # Restore initial stdout/err in case there is more output in this
+        # process after shutting down the pager process.
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+
+
+def _PipePager(pager):
+    global pager_process, old_stdout, old_stderr
+    assert pager_process is None, "Only one active pager process at a time"
+    # Create pager process, piping stdout/err into its stdin.
+    try:
+        pager_process = subprocess.Popen(
+            [pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr
+        )
+    except FileNotFoundError:
+        sys.exit(f'fatal: cannot start pager "{pager}"')
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = pager_process.stdin
+    sys.stderr = pager_process.stdin
+
+
+def _ForkPager(pager):
+    global active
+    # This process turns into the pager; a child it forks will
+    # do the real processing and output back to the pager. This
+    # is necessary to keep the pager in control of the tty.
+    try:
+        r, w = os.pipe()
+        pid = os.fork()
+        if not pid:
+            os.dup2(w, 1)
+            os.dup2(w, 2)
+            os.close(r)
+            os.close(w)
+            active = True
+            return
+
+        os.dup2(r, 0)
+        os.close(r)
+        os.close(w)
+
+        _BecomePager(pager)
+    except Exception:
+        print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
+        sys.exit(255)
 
-    _BecomePager(pager)
-  except Exception:
-    print >>sys.stderr, "fatal: cannot start pager '%s'" % pager
-    os.exit(255)
 
 def _SelectPager(globalConfig):
-  try:
-    return os.environ['GIT_PAGER']
-  except KeyError:
-    pass
+    try:
+        return os.environ["GIT_PAGER"]
+    except KeyError:
+        pass
 
-  pager = globalConfig.GetString('core.pager')
-  if pager:
-    return pager
+    pager = globalConfig.GetString("core.pager")
+    if pager:
+        return pager
 
-  try:
-    return os.environ['PAGER']
-  except KeyError:
-    pass
+    try:
+        return os.environ["PAGER"]
+    except KeyError:
+        pass
+
+    return "less"
 
-  return 'less'
 
 def _BecomePager(pager):
-  # Delaying execution of the pager until we have output
-  # ready works around a long-standing bug in popularly
-  # available versions of 'less', a better 'more'.
-  #
-  a, b, c = select.select([0], [], [0])
+    # Delaying execution of the pager until we have output
+    # ready works around a long-standing bug in popularly
+    # available versions of 'less', a better 'more'.
+    _a, _b, _c = select.select([0], [], [0])
 
-  os.environ['LESS'] = 'FRSX'
+    # This matches the behavior of git, which sets $LESS to `FRX` if it is not
+    # set. See:
+    # https://git-scm.com/docs/git-config#Documentation/git-config.txt-corepager
+    os.environ.setdefault("LESS", "FRX")
 
-  try:
-    os.execvp(pager, [pager])
-  except OSError, e:
-    os.execv('/bin/sh', ['sh', '-c', pager])
+    try:
+        os.execvp(pager, [pager])
+    except OSError:
+        os.execv("/bin/sh", ["sh", "-c", pager])

platform_utils.py

@@ -0,0 +1,259 @@
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+import platform
+import shutil
+import stat
+
+
+def isWindows():
+    """Returns True when running with the native port of Python for Windows,
+    False when running on any other platform (including the Cygwin port of
+    Python).
+    """
+    # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
+    return platform.system() == "Windows"
+
+
+def symlink(source, link_name):
+    """Creates a symbolic link pointing to source named link_name.
+
+    Note: On Windows, source must exist on disk, as the implementation needs
+    to know whether to create a "File" or a "Directory" symbolic link.
+    """
+    if isWindows():
+        import platform_utils_win32
+
+        source = _validate_winpath(source)
+        link_name = _validate_winpath(link_name)
+        target = os.path.join(os.path.dirname(link_name), source)
+        if isdir(target):
+            platform_utils_win32.create_dirsymlink(
+                _makelongpath(source), link_name
+            )
+        else:
+            platform_utils_win32.create_filesymlink(
+                _makelongpath(source), link_name
+            )
+    else:
+        return os.symlink(source, link_name)
+
+
+def _validate_winpath(path):
+    path = os.path.normpath(path)
+    if _winpath_is_valid(path):
+        return path
+    raise ValueError(
+        f'Path "{path}" must be a relative path or an absolute '
+        "path starting with a drive letter"
+    )
+
+
+def _winpath_is_valid(path):
+    """Windows only: returns True if path is relative (e.g. ".\\foo") or is
+    absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
+    is ambiguous (e.g. "x:foo" or "\\foo").
+    """
+    assert isWindows()
+    path = os.path.normpath(path)
+    drive, tail = os.path.splitdrive(path)
+    if tail:
+        if not drive:
+            return tail[0] != os.sep  # "\\foo" is invalid
+        else:
+            return tail[0] == os.sep  # "x:foo" is invalid
+    else:
+        return not drive  # "x:" is invalid
+
+
+def _makelongpath(path):
+    """Return the input path normalized to support the Windows long path syntax
+    ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+    MAX_PATH limit.
+    """
+    if isWindows():
+        # Note: MAX_PATH is 260, but, for directories, the maximum value is
+        # actually 246.
+        if len(path) < 246:
+            return path
+        if path.startswith("\\\\?\\"):
+            return path
+        if not os.path.isabs(path):
+            return path
+        # Append prefix and ensure unicode so that the special longpath syntax
+        # is supported by underlying Win32 API calls
+        return "\\\\?\\" + os.path.normpath(path)
+    else:
+        return path
+
+
+def rmtree(path, ignore_errors=False):
+    """shutil.rmtree(path) wrapper with support for long paths on Windows.
+
+    Availability: Unix, Windows.
+    """
+    onerror = None
+    if isWindows():
+        path = _makelongpath(path)
+        onerror = handle_rmtree_error
+    shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
+
+
+def handle_rmtree_error(function, path, excinfo):
+    # Allow deleting read-only files.
+    os.chmod(path, stat.S_IWRITE)
+    function(path)
+
+
+def rename(src, dst):
+    """os.rename(src, dst) wrapper with support for long paths on Windows.
+
+    Availability: Unix, Windows.
+    """
+    if isWindows():
+        # On Windows, rename fails if destination exists, see
+        # https://docs.python.org/2/library/os.html#os.rename
+        try:
+            os.rename(_makelongpath(src), _makelongpath(dst))
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                os.remove(_makelongpath(dst))
+                os.rename(_makelongpath(src), _makelongpath(dst))
+            else:
+                raise
+    else:
+        shutil.move(src, dst)
+
+
+def remove(path, missing_ok=False):
+    """Remove (delete) the file path. This is a replacement for os.remove that
+    allows deleting read-only files on Windows, with support for long paths and
+    for deleting directory symbolic links.
+
+    Availability: Unix, Windows.
+    """
+    longpath = _makelongpath(path) if isWindows() else path
+    try:
+        os.remove(longpath)
+    except OSError as e:
+        if e.errno == errno.EACCES:
+            os.chmod(longpath, stat.S_IWRITE)
+            # Directory symbolic links must be deleted with 'rmdir'.
+            if islink(longpath) and isdir(longpath):
+                os.rmdir(longpath)
+            else:
+                os.remove(longpath)
+        elif (
+            e.errno == errno.EROFS
+            and missing_ok
+            and not os.path.exists(longpath)
+        ):
+            pass
+        elif missing_ok and e.errno == errno.ENOENT:
+            pass
+        else:
+            raise
+
+
+def walk(top, topdown=True, onerror=None, followlinks=False):
+    """os.walk(path) wrapper with support for long paths on Windows.
+
+    Availability: Windows, Unix.
+    """
+    if isWindows():
+        return _walk_windows_impl(top, topdown, onerror, followlinks)
+    else:
+        return os.walk(top, topdown, onerror, followlinks)
+
+
+def _walk_windows_impl(top, topdown, onerror, followlinks):
+    try:
+        names = listdir(top)
+    except Exception as err:
+        if onerror is not None:
+            onerror(err)
+        return
+
+    dirs, nondirs = [], []
+    for name in names:
+        if isdir(os.path.join(top, name)):
+            dirs.append(name)
+        else:
+            nondirs.append(name)
+
+    if topdown:
+        yield top, dirs, nondirs
+    for name in dirs:
+        new_path = os.path.join(top, name)
+        if followlinks or not islink(new_path):
+            yield from _walk_windows_impl(
+                new_path, topdown, onerror, followlinks
+            )
+    if not topdown:
+        yield top, dirs, nondirs
+
+
+def listdir(path):
+    """os.listdir(path) wrapper with support for long paths on Windows.
+
+    Availability: Windows, Unix.
+    """
+    return os.listdir(_makelongpath(path))
+
+
+def rmdir(path):
+    """os.rmdir(path) wrapper with support for long paths on Windows.
+
+    Availability: Windows, Unix.
+    """
+    os.rmdir(_makelongpath(path))
+
+
+def isdir(path):
+    """os.path.isdir(path) wrapper with support for long paths on Windows.
+
+    Availability: Windows, Unix.
+    """
+    return os.path.isdir(_makelongpath(path))
+
+
+def islink(path):
+    """os.path.islink(path) wrapper with support for long paths on Windows.
+
+    Availability: Windows, Unix.
+    """
+    if isWindows():
+        import platform_utils_win32
+
+        return platform_utils_win32.islink(_makelongpath(path))
+    else:
+        return os.path.islink(path)
+
+
+def readlink(path):
+    """Return a string representing the path to which the symbolic link
+    points. The result may be either an absolute or relative pathname;
+    if it is relative, it may be converted to an absolute pathname using
+    os.path.join(os.path.dirname(path), result).
+
+    Availability: Windows, Unix.
+    """
+    if isWindows():
+        import platform_utils_win32
+
+        return platform_utils_win32.readlink(_makelongpath(path))
+    else:
+        return os.readlink(path)

platform_utils_win32.py

@@ -0,0 +1,244 @@
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ctypes import addressof
+from ctypes import byref
+from ctypes import c_buffer
+from ctypes import c_ubyte
+from ctypes import FormatError
+from ctypes import get_last_error
+from ctypes import Structure
+from ctypes import Union
+from ctypes import WinDLL
+from ctypes import WinError
+from ctypes.wintypes import BOOL
+from ctypes.wintypes import BOOLEAN
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import HANDLE
+from ctypes.wintypes import LPCWSTR
+from ctypes.wintypes import LPDWORD
+from ctypes.wintypes import LPVOID
+from ctypes.wintypes import ULONG
+from ctypes.wintypes import USHORT
+from ctypes.wintypes import WCHAR
+import errno
+
+
+kernel32 = WinDLL("kernel32", use_last_error=True)
+
+UCHAR = c_ubyte
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_NOT_SUPPORTED = 50
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOLEAN
+CreateSymbolicLinkW.argtypes = (
+    LPCWSTR,  # lpSymlinkFileName In
+    LPCWSTR,  # lpTargetFileName In
+    DWORD,  # dwFlags In
+)
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+# symlink support for CreateSymbolicLink() starting with Windows 10 (1703,
+# v10.0.14972)
+SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
+
+GetFileAttributesW = kernel32.GetFileAttributesW
+GetFileAttributesW.restype = DWORD
+GetFileAttributesW.argtypes = (LPCWSTR,)  # lpFileName In
+
+INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
+
+CreateFileW = kernel32.CreateFileW
+CreateFileW.restype = HANDLE
+CreateFileW.argtypes = (
+    LPCWSTR,  # lpFileName In
+    DWORD,  # dwDesiredAccess In
+    DWORD,  # dwShareMode In
+    LPVOID,  # lpSecurityAttributes In_opt
+    DWORD,  # dwCreationDisposition In
+    DWORD,  # dwFlagsAndAttributes In
+    HANDLE,  # hTemplateFile In_opt
+)
+
+CloseHandle = kernel32.CloseHandle
+CloseHandle.restype = BOOL
+CloseHandle.argtypes = (HANDLE,)  # hObject In
+
+INVALID_HANDLE_VALUE = HANDLE(-1).value
+OPEN_EXISTING = 3
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+DeviceIoControl = kernel32.DeviceIoControl
+DeviceIoControl.restype = BOOL
+DeviceIoControl.argtypes = (
+    HANDLE,  # hDevice In
+    DWORD,  # dwIoControlCode In
+    LPVOID,  # lpInBuffer In_opt
+    DWORD,  # nInBufferSize In
+    LPVOID,  # lpOutBuffer Out_opt
+    DWORD,  # nOutBufferSize In
+    LPDWORD,  # lpBytesReturned Out_opt
+    LPVOID,  # lpOverlapped Inout_opt
+)
+
+# Device I/O control flags and options
+FSCTL_GET_REPARSE_POINT = 0x000900A8
+IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
+IO_REPARSE_TAG_SYMLINK = 0xA000000C
+MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
+
+
+class GENERIC_REPARSE_BUFFER(Structure):
+    _fields_ = (("DataBuffer", UCHAR * 1),)
+
+
+class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
+    _fields_ = (
+        ("SubstituteNameOffset", USHORT),
+        ("SubstituteNameLength", USHORT),
+        ("PrintNameOffset", USHORT),
+        ("PrintNameLength", USHORT),
+        ("Flags", ULONG),
+        ("PathBuffer", WCHAR * 1),
+    )
+
+    @property
+    def PrintName(self):
+        arrayt = WCHAR * (self.PrintNameLength // 2)
+        offset = type(self).PathBuffer.offset + self.PrintNameOffset
+        return arrayt.from_address(addressof(self) + offset).value
+
+
+class MOUNT_POINT_REPARSE_BUFFER(Structure):
+    _fields_ = (
+        ("SubstituteNameOffset", USHORT),
+        ("SubstituteNameLength", USHORT),
+        ("PrintNameOffset", USHORT),
+        ("PrintNameLength", USHORT),
+        ("PathBuffer", WCHAR * 1),
+    )
+
+    @property
+    def PrintName(self):
+        arrayt = WCHAR * (self.PrintNameLength // 2)
+        offset = type(self).PathBuffer.offset + self.PrintNameOffset
+        return arrayt.from_address(addressof(self) + offset).value
+
+
+class REPARSE_DATA_BUFFER(Structure):
+    class REPARSE_BUFFER(Union):
+        _fields_ = (
+            ("SymbolicLinkReparseBuffer", SYMBOLIC_LINK_REPARSE_BUFFER),
+            ("MountPointReparseBuffer", MOUNT_POINT_REPARSE_BUFFER),
+            ("GenericReparseBuffer", GENERIC_REPARSE_BUFFER),
+        )
+
+    _fields_ = (
+        ("ReparseTag", ULONG),
+        ("ReparseDataLength", USHORT),
+        ("Reserved", USHORT),
+        ("ReparseBuffer", REPARSE_BUFFER),
+    )
+    _anonymous_ = ("ReparseBuffer",)
+
+
+def create_filesymlink(source, link_name):
+    """Creates a Windows file symbolic link source pointing to link_name."""
+    _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+    """Creates a Windows directory symbolic link source pointing to link_name."""  # noqa: E501
+    _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+    if not CreateSymbolicLinkW(
+        link_name,
+        source,
+        dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE,
+    ):
+        # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0  # noqa: E501
+        # "the unprivileged create flag is unsupported below Windows 10 (1703,
+        # v10.0.14972). retry without it."
+        if not CreateSymbolicLinkW(link_name, source, dwFlags):
+            code = get_last_error()
+            error_desc = FormatError(code).strip()
+            if code == ERROR_PRIVILEGE_NOT_HELD:
+                raise OSError(errno.EPERM, error_desc, link_name)
+            _raise_winerror(code, f'Error creating symbolic link "{link_name}"')
+
+
+def islink(path):
+    result = GetFileAttributesW(path)
+    if result == INVALID_FILE_ATTRIBUTES:
+        return False
+    return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
+
+
+def readlink(path):
+    reparse_point_handle = CreateFileW(
+        path,
+        0,
+        0,
+        None,
+        OPEN_EXISTING,
+        FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
+        None,
+    )
+    if reparse_point_handle == INVALID_HANDLE_VALUE:
+        _raise_winerror(
+            get_last_error(), f'Error opening symbolic link "{path}"'
+        )
+    target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
+    n_bytes_returned = DWORD()
+    io_result = DeviceIoControl(
+        reparse_point_handle,
+        FSCTL_GET_REPARSE_POINT,
+        None,
+        0,
+        target_buffer,
+        len(target_buffer),
+        byref(n_bytes_returned),
+        None,
+    )
+    CloseHandle(reparse_point_handle)
+    if not io_result:
+        _raise_winerror(
+            get_last_error(), f'Error reading symbolic link "{path}"'
+        )
+    rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
+    if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
+        return rdb.SymbolicLinkReparseBuffer.PrintName
+    elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
+        return rdb.MountPointReparseBuffer.PrintName
+    # Unsupported reparse point type.
+    _raise_winerror(
+        ERROR_NOT_SUPPORTED, f'Error reading symbolic link "{path}"'
+    )
+
+
+def _raise_winerror(code, error_desc):
+    win_error_desc = FormatError(code).strip()
+    error_desc = f"{error_desc}: {win_error_desc}"
+    raise WinError(code, error_desc)

progress.py

@@ -0,0 +1,223 @@
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+
+
+try:
+    import threading as _threading
+except ImportError:
+    import dummy_threading as _threading
+
+from repo_trace import IsTraceToStderr
+
+
+_TTY = sys.stderr.isatty()
+
+# This will erase all content in the current line (wherever the cursor is).
+# It does not move the cursor, so this is usually followed by \r to move to
+# column 0.
+CSI_ERASE_LINE = "\x1b[2K"
+
+# This will erase all content in the current line after the cursor.  This is
+# useful for partial updates & progress messages as the terminal can display
+# it better.
+CSI_ERASE_LINE_AFTER = "\x1b[K"
+
+
+def convert_to_hms(total):
+    """Converts a period of seconds to hours, minutes, and seconds."""
+    hours, rem = divmod(total, 3600)
+    mins, secs = divmod(rem, 60)
+    return int(hours), int(mins), secs
+
+
+def duration_str(total):
+    """A less noisy timedelta.__str__.
+
+    The default timedelta stringification contains a lot of leading zeros and
+    uses microsecond resolution.  This makes for noisy output.
+    """
+    hours, mins, secs = convert_to_hms(total)
+    ret = f"{secs:.3f}s"
+    if mins:
+        ret = f"{mins}m{ret}"
+    if hours:
+        ret = f"{hours}h{ret}"
+    return ret
+
+
+def elapsed_str(total):
+    """Returns seconds in the format [H:]MM:SS.
+
+    Does not display a leading zero for minutes if under 10 minutes. This should
+    be used when displaying elapsed time in a progress indicator.
+    """
+    hours, mins, secs = convert_to_hms(total)
+    ret = f"{int(secs):>02d}"
+    if total >= 3600:
+        # Show leading zeroes if over an hour.
+        ret = f"{mins:>02d}:{ret}"
+    else:
+        ret = f"{mins}:{ret}"
+    if hours:
+        ret = f"{hours}:{ret}"
+    return ret
+
+
+def jobs_str(total):
+    return f"{total} job{'s' if total > 1 else ''}"
+
+
+class Progress:
+    def __init__(
+        self,
+        title,
+        total=0,
+        units="",
+        delay=True,
+        quiet=False,
+        show_elapsed=False,
+        elide=False,
+    ):
+        self._title = title
+        self._total = total
+        self._done = 0
+        self._start = time.time()
+        self._show = not delay
+        self._units = units
+        self._elide = elide and _TTY
+        self._quiet = quiet
+
+        # Only show the active jobs section if we run more than one in parallel.
+        self._show_jobs = False
+        self._active = 0
+
+        # Save the last message for displaying on refresh.
+        self._last_msg = None
+        self._show_elapsed = show_elapsed
+        self._update_event = _threading.Event()
+        self._update_thread = _threading.Thread(
+            target=self._update_loop,
+        )
+        self._update_thread.daemon = True
+
+        if not quiet and show_elapsed:
+            self._update_thread.start()
+
+    def _update_loop(self):
+        while True:
+            self.update(inc=0)
+            if self._update_event.wait(timeout=1):
+                return
+
+    def _write(self, s):
+        s = "\r" + s
+        if self._elide:
+            col = os.get_terminal_size(sys.stderr.fileno()).columns
+            if len(s) > col:
+                s = s[: col - 1] + ".."
+        sys.stderr.write(s)
+        sys.stderr.flush()
+
+    def start(self, name):
+        self._active += 1
+        if not self._show_jobs:
+            self._show_jobs = self._active > 1
+        self.update(inc=0, msg="started " + name)
+
+    def finish(self, name):
+        self.update(msg="finished " + name)
+        self._active -= 1
+
+    def update(self, inc=1, msg=None):
+        """Updates the progress indicator.
+
+        Args:
+            inc: The number of items completed.
+            msg: The message to display. If None, use the last message.
+        """
+        self._done += inc
+        if msg is None:
+            msg = self._last_msg
+        self._last_msg = msg
+
+        if not _TTY or IsTraceToStderr() or self._quiet:
+            return
+
+        elapsed_sec = time.time() - self._start
+        if not self._show:
+            if 0.5 <= elapsed_sec:
+                self._show = True
+            else:
+                return
+
+        if self._total <= 0:
+            self._write(
+                "%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER)
+            )
+        else:
+            p = (100 * self._done) / self._total
+            if self._show_jobs:
+                jobs = f"[{jobs_str(self._active)}] "
+            else:
+                jobs = ""
+            if self._show_elapsed:
+                elapsed = f" {elapsed_str(elapsed_sec)} |"
+            else:
+                elapsed = ""
+            self._write(
+                "%s: %2d%% %s(%d%s/%d%s)%s %s%s"
+                % (
+                    self._title,
+                    p,
+                    jobs,
+                    self._done,
+                    self._units,
+                    self._total,
+                    self._units,
+                    elapsed,
+                    msg,
+                    CSI_ERASE_LINE_AFTER,
+                )
+            )
+
+    def end(self):
+        self._update_event.set()
+        if not _TTY or IsTraceToStderr() or self._quiet:
+            return
+
+        duration = duration_str(time.time() - self._start)
+        if self._total <= 0:
+            self._write(
+                "%s: %d, done in %s%s\n"
+                % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER)
+            )
+        else:
+            p = (100 * self._done) / self._total
+            self._write(
+                "%s: %3d%% (%d%s/%d%s), done in %s%s\n"
+                % (
+                    self._title,
+                    p,
+                    self._done,
+                    self._units,
+                    self._total,
+                    self._units,
+                    duration,
+                    CSI_ERASE_LINE_AFTER,
+                )
+            )

pyproject.toml

@@ -0,0 +1,23 @@
+# Copyright 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[tool.black]
+line-length = 80
+# NB: Keep in sync with tox.ini.
+target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] #, 'py312'
+
+[tool.pytest.ini_options]
+markers = """
+    skip_cq: Skip tests in the CQ. Should be rarely used!
+"""

release/README.md

@@ -0,0 +1,2 @@
+These are helper tools for managing official releases.
+See the [release process](../docs/release-process.md) document for more details.

release/sign-launcher.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for signing repo launcher scripts correctly.
+
+This is intended to be run only by the official Repo release managers.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+import util
+
+
+def sign(opts):
+    """Sign the launcher!"""
+    output = ""
+    for key in opts.keys:
+        # We use ! at the end of the key so that gpg uses this specific key.
+        # Otherwise it uses the key as a lookup into the overall key and uses
+        # the default signing key.  i.e. It will see that KEYID_RSA is a subkey
+        # of another key, and use the primary key to sign instead of the subkey.
+        cmd = [
+            "gpg",
+            "--homedir",
+            opts.gpgdir,
+            "-u",
+            f"{key}!",
+            "--batch",
+            "--yes",
+            "--armor",
+            "--detach-sign",
+            "--output",
+            "-",
+            opts.launcher,
+        ]
+        ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+        output += ret.stdout
+
+    # Save the combined signatures into one file.
+    with open(f"{opts.launcher}.asc", "w", encoding="utf-8") as fp:
+        fp.write(output)
+
+
+def check(opts):
+    """Check the signature."""
+    util.run(opts, ["gpg", "--verify", f"{opts.launcher}.asc"])
+
+
+def get_version(opts):
+    """Get the version from |launcher|."""
+    # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+    launcher = os.path.join(".", opts.launcher)
+    cmd = [launcher, "--version"]
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+    m = re.search(r"repo launcher version ([0-9.]+)", ret.stdout)
+    if not m:
+        sys.exit(f"{opts.launcher}: unable to detect repo version")
+    return m.group(1)
+
+
+def postmsg(opts, version):
+    """Helpful info to show at the end for release manager."""
+    print(
+        f"""
+Repo launcher bucket:
+  gs://git-repo-downloads/
+
+You should first upload it with a specific version:
+  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
+
+Then to make it the public default:
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+
+NB: If a rollback is necessary, the GS bucket archives old versions, and may be
+    accessed by specifying their unique id number.
+  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
+"""  # noqa: E501
+    )
+
+
+def get_parser():
+    """Get a CLI parser."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        dest="dryrun",
+        action="store_true",
+        help="show everything that would be done",
+    )
+    parser.add_argument(
+        "--gpgdir",
+        default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
+        help="path to dedicated gpg dir with release keys "
+        "(default: ~/.gnupg/repo/)",
+    )
+    parser.add_argument(
+        "--keyid",
+        dest="keys",
+        default=[],
+        action="append",
+        help="alternative signing keys to use",
+    )
+    parser.add_argument(
+        "launcher",
+        default=os.path.join(util.TOPDIR, "repo"),
+        nargs="?",
+        help="the launcher script to sign",
+    )
+    return parser
+
+
+def main(argv):
+    """The main func!"""
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    if not os.path.exists(opts.gpgdir):
+        parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
+    if not os.path.exists(opts.launcher):
+        parser.error(f"launcher does not exist: {opts.launcher}")
+
+    opts.launcher = os.path.relpath(opts.launcher)
+    print(
+        f'Signing "{opts.launcher}" launcher script and saving to '
+        f'"{opts.launcher}.asc"'
+    )
+
+    if opts.keys:
+        print(f'Using custom keys to sign: {" ".join(opts.keys)}')
+    else:
+        print("Using official Repo release keys to sign")
+        opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
+        util.import_release_key(opts)
+
+    version = get_version(opts)
+    sign(opts)
+    check(opts)
+    postmsg(opts, version)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

release/sign-tag.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for signing repo release tags correctly.
+
+This is intended to be run only by the official Repo release managers, but it
+could be run by people maintaining their own fork of the project.
+
+NB: Check docs/release-process.md for production freeze information.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+import util
+
+
+# We currently sign with the old DSA key as it's been around the longest.
+# We should transition to RSA by Jun 2020, and ECC by Jun 2021.
+KEYID = util.KEYID_DSA
+
+# Regular expression to validate tag names.
+RE_VALID_TAG = r"^v([0-9]+[.])+[0-9]+$"
+
+
+def sign(opts):
+    """Tag the commit & sign it!"""
+    # We use ! at the end of the key so that gpg uses this specific key.
+    # Otherwise it uses the key as a lookup into the overall key and uses the
+    # default signing key.  i.e. It will see that KEYID_RSA is a subkey of
+    # another key, and use the primary key to sign instead of the subkey.
+    cmd = [
+        "git",
+        "tag",
+        "-s",
+        opts.tag,
+        "-u",
+        f"{opts.key}!",
+        "-m",
+        f"repo {opts.tag}",
+        opts.commit,
+    ]
+
+    key = "GNUPGHOME"
+    print("+", f'export {key}="{opts.gpgdir}"')
+    oldvalue = os.getenv(key)
+    os.putenv(key, opts.gpgdir)
+    util.run(opts, cmd)
+    if oldvalue is None:
+        os.unsetenv(key)
+    else:
+        os.putenv(key, oldvalue)
+
+
+def check(opts):
+    """Check the signature."""
+    util.run(opts, ["git", "tag", "--verify", opts.tag])
+
+
+def postmsg(opts):
+    """Helpful info to show at the end for release manager."""
+    cmd = ["git", "rev-parse", "remotes/origin/stable"]
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+    current_release = ret.stdout.strip()
+
+    cmd = [
+        "git",
+        "log",
+        "--format=%h (%aN) %s",
+        "--no-merges",
+        f"remotes/origin/stable..{opts.tag}",
+    ]
+    ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
+    shortlog = ret.stdout.strip()
+
+    print(
+        f"""
+Here's the short log since the last release.
+{shortlog}
+
+To push release to the public:
+  git push origin {opts.commit}:stable {opts.tag} -n
+NB: People will start upgrading to this version immediately.
+
+To roll back a release:
+  git push origin --force {current_release}:stable -n
+"""
+    )
+
+
+def get_parser():
+    """Get a CLI parser."""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        dest="dryrun",
+        action="store_true",
+        help="show everything that would be done",
+    )
+    parser.add_argument(
+        "--gpgdir",
+        default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
+        help="path to dedicated gpg dir with release keys "
+        "(default: ~/.gnupg/repo/)",
+    )
+    parser.add_argument(
+        "-f", "--force", action="store_true", help="force signing of any tag"
+    )
+    parser.add_argument(
+        "--keyid", dest="key", help="alternative signing key to use"
+    )
+    parser.add_argument("tag", help='the tag to create (e.g. "v2.0")')
+    parser.add_argument(
+        "commit", default="HEAD", nargs="?", help="the commit to tag"
+    )
+    return parser
+
+
+def main(argv):
+    """The main func!"""
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    if not os.path.exists(opts.gpgdir):
+        parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
+
+    if not opts.force and not re.match(RE_VALID_TAG, opts.tag):
+        parser.error(
+            f'tag "{opts.tag}" does not match regex "{RE_VALID_TAG}"; '
+            "use --force to sign anyways"
+        )
+
+    if opts.key:
+        print(f"Using custom key to sign: {opts.key}")
+    else:
+        print("Using official Repo release key to sign")
+        opts.key = KEYID
+        util.import_release_key(opts)
+
+    sign(opts)
+    check(opts)
+    postmsg(opts)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

release/update-hooks

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for updating hooks from their various upstreams."""
+
+import argparse
+import base64
+import json
+from pathlib import Path
+import sys
+from typing import List, Optional
+import urllib.request
+
+
+assert sys.version_info >= (3, 8), "Python 3.8+ required"
+
+
+TOPDIR = Path(__file__).resolve().parent.parent
+HOOKS_DIR = TOPDIR / "hooks"
+
+
+def update_hook_commit_msg() -> None:
+    """Update commit-msg hook from Gerrit."""
+    hook = HOOKS_DIR / "commit-msg"
+    print(
+        f"{hook.name}: Updating from https://gerrit.googlesource.com/gerrit/"
+        "+/HEAD/resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+
+    # Get the current commit.
+    url = "https://gerrit.googlesource.com/gerrit/+/HEAD?format=JSON"
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+    # Discard the xss protection.
+    data = data.split(b"\n", 1)[1]
+    data = json.loads(data)
+    commit = data["commit"]
+
+    # Fetch the data for that commit.
+    url = (
+        f"https://gerrit.googlesource.com/gerrit/+/{commit}/"
+        "resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+    with urllib.request.urlopen(f"{url}?format=TEXT") as fp:
+        data = fp.read()
+
+    # gitiles base64 encodes text data.
+    data = base64.b64decode(data)
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://gerrit.googlesource.com/gerrit/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def update_hook_pre_auto_gc() -> None:
+    """Update pre-auto-gc hook from git."""
+    hook = HOOKS_DIR / "pre-auto-gc"
+    print(
+        f"{hook.name}: Updating from https://github.com/git/git/"
+        "HEAD/contrib/hooks/pre-auto-gc-battery"
+    )
+
+    # Get the current commit.
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    url = "https://api.github.com/repos/git/git/git/refs/heads/master"
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req) as fp:
+        data = fp.read()
+    data = json.loads(data)
+
+    # Fetch the data for that commit.
+    commit = data["object"]["sha"]
+    url = (
+        f"https://raw.githubusercontent.com/git/git/{commit}/"
+        "contrib/hooks/pre-auto-gc-battery"
+    )
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://github.com/git/git/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def main(argv: Optional[List[str]] = None) -> Optional[int]:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.parse_args(argv)
+
+    update_hook_commit_msg()
+    update_hook_pre_auto_gc()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

release/update-manpages

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for generating manual page for all repo commands.
+
+This is intended to be run before every official Repo release.
+"""
+
+import sys
+
+import update_manpages
+
+
+sys.exit(update_manpages.main(sys.argv[1:]))

release/update_manpages.py

@@ -0,0 +1,183 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for generating manual page for all repo commands.
+
+Most code lives in this module so it can be unittested.
+"""
+
+import argparse
+import functools
+import multiprocessing
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import List
+
+
+THIS_FILE = Path(__file__).resolve()
+TOPDIR = THIS_FILE.parent.parent
+MANDIR = TOPDIR.joinpath("man")
+
+# Load repo local modules.
+sys.path.insert(0, str(TOPDIR))
+from git_command import RepoSourceVersion
+import subcmds
+
+
+def worker(cmd, **kwargs):
+    subprocess.run(cmd, **kwargs)
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """Get argument parser."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--check",
+        "--dry-run",
+        action="store_const",
+        const=True,
+        help="Check if changes are necessary; don't actually change files",
+    )
+    return parser
+
+
+def main(argv: List[str]) -> int:
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    if not shutil.which("help2man"):
+        sys.exit("Please install help2man to continue.")
+
+    # Let repo know we're generating man pages so it can avoid some dynamic
+    # behavior (like probing active number of CPUs).  We use a weird name &
+    # value to make it less likely for users to set this var themselves.
+    os.environ["_REPO_GENERATE_MANPAGES_"] = " indeed! "
+
+    # "repo branch" is an alias for "repo branches".
+    del subcmds.all_commands["branch"]
+    (MANDIR / "repo-branch.1").write_text(".so man1/repo-branches.1")
+
+    version = RepoSourceVersion()
+    cmdlist = [
+        [
+            "help2man",
+            "-N",
+            "-n",
+            f"repo {cmd} - manual page for repo {cmd}",
+            "-S",
+            f"repo {cmd}",
+            "-m",
+            "Repo Manual",
+            f"--version-string={version}",
+            "-o",
+            MANDIR.joinpath(f"repo-{cmd}.1.tmp"),
+            "./repo",
+            "-h",
+            f"help {cmd}",
+        ]
+        for cmd in subcmds.all_commands
+    ]
+    cmdlist.append(
+        [
+            "help2man",
+            "-N",
+            "-n",
+            "repository management tool built on top of git",
+            "-S",
+            "repo",
+            "-m",
+            "Repo Manual",
+            f"--version-string={version}",
+            "-o",
+            MANDIR.joinpath("repo.1.tmp"),
+            "./repo",
+            "-h",
+            "--help-all",
+        ]
+    )
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir = Path(tempdir)
+        repo_dir = tempdir / ".repo"
+        repo_dir.mkdir()
+        (repo_dir / "repo").symlink_to(TOPDIR)
+
+        # Create a repo wrapper using the active Python executable.  We can't
+        # pass this directly to help2man as it's too simple, so insert it via
+        # shebang.
+        data = (TOPDIR / "repo").read_text(encoding="utf-8")
+        tempbin = tempdir / "repo"
+        tempbin.write_text(f"#!{sys.executable}\n" + data, encoding="utf-8")
+        tempbin.chmod(0o755)
+
+        # Run all cmd in parallel, and wait for them to finish.
+        with multiprocessing.Pool() as pool:
+            pool.map(
+                functools.partial(worker, cwd=tempdir, check=True), cmdlist
+            )
+
+    ret = 0
+    for tmp_path in MANDIR.glob("*.1.tmp"):
+        path = tmp_path.parent / tmp_path.stem
+        old_data = path.read_text() if path.exists() else ""
+
+        data = tmp_path.read_text()
+        tmp_path.unlink()
+
+        data = replace_regex(data)
+
+        # If the only thing that changed was the date, don't refresh.  This
+        # avoids a lot of noise when only one file actually updates.
+        old_data = re.sub(
+            r'^(\.TH REPO "1" ")([^"]+)', r"\1", old_data, flags=re.M
+        )
+        new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r"\1", data, flags=re.M)
+        if old_data != new_data:
+            if opts.check:
+                ret = 1
+                print(
+                    f"{THIS_FILE.name}: {path.name}: "
+                    "man page needs regenerating",
+                    file=sys.stderr,
+                )
+            else:
+                path.write_text(data)
+
+    return ret
+
+
+def replace_regex(data):
+    """Replace semantically null regexes in the data.
+
+    Args:
+        data: manpage text.
+
+    Returns:
+        Updated manpage text.
+    """
+    regex = (
+        (r"(It was generated by help2man) [0-9.]+", r"\g<1>."),
+        (r"^\033\[[0-9;]*m([^\033]*)\033\[m", r"\g<1>"),
+        (r"^\.IP\n(.*:)\n", r".SS \g<1>\n"),
+        (r"^\.PP\nDescription", r".SH DETAILS"),
+    )
+    for pattern, replacement in regex:
+        data = re.sub(pattern, replacement, data, flags=re.M)
+    return data

release/util.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Random utility code for release tools."""
+
+import os
+import re
+import shlex
+import subprocess
+import sys
+
+
+assert sys.version_info >= (3, 6), "This module requires Python 3.6+"
+
+
+TOPDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+HOMEDIR = os.path.expanduser("~")
+
+
+# These are the release keys we sign with.
+KEYID_DSA = "8BB9AD793E8E6153AF0F9A4416530D5E920F5C65"
+KEYID_RSA = "A34A13BE8E76BFF46A0C022DA2E75A824AAB9624"
+KEYID_ECC = "E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39"
+
+
+def cmdstr(cmd):
+    """Get a nicely quoted shell command."""
+    return " ".join(shlex.quote(x) for x in cmd)
+
+
+def run(opts, cmd, check=True, **kwargs):
+    """Helper around subprocess.run to include logging."""
+    print("+", cmdstr(cmd))
+    if opts.dryrun:
+        cmd = ["true", "--"] + cmd
+    try:
+        return subprocess.run(cmd, check=check, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print(f"aborting: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def import_release_key(opts):
+    """Import the public key of the official release repo signing key."""
+    # Extract the key from our repo launcher.
+    launcher = getattr(opts, "launcher", os.path.join(TOPDIR, "repo"))
+    print(f'Importing keys from "{launcher}" launcher script')
+    with open(launcher, encoding="utf-8") as fp:
+        data = fp.read()
+
+    keys = re.findall(
+        r"\n-----BEGIN PGP PUBLIC KEY BLOCK-----\n[^-]*"
+        r"\n-----END PGP PUBLIC KEY BLOCK-----\n",
+        data,
+        flags=re.M,
+    )
+    run(opts, ["gpg", "--import"], input="\n".join(keys).encode("utf-8"))
+
+    print("Marking keys as fully trusted")
+    run(
+        opts,
+        ["gpg", "--import-ownertrust"],
+        input=f"{KEYID_DSA}:6:\n".encode("utf-8"),
+    )

repo_logging.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logic for printing user-friendly logs in repo."""
+
+import logging
+
+from color import Coloring
+from error import RepoExitError
+
+
+SEPARATOR = "=" * 80
+MAX_PRINT_ERRORS = 5
+
+
+class _ConfigMock:
+    """Default coloring config to use when Logging.config is not set."""
+
+    def __init__(self):
+        self.default_values = {"color.ui": "auto"}
+
+    def GetString(self, x):
+        return self.default_values.get(x, None)
+
+
+class _LogColoring(Coloring):
+    """Coloring outstream for logging."""
+
+    def __init__(self, config):
+        super().__init__(config, "logs")
+        self.error = self.nofmt_colorer("error", fg="red")
+        self.warning = self.nofmt_colorer("warn", fg="yellow")
+        self.levelMap = {
+            "WARNING": self.warning,
+            "ERROR": self.error,
+        }
+
+
+class _LogColoringFormatter(logging.Formatter):
+    """Coloring formatter for logging."""
+
+    def __init__(self, config=None, *args, **kwargs):
+        self.config = config if config else _ConfigMock()
+        self.colorer = _LogColoring(self.config)
+        super().__init__(*args, **kwargs)
+
+    def format(self, record):
+        """Formats |record| with color."""
+        msg = super().format(record)
+        colorer = self.colorer.levelMap.get(record.levelname)
+        return msg if not colorer else colorer(msg)
+
+
+class RepoLogger(logging.Logger):
+    """Repo Logging Module."""
+
+    def __init__(self, name: str, config=None, **kwargs):
+        super().__init__(name, **kwargs)
+        handler = logging.StreamHandler()
+        handler.setFormatter(_LogColoringFormatter(config))
+        self.addHandler(handler)
+
+    def log_aggregated_errors(self, err: RepoExitError):
+        """Print all aggregated logs."""
+        self.error(SEPARATOR)
+
+        if not err.aggregate_errors:
+            self.error("Repo command failed: %s", type(err).__name__)
+            self.error("\t%s", str(err))
+            return
+
+        self.error(
+            "Repo command failed due to the following `%s` errors:",
+            type(err).__name__,
+        )
+        self.error(
+            "\n".join(str(e) for e in err.aggregate_errors[:MAX_PRINT_ERRORS])
+        )
+
+        diff = len(err.aggregate_errors) - MAX_PRINT_ERRORS
+        if diff > 0:
+            self.error("+%d additional errors...", diff)

repo_trace.py

@@ -0,0 +1,171 @@
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logic for tracing repo interactions.
+
+Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+
+Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
+To also include trace outputs in stderr do `repo --trace_to_stderr ...`
+"""
+
+import contextlib
+import os
+import sys
+import tempfile
+import time
+
+import platform_utils
+
+
+# Env var to implicitly turn on tracing.
+REPO_TRACE = "REPO_TRACE"
+
+# Temporarily set tracing to always on unless user expicitly sets to 0.
+_TRACE = os.environ.get(REPO_TRACE) != "0"
+_TRACE_TO_STDERR = False
+_TRACE_FILE = None
+_TRACE_FILE_NAME = "TRACE_FILE"
+_MAX_SIZE = 70  # in MiB
+_NEW_COMMAND_SEP = "+++++++++++++++NEW COMMAND+++++++++++++++++++"
+
+
+def IsTraceToStderr():
+    """Whether traces are written to stderr."""
+    return _TRACE_TO_STDERR
+
+
+def IsTrace():
+    """Whether tracing is enabled."""
+    return _TRACE
+
+
+def SetTraceToStderr():
+    """Enables tracing logging to stderr."""
+    global _TRACE_TO_STDERR
+    _TRACE_TO_STDERR = True
+
+
+def SetTrace():
+    """Enables tracing."""
+    global _TRACE
+    _TRACE = True
+
+
+def _SetTraceFile(quiet):
+    """Sets the trace file location."""
+    global _TRACE_FILE
+    _TRACE_FILE = _GetTraceFile(quiet)
+
+
+class Trace(contextlib.ContextDecorator):
+    """Used to capture and save git traces."""
+
+    def _time(self):
+        """Generate nanoseconds of time in a py3.6 safe way"""
+        return int(time.time() * 1e9)
+
+    def __init__(self, fmt, *args, first_trace=False, quiet=True):
+        """Initialize the object.
+
+        Args:
+            fmt: The format string for the trace.
+            *args: Arguments to pass to formatting.
+            first_trace: Whether this is the first trace of a `repo` invocation.
+            quiet: Whether to suppress notification of trace file location.
+        """
+        if not IsTrace():
+            return
+        self._trace_msg = fmt % args
+
+        if not _TRACE_FILE:
+            _SetTraceFile(quiet)
+
+        if first_trace:
+            _ClearOldTraces()
+            self._trace_msg = f"{_NEW_COMMAND_SEP} {self._trace_msg}"
+
+    def __enter__(self):
+        if not IsTrace():
+            return self
+
+        print_msg = (
+            f"PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n"
+        )
+
+        with open(_TRACE_FILE, "a") as f:
+            print(print_msg, file=f)
+
+        if _TRACE_TO_STDERR:
+            print(print_msg, file=sys.stderr)
+
+        return self
+
+    def __exit__(self, *exc):
+        if not IsTrace():
+            return False
+
+        print_msg = (
+            f"PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n"
+        )
+
+        with open(_TRACE_FILE, "a") as f:
+            print(print_msg, file=f)
+
+        if _TRACE_TO_STDERR:
+            print(print_msg, file=sys.stderr)
+
+        return False
+
+
+def _GetTraceFile(quiet):
+    """Get the trace file or create one."""
+    # TODO: refactor to pass repodir to Trace.
+    repo_dir = os.path.dirname(os.path.dirname(__file__))
+    trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
+    if not quiet:
+        print(f"Trace outputs in {trace_file}", file=sys.stderr)
+    return trace_file
+
+
+def _ClearOldTraces():
+    """Clear the oldest commands if trace file is too big."""
+    try:
+        with open(_TRACE_FILE, errors="ignore") as f:
+            if os.path.getsize(f.name) / (1024 * 1024) <= _MAX_SIZE:
+                return
+            trace_lines = f.readlines()
+    except FileNotFoundError:
+        return
+
+    while sum(len(x) for x in trace_lines) / (1024 * 1024) > _MAX_SIZE:
+        for i, line in enumerate(trace_lines):
+            if "END:" in line and _NEW_COMMAND_SEP in line:
+                trace_lines = trace_lines[i + 1 :]
+                break
+        else:
+            # The last chunk is bigger than _MAX_SIZE, so just throw everything
+            # away.
+            trace_lines = []
+
+    while trace_lines and trace_lines[-1] == "\n":
+        trace_lines = trace_lines[:-1]
+    # Write to a temporary file with a unique name in the same filesystem
+    # before replacing the original trace file.
+    temp_dir, temp_prefix = os.path.split(_TRACE_FILE)
+    with tempfile.NamedTemporaryFile(
+        "w", dir=temp_dir, prefix=temp_prefix, delete=False
+    ) as f:
+        f.writelines(trace_lines)
+    platform_utils.rename(f.name, _TRACE_FILE)

requirements.json

@@ -0,0 +1,59 @@
+# This file declares various requirements for this version of repo.  The
+# launcher script will load it and check the constraints before trying to run
+# us.  This avoids issues of the launcher using an old version of Python (e.g.
+# 3.5) while the codebase has moved on to requiring something much newer (e.g.
+# 3.8).  If the launcher tried to import us, it would fail with syntax errors.
+
+# This is a JSON file with line-level comments allowed.
+
+# Always keep backwards compatibility in mine.  The launcher script is robust
+# against missing values, but when a field is renamed/removed, it means older
+# versions of the launcher script won't be able to enforce the constraint.
+
+# When requiring versions, always use lists as they are easy to parse & compare
+# in Python.  Strings would require futher processing to turn into a list.
+
+# Version constraints should be expressed in pairs: soft & hard.  Soft versions
+# are when we start warning users that their software too old and we're planning
+# on dropping support for it, so they need to start planning system upgrades.
+# Hard versions are when we refuse to work the tool.  Users will be shown an
+# error message before we abort entirely.
+
+# When deciding whether to upgrade a version requirement, check out the distro
+# lists to see who will be impacted:
+# https://gerrit.googlesource.com/git-repo/+/HEAD/docs/release-process.md#Project-References
+
+{
+  # The repo launcher itself.  This allows us to force people to upgrade as some
+  # ignore the warnings about it being out of date, or install ancient versions
+  # to start with for whatever reason.
+  #
+  # NB: Repo launchers started checking this file with repo-2.12, so listing
+  # versions older than that won't make a difference.
+  "repo": {
+    "hard": [2, 11],
+    "soft": [2, 11]
+  },
+
+  # Supported Python versions.
+  #
+  # python-3.6 is in Ubuntu Bionic.
+  # python-3.7 is in Debian Buster.
+  "python": {
+    "hard": [3, 6],
+    "soft": [3, 6]
+  },
+
+  # Supported git versions.
+  #
+  # git-1.9.1 is in Ubuntu Trusty.
+  # git-2.1.4 is in Debian Jessie.
+  # git-2.7.4 is in Ubuntu Xenial.
+  # git-2.11.0 is in Debian Stretch.
+  # git-2.17.0 is in Ubuntu Bionic.
+  # git-2.20.1 is in Debian Buster.
+  "git": {
+    "hard": [1, 9, 1],
+    "soft": [2, 7, 4]
+  }
+}

run_tests

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper to run linters and pytest with the right settings."""
+
+import functools
+import os
+import shutil
+import subprocess
+import sys
+from typing import List
+
+
+ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+@functools.lru_cache()
+def is_ci() -> bool:
+    """Whether we're running in our CI system."""
+    return os.getenv("LUCI_CQ") == "yes"
+
+
+def run_pytest(argv: List[str]) -> int:
+    """Returns the exit code from pytest."""
+    if is_ci():
+        argv = ["-m", "not skip_cq"] + argv
+
+    return subprocess.run(
+        [sys.executable, "-m", "pytest"] + argv,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_pytest_py38(argv: List[str]) -> int:
+    """Returns the exit code from pytest under Python 3.8."""
+    if is_ci():
+        argv = ["-m", "not skip_cq"] + argv
+
+    try:
+        return subprocess.run(
+            [
+                "vpython3",
+                "-vpython-spec",
+                "run_tests.vpython3.8",
+                "-m",
+                "pytest",
+            ]
+            + argv,
+            check=False,
+            cwd=ROOT_DIR,
+        ).returncode
+    except FileNotFoundError:
+        # Skip if the user doesn't have vpython from depot_tools.
+        return 0
+
+
+def run_black():
+    """Returns the exit code from black."""
+    # Black by default only matches .py files.  We have to list standalone
+    # scripts manually.
+    extra_programs = [
+        "repo",
+        "run_tests",
+        "release/update-hooks",
+        "release/update-manpages",
+    ]
+    return subprocess.run(
+        [sys.executable, "-m", "black", "--check", ROOT_DIR] + extra_programs,
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_flake8():
+    """Returns the exit code from flake8."""
+    return subprocess.run(
+        [sys.executable, "-m", "flake8", ROOT_DIR],
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_isort():
+    """Returns the exit code from isort."""
+    return subprocess.run(
+        [sys.executable, "-m", "isort", "--check", ROOT_DIR],
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def run_update_manpages() -> int:
+    """Returns the exit code from release/update-manpages."""
+    # Allow this to fail on CI, but not local devs.
+    if is_ci() and not shutil.which("help2man"):
+        print("update-manpages: help2man not found; skipping test")
+        return 0
+
+    return subprocess.run(
+        [sys.executable, "release/update-manpages", "--check"],
+        check=False,
+        cwd=ROOT_DIR,
+    ).returncode
+
+
+def main(argv):
+    """The main entry."""
+    checks = (
+        functools.partial(run_pytest, argv),
+        functools.partial(run_pytest_py38, argv),
+        run_black,
+        run_flake8,
+        run_isort,
+        run_update_manpages,
+    )
+    # Run all the tests all the time to get full feedback.  Don't exit on the
+    # first error as that makes it more difficult to iterate in the CQ.
+    return 1 if sum(c() for c in checks) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

run_tests.vpython3

@@ -0,0 +1,125 @@
+# This is a vpython "spec" file.
+#
+# Read more about `vpython` and how to modify this file here:
+#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
+# List of available wheels:
+#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
+
+python_version: "3.11"
+
+wheel: <
+  name: "infra/python/wheels/pytest-py3"
+  version: "version:8.3.4"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/py-py2_py3"
+  version: "version:1.11.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/iniconfig-py3"
+  version: "version:1.1.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/packaging-py3"
+  version: "version:23.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pluggy-py3"
+  version: "version:1.5.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/toml-py3"
+  version: "version:0.10.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pyparsing-py3"
+  version: "version:3.0.7"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/attrs-py2_py3"
+  version: "version:21.4.0"
+>
+
+# NB: Keep in sync with constraints.txt.
+wheel: <
+  name: "infra/python/wheels/black-py3"
+  version: "version:25.1.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/mypy-extensions-py3"
+  version: "version:0.4.3"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/tomli-py3"
+  version: "version:2.0.1"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/platformdirs-py3"
+  version: "version:2.5.2"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/pathspec-py3"
+  version: "version:0.9.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/typing-extensions-py3"
+  version: "version:4.3.0"
+>
+
+# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/click-py3"
+  version: "version:8.0.3"
+>
+
+wheel: <
+  name: "infra/python/wheels/flake8-py2_py3"
+  version: "version:6.0.0"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/mccabe-py2_py3"
+  version: "version:0.7.0"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/pyflakes-py2_py3"
+  version: "version:3.0.1"
+>
+
+# Required by flake8==6.0.0
+wheel: <
+  name: "infra/python/wheels/pycodestyle-py2_py3"
+  version: "version:2.10.0"
+>
+
+wheel: <
+  name: "infra/python/wheels/isort-py3"
+  version: "version:5.10.1"
+>

run_tests.vpython3.8

@@ -0,0 +1,67 @@
+# This is a vpython "spec" file.
+#
+# Read more about `vpython` and how to modify this file here:
+#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
+# List of available wheels:
+#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
+
+python_version: "3.8"
+
+wheel: <
+  name: "infra/python/wheels/pytest-py3"
+  version: "version:8.3.4"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/py-py2_py3"
+  version: "version:1.11.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/iniconfig-py3"
+  version: "version:1.1.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/packaging-py3"
+  version: "version:23.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pluggy-py3"
+  version: "version:1.5.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/toml-py3"
+  version: "version:0.10.1"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/tomli-py3"
+  version: "version:2.1.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/pyparsing-py3"
+  version: "version:3.0.7"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/attrs-py2_py3"
+  version: "version:21.4.0"
+>
+
+# Required by pytest==8.3.4
+wheel: <
+  name: "infra/python/wheels/exceptiongroup-py3"
+  version: "version:1.1.2"
+>

setup.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Python packaging for repo."""
+
+import os
+
+import setuptools
+
+
+TOPDIR = os.path.dirname(os.path.abspath(__file__))
+
+
+# Rip out the first intro paragraph.
+with open(os.path.join(TOPDIR, "README.md")) as fp:
+    lines = fp.read().splitlines()[2:]
+    end = lines.index("")
+    long_description = " ".join(lines[0:end])
+
+
+# https://packaging.python.org/tutorials/packaging-projects/
+setuptools.setup(
+    name="repo",
+    version="2",
+    maintainer="Various",
+    maintainer_email="repo-discuss@googlegroups.com",
+    description="Repo helps manage many Git repositories",
+    long_description=long_description,
+    long_description_content_type="text/plain",
+    url="https://gerrit.googlesource.com/git-repo/",
+    project_urls={
+        "Bug Tracker": "https://issues.gerritcodereview.com/issues?q=is:open%20componentid:1370071",  # noqa: E501
+    },
+    # https://pypi.org/classifiers/
+    classifiers=[
+        "Development Status :: 6 - Mature",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Natural Language :: English",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: Microsoft :: Windows :: Windows 10",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3 :: Only",
+        "Topic :: Software Development :: Version Control :: Git",
+    ],
+    python_requires=">=3.6",
+    packages=["subcmds"],
+)

ssh.py

@@ -0,0 +1,351 @@
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common SSH management logic."""
+
+import functools
+import multiprocessing
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+from git_command import git
+import platform_utils
+from repo_trace import Trace
+
+
+PROXY_PATH = os.path.join(os.path.dirname(__file__), "git_ssh")
+
+
+def _run_ssh_version():
+    """run ssh -V to display the version number"""
+    return subprocess.check_output(
+        ["ssh", "-V"], stderr=subprocess.STDOUT
+    ).decode()
+
+
+def _parse_ssh_version(ver_str=None):
+    """parse a ssh version string into a tuple"""
+    if ver_str is None:
+        ver_str = _run_ssh_version()
+    m = re.match(r"^OpenSSH_([0-9.]+)(p[0-9]+)?[\s,]", ver_str)
+    if m:
+        return tuple(int(x) for x in m.group(1).split("."))
+    else:
+        return ()
+
+
+@functools.lru_cache(maxsize=None)
+def version():
+    """return ssh version as a tuple"""
+    try:
+        return _parse_ssh_version()
+    except FileNotFoundError:
+        print("fatal: ssh not installed", file=sys.stderr)
+        sys.exit(1)
+    except subprocess.CalledProcessError as e:
+        print(
+            "fatal: unable to detect ssh version"
+            f" (code={e.returncode}, output={e.stdout})",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+
+URI_SCP = re.compile(r"^([^@:]*@?[^:/]{1,}):")
+URI_ALL = re.compile(r"^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/")
+
+
+class ProxyManager:
+    """Manage various ssh clients & masters that we spawn.
+
+    This will take care of sharing state between multiprocessing children, and
+    make sure that if we crash, we don't leak any of the ssh sessions.
+
+    The code should work with a single-process scenario too, and not add too
+    much overhead due to the manager.
+    """
+
+    # Path to the ssh program to run which will pass our master settings along.
+    # Set here more as a convenience API.
+    proxy = PROXY_PATH
+
+    def __init__(self, manager):
+        # Protect access to the list of active masters.
+        self._lock = multiprocessing.Lock()
+        # List of active masters (pid).  These will be spawned on demand, and we
+        # are responsible for shutting them all down at the end.
+        self._masters = manager.list()
+        # Set of active masters indexed by "host:port" information.
+        # The value isn't used, but multiprocessing doesn't provide a set class.
+        self._master_keys = manager.dict()
+        # Whether ssh masters are known to be broken, so we give up entirely.
+        self._master_broken = manager.Value("b", False)
+        # List of active ssh sesssions.  Clients will be added & removed as
+        # connections finish, so this list is just for safety & cleanup if we
+        # crash.
+        self._clients = manager.list()
+        # Path to directory for holding master sockets.
+        self._sock_path = None
+
+    def __enter__(self):
+        """Enter a new context."""
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Exit a context & clean up all resources."""
+        self.close()
+
+    def add_client(self, proc):
+        """Track a new ssh session."""
+        self._clients.append(proc.pid)
+
+    def remove_client(self, proc):
+        """Remove a completed ssh session."""
+        try:
+            self._clients.remove(proc.pid)
+        except ValueError:
+            pass
+
+    def add_master(self, proc):
+        """Track a new master connection."""
+        self._masters.append(proc.pid)
+
+    def _terminate(self, procs):
+        """Kill all |procs|."""
+        for pid in procs:
+            try:
+                os.kill(pid, signal.SIGTERM)
+                os.waitpid(pid, 0)
+            except OSError:
+                pass
+
+        # The multiprocessing.list() API doesn't provide many standard list()
+        # methods, so we have to manually clear the list.
+        while True:
+            try:
+                procs.pop(0)
+            except:  # noqa: E722
+                break
+
+    def close(self):
+        """Close this active ssh session.
+
+        Kill all ssh clients & masters we created, and nuke the socket dir.
+        """
+        self._terminate(self._clients)
+        self._terminate(self._masters)
+
+        d = self.sock(create=False)
+        if d:
+            try:
+                platform_utils.rmdir(os.path.dirname(d))
+            except OSError:
+                pass
+
+    def _open_unlocked(self, host, port=None):
+        """Make sure a ssh master session exists for |host| & |port|.
+
+        If one doesn't exist already, we'll create it.
+
+        We won't grab any locks, so the caller has to do that.  This helps keep
+        the business logic of actually creating the master separate from
+        grabbing locks.
+        """
+        # Check to see whether we already think that the master is running; if
+        # we think it's already running, return right away.
+        if port is not None:
+            key = f"{host}:{port}"
+        else:
+            key = host
+
+        if key in self._master_keys:
+            return True
+
+        if self._master_broken.value or "GIT_SSH" in os.environ:
+            # Failed earlier, so don't retry.
+            return False
+
+        # We will make two calls to ssh; this is the common part of both calls.
+        command_base = ["ssh", "-o", "ControlPath %s" % self.sock(), host]
+        if port is not None:
+            command_base[1:1] = ["-p", str(port)]
+
+        # Since the key wasn't in _master_keys, we think that master isn't
+        # running... but before actually starting a master, we'll double-check.
+        # This can be important because we can't tell that that 'git@myhost.com'
+        # is the same as 'myhost.com' where "User git" is setup in the user's
+        # ~/.ssh/config file.
+        check_command = command_base + ["-O", "check"]
+        with Trace("Call to ssh (check call): %s", " ".join(check_command)):
+            try:
+                check_process = subprocess.Popen(
+                    check_command,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+                check_process.communicate()  # read output, but ignore it...
+                isnt_running = check_process.wait()
+
+                if not isnt_running:
+                    # Our double-check found that the master _was_ infact
+                    # running.  Add to the list of keys.
+                    self._master_keys[key] = True
+                    return True
+            except Exception:
+                # Ignore excpetions.  We we will fall back to the normal command
+                # and print to the log there.
+                pass
+
+        # Git protocol V2 is a new feature in git 2.18.0, made default in
+        # git 2.26.0
+        # It is faster and more efficient than V1.
+        # To enable it when using SSH, the environment variable GIT_PROTOCOL
+        # must be set in the SSH side channel when establishing the connection
+        # to the git server.
+        # See https://git-scm.com/docs/protocol-v2#_ssh_and_file_transport
+        # Normally git does this by itself. But here, where the SSH connection
+        # is established manually over ControlMaster via the repo-tool, it must
+        # be passed in explicitly instead.
+        # Based on https://git-scm.com/docs/gitprotocol-pack#_extra_parameters,
+        # GIT_PROTOCOL is considered an "Extra Parameter" and must be ignored
+        # by servers that do not understand it. This means that it is safe to
+        # set it even when connecting to older servers.
+        # It should also be safe to set the environment variable for older
+        # local git versions, since it is only part of the ssh side channel.
+        git_protocol_version = _get_git_protocol_version()
+        ssh_git_protocol_args = [
+            "-o",
+            f"SetEnv GIT_PROTOCOL=version={git_protocol_version}",
+        ]
+
+        command = (
+            command_base[:1]
+            + ["-M", "-N", *ssh_git_protocol_args]
+            + command_base[1:]
+        )
+        p = None
+        try:
+            with Trace("Call to ssh: %s", " ".join(command)):
+                p = subprocess.Popen(command)
+        except Exception as e:
+            self._master_broken.value = True
+            print(
+                "\nwarn: cannot enable ssh control master for %s:%s\n%s"
+                % (host, port, str(e)),
+                file=sys.stderr,
+            )
+            return False
+
+        time.sleep(1)
+        ssh_died = p.poll() is not None
+        if ssh_died:
+            return False
+
+        self.add_master(p)
+        self._master_keys[key] = True
+        return True
+
+    def _open(self, host, port=None):
+        """Make sure a ssh master session exists for |host| & |port|.
+
+        If one doesn't exist already, we'll create it.
+
+        This will obtain any necessary locks to avoid inter-process races.
+        """
+        # Bail before grabbing the lock if we already know that we aren't going
+        # to try creating new masters below.
+        if sys.platform in ("win32", "cygwin"):
+            return False
+
+        # Acquire the lock.  This is needed to prevent opening multiple masters
+        # for the same host when we're running "repo sync -jN" (for N > 1) _and_
+        # the manifest <remote fetch="ssh://xyz"> specifies a different host
+        # from the one that was passed to repo init.
+        with self._lock:
+            return self._open_unlocked(host, port)
+
+    def preconnect(self, url):
+        """If |uri| will create a ssh connection, setup the ssh master for it."""  # noqa: E501
+        m = URI_ALL.match(url)
+        if m:
+            scheme = m.group(1)
+            host = m.group(2)
+            if ":" in host:
+                host, port = host.split(":")
+            else:
+                port = None
+            if scheme in ("ssh", "git+ssh", "ssh+git"):
+                return self._open(host, port)
+            return False
+
+        m = URI_SCP.match(url)
+        if m:
+            host = m.group(1)
+            return self._open(host)
+
+        return False
+
+    def sock(self, create=True):
+        """Return the path to the ssh socket dir.
+
+        This has all the master sockets so clients can talk to them.
+        """
+        if self._sock_path is None:
+            if not create:
+                return None
+            tmp_dir = "/tmp"
+            if not os.path.exists(tmp_dir):
+                tmp_dir = tempfile.gettempdir()
+            if version() < (6, 7):
+                tokens = "%r@%h:%p"
+            else:
+                tokens = "%C"  # hash of %l%h%p%r
+            self._sock_path = os.path.join(
+                tempfile.mkdtemp("", "ssh-", tmp_dir), "master-" + tokens
+            )
+        return self._sock_path
+
+
+@functools.lru_cache(maxsize=1)
+def _get_git_protocol_version() -> str:
+    """Return the git protocol version.
+
+    The version is found by first reading the global git config.
+    If no git config for protocol version exists, try to deduce the default
+    protocol version based on the git version.
+
+    See https://git-scm.com/docs/gitprotocol-v2 for details.
+    """
+    try:
+        return subprocess.check_output(
+            ["git", "config", "--get", "--global", "protocol.version"],
+            encoding="utf-8",
+            stderr=subprocess.PIPE,
+        ).strip()
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            # Exit code 1 means that the git config key was not found.
+            # Try to imitate the defaults that git would have used.
+            git_version = git.version_tuple()
+            if git_version >= (2, 26, 0):
+                # Since git version 2.26, protocol v2 is the default.
+                return "2"
+            return "1"
+        # Other exit codes indicate error with reading the config.
+        raise

subcmds/__init__.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,35 +14,35 @@
 
 import os
 
-all = {}
+
+# A mapping of the subcommand name to the class that implements it.
+all_commands = {}
+all_modules = []
 
 my_dir = os.path.dirname(__file__)
 for py in os.listdir(my_dir):
-  if py == '__init__.py':
-    continue
+    if py == "__init__.py":
+        continue
 
-  if py.endswith('.py'):
-    name = py[:-3]
+    if py.endswith(".py"):
+        name = py[:-3]
 
-    clsn = name.capitalize()
-    while clsn.find('_') > 0:
-      h = clsn.index('_')
-      clsn = clsn[0:h] + clsn[h + 1:].capitalize()
+        clsn = name.capitalize()
+        while clsn.find("_") > 0:
+            h = clsn.index("_")
+            clsn = clsn[0:h] + clsn[h + 1 :].capitalize()
 
-    mod = __import__(__name__,
-                     globals(),
-                     locals(),
-                     ['%s' % name])
-    mod = getattr(mod, name)
-    try:
-      cmd = getattr(mod, clsn)()
-    except AttributeError:
-      raise SyntaxError, '%s/%s does not define class %s' % (
-                         __name__, py, clsn)
+        mod = __import__(__name__, globals(), locals(), ["%s" % name])
+        mod = getattr(mod, name)
+        try:
+            cmd = getattr(mod, clsn)
+        except AttributeError:
+            raise SyntaxError(f"{__name__}/{py} does not define class {clsn}")
 
-    name = name.replace('_', '-')
-    cmd.NAME = name
-    all[name] = cmd
+        name = name.replace("_", "-")
+        cmd.NAME = name
+        all_commands[name] = cmd
+        all_modules.append(mod)
 
-if 'help' in all:
-  all['help'].commands = all
+# Add 'branch' as an alias for 'branches'.
+all_commands["branch"] = all_commands["branches"]

subcmds/abandon.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,30 +12,149 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+import collections
+import functools
+import itertools
+
 from command import Command
+from command import DEFAULT_LOCAL_JOBS
+from error import RepoError
+from error import RepoExitError
 from git_command import git
+from progress import Progress
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class AbandonError(RepoExitError):
+    """Exit error when abandon command fails."""
+
 
 class Abandon(Command):
-  common = True
-  helpSummary = "Permanently abandon a development branch"
-  helpUsage = """
-%prog <branchname> [<project>...]
+    COMMON = True
+    helpSummary = "Permanently abandon a development branch"
+    helpUsage = """
+%prog [--all | <branchname>] [<project>...]
 
 This subcommand permanently abandons a development branch by
 deleting it (and all its history) from your local repository.
 
 It is equivalent to "git branch -D <branchname>".
 """
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-  def Execute(self, opt, args):
-    if not args:
-      self.Usage()
+    def _Options(self, p):
+        p.add_option(
+            "--all",
+            action="store_true",
+            help="delete all branches in all projects",
+        )
 
-    nb = args[0]
-    if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
-      sys.exit(1)
+    def ValidateOptions(self, opt, args):
+        if not opt.all and not args:
+            self.Usage()
 
-    for project in self.GetProjects(args[1:]):
-      project.AbandonBranch(nb)
+        if not opt.all:
+            branches = args[0].split()
+            invalid_branches = [
+                x for x in branches if not git.check_ref_format(f"heads/{x}")
+            ]
+
+            if invalid_branches:
+                self.OptionParser.error(
+                    f"{invalid_branches} are not valid branch names"
+                )
+        else:
+            args.insert(0, "'All local branches'")
+
+    @classmethod
+    def _ExecuteOne(cls, all_branches, nb, project_idx):
+        """Abandon one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
+        if all_branches:
+            branches = project.GetBranches()
+        else:
+            branches = nb
+
+        ret = {}
+        errors = []
+        for name in branches:
+            status = None
+            try:
+                status = project.AbandonBranch(name)
+            except RepoError as e:
+                status = False
+                errors.append(e)
+            if status is not None:
+                ret[name] = status
+
+        return (ret, project_idx, errors)
+
+    def Execute(self, opt, args):
+        nb = args[0].split()
+        err = collections.defaultdict(list)
+        success = collections.defaultdict(list)
+        aggregate_errors = []
+        all_projects = self.GetProjects(
+            args[1:], all_manifests=not opt.this_manifest_only
+        )
+        _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+
+        def _ProcessResults(_pool, pm, states):
+            for results, project_idx, errors in states:
+                project = all_projects[project_idx]
+                for branch, status in results.items():
+                    if status:
+                        success[branch].append(project)
+                    else:
+                        err[branch].append(project)
+                aggregate_errors.extend(errors)
+                pm.update(msg="")
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, opt.all, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Abandon {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
+
+        width = max(
+            itertools.chain(
+                [25], (len(x) for x in itertools.chain(success, err))
+            )
+        )
+        if err:
+            for br in err.keys():
+                err_msg = "error: cannot abandon %s" % br
+                logger.error(err_msg)
+                for proj in err[br]:
+                    logger.error(" " * len(err_msg) + " | %s", _RelPath(proj))
+            raise AbandonError(aggregate_errors=aggregate_errors)
+        elif not success:
+            logger.error("error: no project has local branch(es) : %s", nb)
+            raise AbandonError(aggregate_errors=aggregate_errors)
+        else:
+            # Everything below here is displaying status.
+            if opt.quiet:
+                return
+            print("Abandoned branches:")
+            for br in success.keys():
+                if len(all_projects) > 1 and len(all_projects) == len(
+                    success[br]
+                ):
+                    result = "all project"
+                else:
+                    result = "%s" % (
+                        ("\n" + " " * width + "| ").join(
+                            _RelPath(p) for p in success[br]
+                        )
+                    )
+                print(f"{br}{' ' * (width - len(br))}| {result}\n")

subcmds/branches.py

@@ -0,0 +1,215 @@
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import sys
+
+from color import Coloring
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
+
+
+class BranchColoring(Coloring):
+    def __init__(self, config):
+        Coloring.__init__(self, config, "branch")
+        self.current = self.printer("current", fg="green")
+        self.local = self.printer("local")
+        self.notinproject = self.printer("notinproject", fg="red")
+
+
+class BranchInfo:
+    def __init__(self, name):
+        self.name = name
+        self.current = 0
+        self.published = 0
+        self.published_equal = 0
+        self.projects = []
+
+    def add(self, b):
+        if b.current:
+            self.current += 1
+        if b.published:
+            self.published += 1
+        if b.revision == b.published:
+            self.published_equal += 1
+        self.projects.append(b)
+
+    @property
+    def IsCurrent(self):
+        return self.current > 0
+
+    @property
+    def IsSplitCurrent(self):
+        return self.current != 0 and self.current != len(self.projects)
+
+    @property
+    def IsPublished(self):
+        return self.published > 0
+
+    @property
+    def IsPublishedEqual(self):
+        return self.published_equal == len(self.projects)
+
+
+class Branches(Command):
+    COMMON = True
+    helpSummary = "View current topic branches"
+    helpUsage = """
+%prog [<project>...]
+
+Summarizes the currently available topic branches.
+
+# Branch Display
+
+The branch display output by this command is organized into four
+columns of information; for example:
+
+ *P nocolor                   | in repo
+    repo2                     |
+
+The first column contains a * if the branch is the currently
+checked out branch in any of the specified projects, or a blank
+if no project has the branch checked out.
+
+The second column contains either blank, p or P, depending upon
+the upload status of the branch.
+
+ (blank): branch not yet published by repo upload
+       P: all commits were published by repo upload
+       p: only some commits were published by repo upload
+
+The third column contains the branch name.
+
+The fourth column (after the | separator) lists the projects that
+the branch appears in, or does not appear in.  If no project list
+is shown, then the branch appears in all projects.
+
+"""
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+
+    @classmethod
+    def _ExpandProjectToBranches(cls, project_idx):
+        """Expands a project into a list of branch names & associated info.
+
+        Args:
+            project_idx: project.Project index
+
+        Returns:
+            List[Tuple[str, git_config.Branch, int]]
+        """
+        branches = []
+        project = cls.get_parallel_context()["projects"][project_idx]
+        for name, b in project.GetBranches().items():
+            branches.append((name, b, project_idx))
+        return branches
+
+    def Execute(self, opt, args):
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+        out = BranchColoring(self.manifest.manifestProject.config)
+        all_branches = {}
+        project_cnt = len(projects)
+
+        def _ProcessResults(_pool, _output, results):
+            for name, b, project_idx in itertools.chain.from_iterable(results):
+                b.project = projects[project_idx]
+                if name not in all_branches:
+                    all_branches[name] = BranchInfo(name)
+                all_branches[name].add(b)
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                self._ExpandProjectToBranches,
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
+
+        names = sorted(all_branches)
+
+        if not names:
+            print("   (no branches)", file=sys.stderr)
+            return
+
+        width = 25
+        for name in names:
+            if width < len(name):
+                width = len(name)
+
+        for name in names:
+            i = all_branches[name]
+            in_cnt = len(i.projects)
+
+            if i.IsCurrent:
+                current = "*"
+                hdr = out.current
+            else:
+                current = " "
+                hdr = out.local
+
+            if i.IsPublishedEqual:
+                published = "P"
+            elif i.IsPublished:
+                published = "p"
+            else:
+                published = " "
+
+            # A branch name can contain a percent sign, so we need to escape it.
+            # Escape after f-string formatting to properly account for leading
+            # spaces.
+            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
+            out.write(" |")
+
+            _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
+            if in_cnt < project_cnt:
+                fmt = out.write
+                paths = []
+                non_cur_paths = []
+                if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
+                    in_type = "in"
+                    for b in i.projects:
+                        relpath = _RelPath(b.project)
+                        if not i.IsSplitCurrent or b.current:
+                            paths.append(relpath)
+                        else:
+                            non_cur_paths.append(relpath)
+                else:
+                    fmt = out.notinproject
+                    in_type = "not in"
+                    have = set()
+                    for b in i.projects:
+                        have.add(_RelPath(b.project))
+                    for p in projects:
+                        if _RelPath(p) not in have:
+                            paths.append(_RelPath(p))
+
+                s = f" {in_type} {', '.join(paths)}"
+                if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+                    fmt = out.current if i.IsCurrent else fmt
+                    fmt(s)
+                else:
+                    fmt(" %s:" % in_type)
+                    fmt = out.current if i.IsCurrent else out.write
+                    for p in paths:
+                        out.nl()
+                        fmt(width * " " + "          %s" % p)
+                    fmt = out.write
+                    for p in non_cur_paths:
+                        out.nl()
+                        fmt(width * " " + "          %s" % p)
+            else:
+                out.write(" in all projects")
+            out.nl()

subcmds/checkout.py

@@ -0,0 +1,114 @@
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+from typing import NamedTuple
+
+from command import Command
+from command import DEFAULT_LOCAL_JOBS
+from error import GitError
+from error import RepoExitError
+from progress import Progress
+from repo_logging import RepoLogger
+
+
+logger = RepoLogger(__file__)
+
+
+class CheckoutBranchResult(NamedTuple):
+    # Whether the Project is on the branch (i.e. branch exists and no errors)
+    result: bool
+    project_idx: int
+    error: Exception
+
+
+class CheckoutCommandError(RepoExitError):
+    """Exception thrown when checkout command fails."""
+
+
+class MissingBranchError(RepoExitError):
+    """Exception thrown when no project has specified branch."""
+
+
+class Checkout(Command):
+    COMMON = True
+    helpSummary = "Checkout a branch for development"
+    helpUsage = """
+%prog <branchname> [<project>...]
+"""
+    helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
+
+The command is equivalent to:
+
+  repo forall [<project>...] -c git checkout <branchname>
+"""
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+
+    def ValidateOptions(self, opt, args):
+        if not args:
+            self.Usage()
+
+    @classmethod
+    def _ExecuteOne(cls, nb, project_idx):
+        """Checkout one project."""
+        error = None
+        result = None
+        project = cls.get_parallel_context()["projects"][project_idx]
+        try:
+            result = project.CheckoutBranch(nb)
+        except GitError as e:
+            error = e
+        return CheckoutBranchResult(result, project_idx, error)
+
+    def Execute(self, opt, args):
+        nb = args[0]
+        err = []
+        err_projects = []
+        success = []
+        all_projects = self.GetProjects(
+            args[1:], all_manifests=not opt.this_manifest_only
+        )
+
+        def _ProcessResults(_pool, pm, results):
+            for result in results:
+                project = all_projects[result.project_idx]
+                if result.error is not None:
+                    err.append(result.error)
+                    err_projects.append(project)
+                elif result.result:
+                    success.append(project)
+                pm.update(msg="")
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Checkout {nb}", len(all_projects), quiet=opt.quiet
+                ),
+            )
+
+        if err_projects:
+            for p in err_projects:
+                logger.error("error: %s/: cannot checkout %s", p.relpath, nb)
+            raise CheckoutCommandError(aggregate_errors=err)
+        elif not success:
+            msg = f"error: no project has branch {nb}"
+            logger.error(msg)
+            raise MissingBranchError(msg)

subcmds/cherry_pick.py

@@ -0,0 +1,145 @@
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+
+from command import Command
+from error import GitError
+from git_command import GitCommand
+from repo_logging import RepoLogger
+
+
+CHANGE_ID_RE = re.compile(r"^\s*Change-Id: I([0-9a-f]{40})\s*$")
+logger = RepoLogger(__file__)
+
+
+class CherryPick(Command):
+    COMMON = True
+    helpSummary = "Cherry-pick a change."
+    helpUsage = """
+%prog <sha1>
+"""
+    helpDescription = """
+'%prog' cherry-picks a change from one branch to another.
+The change id will be updated, and a reference to the old
+change id will be added.
+"""
+
+    def ValidateOptions(self, opt, args):
+        if len(args) != 1:
+            self.Usage()
+
+    def Execute(self, opt, args):
+        reference = args[0]
+
+        p = GitCommand(
+            None,
+            ["rev-parse", "--verify", reference],
+            capture_stdout=True,
+            capture_stderr=True,
+            verify_command=True,
+        )
+        try:
+            p.Wait()
+        except GitError:
+            logger.error(p.stderr)
+            raise
+
+        sha1 = p.stdout.strip()
+
+        p = GitCommand(
+            None,
+            ["cat-file", "commit", sha1],
+            capture_stdout=True,
+            verify_command=True,
+        )
+
+        try:
+            p.Wait()
+        except GitError:
+            logger.error("error: Failed to retrieve old commit message")
+            raise
+
+        old_msg = self._StripHeader(p.stdout)
+
+        p = GitCommand(
+            None,
+            ["cherry-pick", sha1],
+            capture_stdout=True,
+            capture_stderr=True,
+            verify_command=True,
+        )
+
+        try:
+            p.Wait()
+        except GitError as e:
+            logger.error(e)
+            logger.warning(
+                "NOTE: When committing (please see above) and editing the "
+                "commit message, please remove the old Change-Id-line and "
+                "add:\n%s",
+                self._GetReference(sha1),
+            )
+            raise
+
+        if p.stdout:
+            print(p.stdout.strip(), file=sys.stdout)
+        if p.stderr:
+            print(p.stderr.strip(), file=sys.stderr)
+
+        # The cherry-pick was applied correctly. We just need to edit
+        # the commit message.
+        new_msg = self._Reformat(old_msg, sha1)
+
+        p = GitCommand(
+            None,
+            ["commit", "--amend", "-F", "-"],
+            input=new_msg,
+            capture_stdout=True,
+            capture_stderr=True,
+            verify_command=True,
+        )
+        try:
+            p.Wait()
+        except GitError:
+            logger.error("error: Failed to update commit message")
+            raise
+
+    def _IsChangeId(self, line):
+        return CHANGE_ID_RE.match(line)
+
+    def _GetReference(self, sha1):
+        return "(cherry picked from commit %s)" % sha1
+
+    def _StripHeader(self, commit_msg):
+        lines = commit_msg.splitlines()
+        return "\n".join(lines[lines.index("") + 1 :])
+
+    def _Reformat(self, old_msg, sha1):
+        new_msg = []
+
+        for line in old_msg.splitlines():
+            if not self._IsChangeId(line):
+                new_msg.append(line)
+
+        # Add a blank line between the message and the change id/reference.
+        try:
+            if new_msg[-1].strip() != "":
+                new_msg.append("")
+        except IndexError:
+            pass
+
+        new_msg.append(self._GetReference(sha1))
+        return "\n".join(new_msg)

subcmds/diff.py

@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,15 +12,75 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+import io
+
+from command import DEFAULT_LOCAL_JOBS
 from command import PagedCommand
 
-class Diff(PagedCommand):
-  common = True
-  helpSummary = "Show changes between commit and working tree"
-  helpUsage = """
-%prog [<project>...]
-"""
 
-  def Execute(self, opt, args):
-    for project in self.GetProjects(args):
-      project.PrintWorkTreeDiff()
+class Diff(PagedCommand):
+    COMMON = True
+    helpSummary = "Show changes between commit and working tree"
+    helpUsage = """
+%prog [<project>...]
+
+The -u option causes '%prog' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
+"""
+    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+
+    def _Options(self, p):
+        p.add_option(
+            "-u",
+            "--absolute",
+            action="store_true",
+            help="paths are relative to the repository root",
+        )
+
+    @classmethod
+    def _ExecuteOne(cls, absolute, local, project_idx):
+        """Obtains the diff for a specific project.
+
+        Args:
+            absolute: Paths are relative to the root.
+            local: a boolean, if True, the path is relative to the local
+                (sub)manifest.  If false, the path is relative to the outermost
+                manifest.
+            project_idx: Project index to get status of.
+
+        Returns:
+            The status of the project.
+        """
+        buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
+        ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
+        return (ret, buf.getvalue())
+
+    def Execute(self, opt, args):
+        all_projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+
+        def _ProcessResults(_pool, _output, results):
+            ret = 0
+            for state, output in results:
+                if output:
+                    print(output, end="")
+                if not state:
+                    ret = 1
+            return ret
+
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            return self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne, opt.absolute, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )