launcher: add test for version requirements

Make sure the modules stay in sync in case one is updated but we
forgot to update the other.

Bug: https://crbug.com/gerrit/13795
Change-Id: I6de9533d45c083e5f7ad792ee6d541e23647de3f
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/292444
Reviewed-by: Michael Mortensen <mmortensen@google.com>
Tested-by: Mike Frysinger <vapier@google.com>

.gitignore

@@ -7,6 +7,7 @@ __pycache__
 .repopickle_*
 /repoc
 /.tox
+/.venv
 
 # PyCharm related
 /.idea/

docs/manifest-format.md

@@ -111,6 +111,10 @@ A description of the elements and their attributes follows.
 
 The root element of the file.
 
+### Element notice
+
+Arbitrary text that is displayed to users whenever `repo sync` finishes.
+The content is simply passed through as it exists in the manifest.
 
 ### Element remote
 
@@ -263,7 +267,7 @@ Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
 "all", and each project automatically belongs to a group of
 its name:`name` and path:`path`.  E.g. for
-<project name="monkeys" path="barrel-of"/>, that project
+`<project name="monkeys" path="barrel-of"/>`, that project
 definition is implicitly in the following manifest groups:
 default, name:monkeys, and path:barrel-of.  If you place a project in the
 group "notdefault", it will not be automatically downloaded by repo.
@@ -360,6 +364,19 @@ This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
+### Element repo-hooks
+
+NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.
+
+Only one repo-hooks element may be specified at a time.
+Attempting to redefine it will fail to parse.
+
+Attribute `in-project`: The project where the hooks are defined.  The value
+must match the `name` attribute (**not** the `path` attribute) of a previously
+defined `project` element.
+
+Attribute `enabled-list`: List of hooks to use, whitespace or comma separated.
+
 ### Element include
 
 This element provides the capability of including another manifest

git_trace2_event_log.py

@@ -0,0 +1,197 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provide event logging in the git trace2 EVENT format.
+
+The git trace2 EVENT format is defined at:
+https://www.kernel.org/pub/software/scm/git/docs/technical/api-trace2.html#_event_format
+https://git-scm.com/docs/api-trace2#_the_event_format_target
+
+  Usage:
+
+  git_trace_log = EventLog()
+  git_trace_log.StartEvent()
+  ...
+  git_trace_log.ExitEvent()
+  git_trace_log.Write()
+"""
+
+
+import datetime
+import json
+import os
+import sys
+import tempfile
+import threading
+
+from git_command import GitCommand, RepoSourceVersion
+
+
+class EventLog(object):
+  """Event log that records events that occurred during a repo invocation.
+
+  Events are written to the log as a consecutive JSON entries, one per line.
+  Entries follow the git trace2 EVENT format.
+
+  Each entry contains the following common keys:
+  - event: The event name
+  - sid: session-id - Unique string to allow process instance to be identified.
+  - thread: The thread name.
+  - time: is the UTC time of the event.
+
+  Valid 'event' names and event specific fields are documented here:
+  https://git-scm.com/docs/api-trace2#_event_format
+  """
+
+  def __init__(self, env=None):
+    """Initializes the event log."""
+    self._log = []
+    # Try to get session-id (sid) from environment (setup in repo launcher).
+    KEY = 'GIT_TRACE2_PARENT_SID'
+    if env is None:
+      env = os.environ
+
+    now = datetime.datetime.utcnow()
+
+    # Save both our sid component and the complete sid.
+    # We use our sid component (self._sid) as the unique filename prefix and
+    # the full sid (self._full_sid) in the log itself.
+    self._sid = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid())
+    parent_sid = env.get(KEY)
+    # Append our sid component to the parent sid (if it exists).
+    if parent_sid is not None:
+      self._full_sid = parent_sid + '/' + self._sid
+    else:
+      self._full_sid = self._sid
+
+    # Set/update the environment variable.
+    # Environment handling across systems is messy.
+    try:
+      env[KEY] = self._full_sid
+    except UnicodeEncodeError:
+      env[KEY] = self._full_sid.encode()
+
+    # Add a version event to front of the log.
+    self._AddVersionEvent()
+
+  @property
+  def full_sid(self):
+    return self._full_sid
+
+  def _AddVersionEvent(self):
+    """Adds a 'version' event at the beginning of current log."""
+    version_event = self._CreateEventDict('version')
+    version_event['evt'] = 2
+    version_event['exe'] = RepoSourceVersion()
+    self._log.insert(0, version_event)
+
+  def _CreateEventDict(self, event_name):
+    """Returns a dictionary with the common keys/values for git trace2 events.
+
+    Args:
+      event_name: The event name.
+
+    Returns:
+      Dictionary with the common event fields populated.
+    """
+    return {
+        'event': event_name,
+        'sid': self._full_sid,
+        'thread': threading.currentThread().getName(),
+        'time': datetime.datetime.utcnow().isoformat() + 'Z',
+    }
+
+  def StartEvent(self):
+    """Append a 'start' event to the current log."""
+    start_event = self._CreateEventDict('start')
+    start_event['argv'] = sys.argv
+    self._log.append(start_event)
+
+  def ExitEvent(self, result):
+    """Append an 'exit' event to the current log.
+
+    Args:
+      result: Exit code of the event
+    """
+    exit_event = self._CreateEventDict('exit')
+
+    # Consider 'None' success (consistent with event_log result handling).
+    if result is None:
+      result = 0
+    exit_event['code'] = result
+    self._log.append(exit_event)
+
+  def Write(self, path=None):
+    """Writes the log out to a file.
+
+    Log is only written if 'path' or 'git config --get trace2.eventtarget'
+    provide a valid path to write logs to.
+
+    Logging filename format follows the git trace2 style of being a unique
+    (exclusive writable) file.
+
+    Args:
+      path: Path to where logs should be written.
+
+    Returns:
+      log_path: Path to the log file if log is written, otherwise None
+    """
+    log_path = None
+    # If no logging path is specified, get the path from 'trace2.eventtarget'.
+    if path is None:
+      cmd = ['config', '--get', 'trace2.eventtarget']
+      # TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
+      # system git config variables.
+      p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
+                     bare=True)
+      retval = p.Wait()
+      if retval == 0:
+        # Strip trailing carriage-return in path.
+        path = p.stdout.rstrip('\n')
+      elif retval != 1:
+        # `git config --get` is documented to produce an exit status of `1` if
+        # the requested variable is not present in the configuration. Report any
+        # other return value as an error.
+        print("repo: error: 'git config --get' call failed with return code: %r, stderr: %r" % (
+            retval, p.stderr), file=sys.stderr)
+
+    if isinstance(path, str):
+      # Get absolute path.
+      path = os.path.abspath(os.path.expanduser(path))
+    else:
+      raise TypeError('path: str required but got %s.' % type(path))
+
+    # Git trace2 requires a directory to write log to.
+
+    # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
+    if not os.path.isdir(path):
+      return None
+    # Use NamedTemporaryFile to generate a unique filename as required by git trace2.
+    try:
+      with tempfile.NamedTemporaryFile(mode='x', prefix=self._sid, dir=path,
+                                       delete=False) as f:
+        # TODO(https://crbug.com/gerrit/13706): Support writing events as they
+        # occur.
+        for e in self._log:
+          # Dump in compact encoding mode.
+          # See 'Compact encoding' in Python docs:
+          # https://docs.python.org/3/library/json.html#module-json
+          json.dump(e, f, indent=None, separators=(',', ':'))
+          f.write('\n')
+        log_path = f.name
+    except FileExistsError as err:
+      print('repo: warning: git trace2 logging failed: %r' % err,
+            file=sys.stderr)
+      return None
+    return log_path

gitc_utils.py

@@ -45,7 +45,8 @@ def _set_project_revisions(projects):
   should not be overly large. Recommend calling this function multiple times
   with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
 
-  @param projects: List of project objects to set the revionExpr for.
+  Args:
+    projects: List of project objects to set the revionExpr for.
   """
   # Retrieve the commit id for each project based off of it's current
   # revisionExpr and it is not already a commit id.
@@ -73,7 +74,8 @@ def _manifest_groups(manifest):
   This is the same logic used by Command.GetProjects(), which is used during
   repo sync
 
-  @param manifest: The XmlManifest object
+  Args:
+    manifest: The XmlManifest object
   """
   mp = manifest.manifestProject
   groups = mp.config.GetString('manifest.groups')
@@ -85,9 +87,10 @@ def _manifest_groups(manifest):
 def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   """Generate a manifest for shafsd to use for this GITC client.
 
-  @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
-  @param manifest: A GitcManifest object loaded with the current repo manifest.
-  @param paths: List of project paths we want to update.
+  Args:
+    gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+    manifest: A GitcManifest object loaded with the current repo manifest.
+    paths: List of project paths we want to update.
   """
 
   print('Generating GITC Manifest by fetching revision SHAs for each '
@@ -149,12 +152,15 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
 def save_manifest(manifest, client_dir=None):
   """Save the manifest file in the client_dir.
 
-  @param client_dir: Client directory to save the manifest in.
-  @param manifest: Manifest object to save.
+  Args:
+    manifest: Manifest object to save.
+    client_dir: Client directory to save the manifest in.
   """
   if not client_dir:
-    client_dir = manifest.gitc_client_dir
-  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest_file = manifest.manifestFile
+  else:
+    manifest_file = os.path.join(client_dir, '.manifest')
+  with open(manifest_file, 'w') as f:
     manifest.Save(f, groups=_manifest_groups(manifest))
   # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
   # Give the GITC filesystem time to register the manifest changes.

main.py

@@ -50,6 +50,7 @@ import event_log
 from repo_trace import SetTrace
 from git_command import user_agent
 from git_config import init_ssh, close_ssh, RepoConfig
+from git_trace2_event_log import EventLog
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import GitcAvailableCommand, GitcClientCommand
@@ -82,7 +83,7 @@ if not is_python3():
 #
 # python-3.6 is in Ubuntu Bionic.
 MIN_PYTHON_VERSION_SOFT = (3, 6)
-MIN_PYTHON_VERSION_HARD = (3, 4)
+MIN_PYTHON_VERSION_HARD = (3, 5)
 
 if sys.version_info.major < 3:
   print('repo: error: Python 2 is no longer supported; '
@@ -130,6 +131,8 @@ global_options.add_option('--version',
 global_options.add_option('--event-log',
                           dest='event_log', action='store',
                           help='filename of event log to append timeline to')
+global_options.add_option('--git-trace2-event-log', action='store',
+                          help='directory to write git trace2 event log to')
 
 
 class _Repo(object):
@@ -211,6 +214,7 @@ class _Repo(object):
             file=sys.stderr)
       return 1
 
+    git_trace2_event_log = EventLog()
     cmd.repodir = self.repodir
     cmd.client = RepoClient(cmd.repodir)
     cmd.manifest = cmd.client.manifest
@@ -261,6 +265,8 @@ class _Repo(object):
     start = time.time()
     cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
     cmd.event_log.SetParent(cmd_event)
+    git_trace2_event_log.StartEvent()
+
     try:
       cmd.ValidateOptions(copts, cargs)
       result = cmd.Execute(copts, cargs)
@@ -303,10 +309,13 @@ class _Repo(object):
 
       cmd.event_log.FinishEvent(cmd_event, finish,
                                 result is None or result == 0)
+      git_trace2_event_log.ExitEvent(result)
+
       if gopts.event_log:
         cmd.event_log.Write(os.path.abspath(
                             os.path.expanduser(gopts.event_log)))
 
+      git_trace2_event_log.Write(gopts.git_trace2_event_log)
     return result
 
 

manifest_xml.py

@@ -292,8 +292,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if r.revision is not None:
       e.setAttribute('revision', r.revision)
 
-  def _ParseGroups(self, groups):
-    return [x for x in re.split(r'[,\s]+', groups) if x]
+  def _ParseList(self, field):
+    """Parse fields that contain flattened lists.
+
+    These are whitespace & comma separated.  Empty elements will be discarded.
+    """
+    return [x for x in re.split(r'[,\s]+', field) if x]
 
   def ToXml(self, peg_rev=False, peg_rev_upstream=True, peg_rev_dest_branch=True, groups=None):
     """Return the current manifest XML."""
@@ -302,7 +306,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if groups is None:
       groups = mp.config.GetString('manifest.groups')
     if groups:
-      groups = self._ParseGroups(groups)
+      groups = self._ParseList(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -754,7 +758,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         path = node.getAttribute('path')
         groups = node.getAttribute('groups')
         if groups:
-          groups = self._ParseGroups(groups)
+          groups = self._ParseList(groups)
         revision = node.getAttribute('revision')
         remote = node.getAttribute('remote')
         if remote:
@@ -776,7 +780,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
-        enabled_repo_hooks = self._reqatt(node, 'enabled-list').split()
+        enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
 
         # Only one project can be the hooks project
         if self._repo_hooks_project is not None:
@@ -989,7 +993,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = self._ParseGroups(groups)
+    groups = self._ParseList(groups)
 
     if parent is None:
       relpath, worktree, gitdir, objdir, use_git_worktrees = \

project.py

@@ -2558,6 +2558,8 @@ class Project(object):
 
         base = R_WORKTREE_M
         active_git = self.work_git
+
+        self._InitAnyMRef(HEAD, self.bare_git, detach=True)
       else:
         base = R_M
         active_git = self.bare_git
@@ -2567,7 +2569,7 @@ class Project(object):
   def _InitMirrorHead(self):
     self._InitAnyMRef(HEAD, self.bare_git)
 
-  def _InitAnyMRef(self, ref, active_git):
+  def _InitAnyMRef(self, ref, active_git, detach=False):
     cur = self.bare_ref.symref(ref)
 
     if self.revisionId:
@@ -2580,7 +2582,10 @@ class Project(object):
       dst = remote.ToLocal(self.revisionExpr)
       if cur != dst:
         msg = 'manifest set to %s' % self.revisionExpr
-        active_git.symbolic_ref('-m', msg, ref, dst)
+        if detach:
+          active_git.UpdateRef(ref, dst, message=msg, detach=True)
+        else:
+          active_git.symbolic_ref('-m', msg, ref, dst)
 
   def _CheckDirReference(self, srcdir, destdir, share_refs):
     # Git worktrees don't use symlinks to share at all.

repo

@@ -32,6 +32,13 @@ import subprocess
 import sys
 
 
+# These should never be newer than the main.py version since this needs to be a
+# bit more flexible with older systems.  See that file for more details on the
+# versions we select.
+MIN_PYTHON_VERSION_SOFT = (3, 6)
+MIN_PYTHON_VERSION_HARD = (3, 5)
+
+
 # Keep basic logic in sync with repo_trace.py.
 class Trace(object):
   """Trace helper logic."""
@@ -70,8 +77,6 @@ def check_python_version():
   def reexec(prog):
     exec_command([prog] + sys.argv)
 
-  MIN_PYTHON_VERSION = (3, 6)
-
   ver = sys.version_info
   major = ver.major
   minor = ver.minor
@@ -80,19 +85,26 @@ def check_python_version():
   if (major, minor) < (2, 7):
     print('repo: error: Your Python version is too old. '
           'Please use Python {}.{} or newer instead.'.format(
-              *MIN_PYTHON_VERSION), file=sys.stderr)
+              *MIN_PYTHON_VERSION_SOFT), file=sys.stderr)
     sys.exit(1)
 
   # Try to re-exec the version specific Python 3 if needed.
-  if (major, minor) < MIN_PYTHON_VERSION:
+  if (major, minor) < MIN_PYTHON_VERSION_SOFT:
     # Python makes releases ~once a year, so try our min version +10 to help
     # bridge the gap.  This is the fallback anyways so perf isn't critical.
-    min_major, min_minor = MIN_PYTHON_VERSION
+    min_major, min_minor = MIN_PYTHON_VERSION_SOFT
     for inc in range(0, 10):
       reexec('python{}.{}'.format(min_major, min_minor + inc))
 
-    # Try the generic Python 3 wrapper, but only if it's new enough.  We don't
-    # want to go from (still supported) Python 2.7 to (unsupported) Python 3.5.
+    # Fallback to older versions if possible.
+    for inc in range(MIN_PYTHON_VERSION_SOFT[1] - MIN_PYTHON_VERSION_HARD[1], 0, -1):
+      # Don't downgrade, and don't reexec ourselves (which would infinite loop).
+      if (min_major, min_minor - inc) <= (major, minor):
+        break
+      reexec('python{}.{}'.format(min_major, min_minor - inc))
+
+    # Try the generic Python 3 wrapper, but only if it's new enough.  If it
+    # isn't, we want to just give up below and make the user resolve things.
     try:
       proc = subprocess.Popen(
           ['python3', '-c', 'import sys; '
@@ -103,18 +115,20 @@ def check_python_version():
     except (OSError, subprocess.CalledProcessError):
       python3_ver = None
 
-    # The python3 version looks like it's new enough, so give it a try.
-    if python3_ver and python3_ver >= MIN_PYTHON_VERSION:
+    # If the python3 version looks like it's new enough, give it a try.
+    if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD
+        and python3_ver != (major, minor)):
       reexec('python3')
 
     # We're still here, so diagnose things for the user.
     if major < 3:
-      print('repo: warning: Python 2 is no longer supported; '
-            'Please upgrade to Python {}.{}+.'.format(*MIN_PYTHON_VERSION),
+      print('repo: error: Python 2 is no longer supported; '
+            'Please upgrade to Python {}.{}+.'.format(*MIN_PYTHON_VERSION_HARD),
             file=sys.stderr)
-    else:
+      sys.exit(1)
+    elif (major, minor) < MIN_PYTHON_VERSION_HARD:
       print('repo: error: Python 3 version is too old; '
-            'Please use Python {}.{} or newer.'.format(*MIN_PYTHON_VERSION),
+            'Please use Python {}.{} or newer.'.format(*MIN_PYTHON_VERSION_HARD),
             file=sys.stderr)
       sys.exit(1)
 
@@ -439,9 +453,11 @@ def get_gitc_manifest_dir():
 def gitc_parse_clientdir(gitc_fs_path):
   """Parse a path in the GITC FS and return its client name.
 
-  @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+  Args:
+    gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
 
-  @returns: The GITC client name
+  Returns:
+    The GITC client name.
   """
   if gitc_fs_path == GITC_FS_ROOT_DIR:
     return None

run_tests

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding:utf-8 -*-
 # Copyright 2019 The Android Open Source Project
 #
@@ -20,22 +20,26 @@ from __future__ import print_function
 
 import errno
 import os
+import shutil
 import subprocess
 import sys
 
 
-def run_pytest(cmd, argv):
-  """Run the unittests via |cmd|."""
-  try:
-    return subprocess.call([cmd] + argv)
-  except OSError as e:
-    if e.errno == errno.ENOENT:
-      print('%s: unable to run `%s`: %s' % (__file__, cmd, e), file=sys.stderr)
-      print('%s: Try installing pytest: sudo apt-get install python-pytest' %
-            (__file__,), file=sys.stderr)
-      return 127
-    else:
-      raise
+def find_pytest():
+  """Try to locate a good version of pytest."""
+  # Use the Python 3 version if available.
+  ret = shutil.which('pytest-3')
+  if ret:
+    return ret
+
+  # Hopefully this is a Python 3 version.
+  ret = shutil.which('pytest')
+  if ret:
+    return ret
+
+  print(f'{__file__}: unable to find pytest.', file=sys.stderr)
+  print(f'{__file__}: Try installing: sudo apt-get install python-pytest',
+        file=sys.stderr)
 
 
 def main(argv):
@@ -48,7 +52,8 @@ def main(argv):
     pythonpath += os.pathsep + oldpythonpath
   os.environ['PYTHONPATH'] = pythonpath
 
-  return run_pytest('pytest', argv)
+  pytest = find_pytest()
+  return subprocess.run([pytest] + argv, check=True)
 
 
 if __name__ == '__main__':

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding:utf-8 -*-
 # Copyright 2019 The Android Open Source Project
 #
@@ -55,9 +55,10 @@ setuptools.setup(
         'Operating System :: MacOS :: MacOS X',
         'Operating System :: Microsoft :: Windows :: Windows 10',
         'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3 :: Only',
         'Topic :: Software Development :: Version Control :: Git',
     ],
-    # We support Python 2.7 and Python 3.6+.
-    python_requires='>=2.7, ' + ', '.join('!=3.%i.*' % x for x in range(0, 6)),
+    python_requires='>=3.6',
     packages=['subcmds'],
 )

subcmds/branches.py

@@ -15,10 +15,20 @@
 # limitations under the License.
 
 from __future__ import print_function
+import itertools
+import multiprocessing
 import sys
 from color import Coloring
 from command import Command
 
+# Number of projects to submit to a single worker process at a time.
+# This number represents a tradeoff between the overhead of IPC and finer
+# grained opportunity for parallelism. This particular value was chosen by
+# iterating through powers of two until the overall performance no longer
+# improved. The performance of this batch size is not a function of the
+# number of cores on the system.
+WORKER_BATCH_SIZE = 32
+
 
 class BranchColoring(Coloring):
   def __init__(self, config):
@@ -97,20 +107,32 @@ is shown, then the branch appears in all projects.
 
 """
 
+  def _Options(self, p):
+    """Add flags to CLI parser for this subcommand."""
+    default_jobs = min(multiprocessing.cpu_count(), 8)
+    p.add_option(
+        '-j',
+        '--jobs',
+        type=int,
+        default=default_jobs,
+        help='Number of worker processes to spawn '
+        '(default: %s)' % default_jobs)
+
   def Execute(self, opt, args):
     projects = self.GetProjects(args)
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
+    with multiprocessing.Pool(processes=opt.jobs) as pool:
+      project_branches = pool.imap_unordered(
+          expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE)
 
-    for project in projects:
-      for name, b in project.GetBranches().items():
-        b.project = project
+      for name, b in itertools.chain.from_iterable(project_branches):
         if name not in all_branches:
           all_branches[name] = BranchInfo(name)
         all_branches[name].add(b)
 
-    names = list(sorted(all_branches))
+    names = sorted(all_branches)
 
     if not names:
       print('   (no branches)', file=sys.stderr)
@@ -180,3 +202,19 @@ is shown, then the branch appears in all projects.
       else:
         out.write(' in all projects')
       out.nl()
+
+
+def expand_project_to_branches(project):
+  """Expands a project into a list of branch names & associated information.
+
+  Args:
+    project: project.Project
+
+  Returns:
+    List[Tuple[str, git_config.Branch]]
+  """
+  branches = []
+  for name, b in project.GetBranches().items():
+    b.project = project
+    branches.append((name, b))
+  return branches

subcmds/init.py

@@ -521,7 +521,7 @@ to update the working directory files.
           rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
-      rp.work_git.update_ref('refs/heads/default', rev)
+      rp.work_git.reset('--hard', rev)
       branch.Save()
 
     if opt.worktree:

tests/test_git_trace2_event_log.py

@@ -0,0 +1,169 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the git_trace2_event_log.py module."""
+
+import json
+import tempfile
+import unittest
+
+import git_trace2_event_log
+
+
+class EventLogTestCase(unittest.TestCase):
+  """TestCase for the EventLog module."""
+
+  PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID'
+  PARENT_SID_VALUE = 'parent_sid'
+  SELF_SID_REGEX = r'repo-\d+T\d+Z-.*'
+  FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX)
+
+  def setUp(self):
+    """Load the event_log module every time."""
+    self._event_log_module = None
+    # By default we initialize with the expected case where
+    # repo launches us (so GIT_TRACE2_PARENT_SID is set).
+    env = {
+        self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
+    }
+    self._event_log_module = git_trace2_event_log.EventLog(env=env)
+    self._log_data = None
+
+  def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
+    """Helper function to verify common event log keys."""
+    self.assertIn('event', log_entry)
+    self.assertIn('sid', log_entry)
+    self.assertIn('thread', log_entry)
+    self.assertIn('time', log_entry)
+
+    # Do basic data format validation.
+    self.assertEqual(expected_event_name, log_entry['event'])
+    if full_sid:
+      self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
+    else:
+      self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX)
+    self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$')
+
+  def readLog(self, log_path):
+    """Helper function to read log data into a list."""
+    log_data = []
+    with open(log_path, mode='rb') as f:
+      for line in f:
+        log_data.append(json.loads(line))
+    return log_data
+
+  def test_initial_state_with_parent_sid(self):
+    """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent."""
+    self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX)
+
+  def test_initial_state_no_parent_sid(self):
+    """Test initial state when 'GIT_TRACE2_PARENT_SID' is not set."""
+    # Setup an empty environment dict (no parent sid).
+    self._event_log_module = git_trace2_event_log.EventLog(env={})
+    self.assertRegex(self._event_log_module.full_sid, self.SELF_SID_REGEX)
+
+  def test_version_event(self):
+    """Test 'version' event data is valid.
+
+    Verify that the 'version' event is written even when no other
+    events are addded.
+
+    Expected event log:
+    <version event>
+    """
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    # A log with no added events should only have the version entry.
+    self.assertEqual(len(self._log_data), 1)
+    version_event = self._log_data[0]
+    self.verifyCommonKeys(version_event, expected_event_name='version')
+    # Check for 'version' event specific fields.
+    self.assertIn('evt', version_event)
+    self.assertIn('exe', version_event)
+
+  def test_start_event(self):
+    """Test and validate 'start' event data is valid.
+
+    Expected event log:
+    <version event>
+    <start event>
+    """
+    self._event_log_module.StartEvent()
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 2)
+    start_event = self._log_data[1]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(start_event, expected_event_name='start')
+    # Check for 'start' event specific fields.
+    self.assertIn('argv', start_event)
+    self.assertTrue(isinstance(start_event['argv'], list))
+
+  def test_exit_event_result_none(self):
+    """Test 'exit' event data is valid when result is None.
+
+    We expect None result to be converted to 0 in the exit event data.
+
+    Expected event log:
+    <version event>
+    <exit event>
+    """
+    self._event_log_module.ExitEvent(None)
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 2)
+    exit_event = self._log_data[1]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(exit_event, expected_event_name='exit')
+    # Check for 'exit' event specific fields.
+    self.assertIn('code', exit_event)
+    # 'None' result should convert to 0 (successful) return code.
+    self.assertEqual(exit_event['code'], 0)
+
+  def test_exit_event_result_integer(self):
+    """Test 'exit' event data is valid when result is an integer.
+
+    Expected event log:
+    <version event>
+    <exit event>
+    """
+    self._event_log_module.ExitEvent(2)
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 2)
+    exit_event = self._log_data[1]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(exit_event, expected_event_name='exit')
+    # Check for 'exit' event specific fields.
+    self.assertIn('code', exit_event)
+    self.assertEqual(exit_event['code'], 2)
+
+  # TODO(https://crbug.com/gerrit/13706): Add additional test coverage for
+  # Write() where:
+  # - path=None (using git config call)
+  # - path=<Non-String type> (raises TypeError)
+  # - path=<Non-Directory> (should return None)
+  # - tempfile.NamedTemporaryFile errors with FileExistsError (should return  None)
+
+
+if __name__ == '__main__':
+  unittest.main()

tests/test_manifest_xml.py

@@ -212,6 +212,19 @@ class XmlManifestTests(unittest.TestCase):
         '<manifest></manifest>')
     self.assertEqual(manifest.ToDict(), {})
 
+  def test_repo_hooks(self):
+    """Check repo-hooks settings."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <project name="repohooks" path="src/repohooks"/>
+  <repo-hooks in-project="repohooks" enabled-list="a, b"/>
+</manifest>
+""")
+    self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
+    self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
+
   def test_project_group(self):
     """Check project group settings."""
     manifest = self.getXmlManifest("""

tests/test_project.py

@@ -26,6 +26,7 @@ import tempfile
 import unittest
 
 import error
+import git_command
 import git_config
 import platform_utils
 import project
@@ -38,7 +39,19 @@ def TempGitTree():
   # Python 2 support entirely.
   try:
     tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    subprocess.check_call(['git', 'init'], cwd=tempdir)
+
+    # Tests need to assume, that main is default branch at init,
+    # which is not supported in config until 2.28.
+    cmd = ['git', 'init']
+    if git_command.git_require((2, 28, 0)):
+      cmd += ['--initial-branch=main']
+    else:
+      # Use template dir for init.
+      templatedir = tempfile.mkdtemp(prefix='.test-template')
+      with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
+        fp.write('ref: refs/heads/main\n')
+      cmd += ['--template=', templatedir]
+    subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
   finally:
     platform_utils.rmtree(tempdir)

tests/test_wrapper.py

@@ -25,6 +25,8 @@ import shutil
 import tempfile
 import unittest
 
+import git_command
+import main
 import platform_utils
 from pyversion import is_python3
 import wrapper
@@ -82,6 +84,16 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
     self.assertEqual('', stderr.getvalue())
     self.assertIn('repo launcher version', stdout.getvalue())
 
+  def test_python_constraints(self):
+    """The launcher should never require newer than main.py."""
+    self.assertGreaterEqual(main.MIN_PYTHON_VERSION_HARD,
+                            wrapper.MIN_PYTHON_VERSION_HARD)
+    self.assertGreaterEqual(main.MIN_PYTHON_VERSION_SOFT,
+                            wrapper.MIN_PYTHON_VERSION_SOFT)
+    # Make sure the versions are themselves in sync.
+    self.assertGreaterEqual(wrapper.MIN_PYTHON_VERSION_SOFT,
+                            wrapper.MIN_PYTHON_VERSION_HARD)
+
   def test_init_parser(self):
     """Make sure 'init' GetParser works."""
     parser = self.wrapper.GetParser(gitc_init=False)
@@ -357,7 +369,19 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
     os.mkdir(remote)
-    run_git('init', cwd=remote)
+
+    # Tests need to assume, that main is default branch at init,
+    # which is not supported in config until 2.28.
+    if git_command.git_require((2, 28, 0)):
+      initstr = '--initial-branch=main'
+    else:
+      # Use template dir for init.
+      templatedir = tempfile.mkdtemp(prefix='.test-template')
+      with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
+        fp.write('ref: refs/heads/main\n')
+      initstr = '--template=' + templatedir
+
+    run_git('init', initstr, cwd=remote)
     run_git('commit', '--allow-empty', '-minit', cwd=remote)
     run_git('branch', 'stable', cwd=remote)
     run_git('tag', 'v1.0', cwd=remote)