sync: Implement fetchcmd for standard Git layouts

Introduce support for `repo.fetchcmd` configuration, allowing users to
specify a custom command to fetch project data instead of
`_RemoteFetch`.

When `repo.fetchcmd` is specified, `repo` will execute it instead of
`_RemoteFetch` during the network half of sync. Note that
`repo.fetchcmd` requires `repo.uselocalgitdirs` to be enabled.

The command is executed in a subshell with project-context environment
variables, including the new `REPO_TREV` (target revision resolved to a
commit hash) and `REPO_PROJECT_FETCH_URL`.

After execution, `repo` verifies that the target commit is available and
that the tracking ref and FETCH_HEAD are correctly updated.

Tested with:
```
> ~/git-repo/repo init -u https://android-review.googlesource.com/platform/manifest \
    --repo-url file:///usr/local/google/home/gavinmak/git-repo \
    --groups developers \
    --no-repo-verify \
    --use-local-gitdirs
...

> git config --file .repo/manifests.git/config repo.fetchcmd 'mkdir -p $REPO_PATH && cd $REPO_PATH && if [ ! -d .git ]; then git init && git remote add aosp $REPO_PROJECT_FETCH_URL; fi && git fetch aosp $REPO_TREV && git reset --hard $REPO_TREV && mkdir -p .git/refs/remotes/aosp && echo $REPO_TREV > .git/refs/remotes/aosp/main && echo $REPO_TREV > .git/FETCH_HEAD'

> ~/git-repo/repo sync -j32
warning: repo is not tracking a remote branch, so it will not receive updates; run `repo init --repo-rev=stable` to fix.
You are currently enrolled in Git submodules experiment (go/android-submodules-quickstart).  Use --no-use-superproject to override.

Syncing: 100% (4/4), done in 1m15.686s
Finalizing sync state...
repo sync has finished successfully.
```

Bug: 513329573
Change-Id: I754d3f3c78e86fdeee1a72115297a75b571bc497
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/583883
Reviewed-by: Becky Siegel <beckysiegel@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>

project.py

@@ -1557,43 +1557,25 @@ class Project:
             depth = None
             clone_filter = clone_filter_for_depth
 
-        # See if we can skip the network fetch entirely.
+        custom_fetch_configured = (
+            self.manifest.manifestProject.use_local_gitdirs
+            and self.manifest.manifestProject.fetch_cmd
+            and not isinstance(self, MetaProject)
+        )
+
         remote_fetched = False
-        has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow"))
-        if not (
-            optimized_fetch
-            and IsId(self.revisionExpr)
-            and self._CheckForImmutableRevision(
-                use_superproject=use_superproject
-            )
-            and (
-                has_shallow
-                or (not depth and not self._SharingProjectHasShallow())
-            )
-        ):
+        if custom_fetch_configured:
+            # The custom fetch command is executed unconditionally on every sync
+            # because it is responsible for updating tracking refs and
+            # FETCH_HEAD.
             remote_fetched = True
             try:
-                if not self._RemoteFetch(
-                    initial=is_new,
-                    quiet=quiet,
-                    verbose=verbose,
-                    output_redir=output_redir,
-                    alt_dir=alt_dir,
-                    use_superproject=use_superproject,
-                    current_branch_only=current_branch_only,
-                    tags=tags,
-                    prune=prune,
-                    depth=depth,
-                    submodules=submodules,
-                    force_sync=force_sync,
-                    ssh_proxy=ssh_proxy,
-                    clone_filter=clone_filter,
-                    retry_fetches=retry_fetches,
-                ):
+                if not self._CustomFetch(verbose=verbose):
                     return SyncNetworkHalfResult(
                         remote_fetched,
                         SyncNetworkHalfError(
-                            f"Unable to remote fetch project {self.name}",
+                            f"Unable to fetch project {self.name} using "
+                            "fetchcmd",
                             project=self.name,
                         ),
                     )
@@ -1602,6 +1584,53 @@ class Project:
                     remote_fetched,
                     e,
                 )
+        else:
+            # See if we can skip the standard network fetch entirely.
+            has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow"))
+            skip_fetch = (
+                optimized_fetch
+                and IsId(self.revisionExpr)
+                and self._CheckForImmutableRevision(
+                    use_superproject=use_superproject
+                )
+                and (
+                    has_shallow
+                    or (not depth and not self._SharingProjectHasShallow())
+                )
+            )
+
+            if not skip_fetch:
+                remote_fetched = True
+                try:
+                    if not self._RemoteFetch(
+                        initial=is_new,
+                        quiet=quiet,
+                        verbose=verbose,
+                        output_redir=output_redir,
+                        alt_dir=alt_dir,
+                        use_superproject=use_superproject,
+                        current_branch_only=current_branch_only,
+                        tags=tags,
+                        prune=prune,
+                        depth=depth,
+                        submodules=submodules,
+                        force_sync=force_sync,
+                        ssh_proxy=ssh_proxy,
+                        clone_filter=clone_filter,
+                        retry_fetches=retry_fetches,
+                    ):
+                        return SyncNetworkHalfResult(
+                            remote_fetched,
+                            SyncNetworkHalfError(
+                                f"Unable to remote fetch project {self.name}",
+                                project=self.name,
+                            ),
+                        )
+                except RepoError as e:
+                    return SyncNetworkHalfResult(
+                        remote_fetched,
+                        e,
+                    )
 
         mp = self.manifest.manifestProject
         dissociate = mp.dissociate
@@ -2710,6 +2739,114 @@ class Project:
         )
         command.Wait()
 
+    def _CustomFetch(self, verbose=False) -> bool:
+        """Fetch using a custom command specified by repo.fetchcmd.
+
+        Populates the subshell with project-context environment variables,
+        including REPO_TREV (target revision resolved to a commit hash).
+
+        Expects the target commit to be reachable and tracking refs/FETCH_HEAD
+        to be updated upon a successful 0 exit code.
+
+        For a detailed contract, environment variables, and postconditions,
+        see docs/fetch-cmd.md.
+        """
+        # Resolve REPO_TREV (target revision resolved to a full commit hash).
+        repo_trev = None
+        if self.revisionId and IsId(self.revisionId):
+            repo_trev = self.revisionId
+
+        if not repo_trev:
+            output = self._LsRemote(self.upstream or self.revisionExpr)
+            if output:
+                lines = output.splitlines()
+                if lines:
+                    parts = lines[0].split()
+                    if parts:
+                        repo_trev = parts[0]
+
+        if not repo_trev:
+            logger.error("error: Cannot resolve REPO_TREV for %s", self.name)
+            return False
+
+        env = os.environ.copy()
+        env.update(self.GetEnvVars())
+        env["REPO_TREV"] = repo_trev
+        cmd_str = self.manifest.manifestProject.fetch_cmd
+
+        if verbose:
+            print(f"Running fetchcmd: {cmd_str} for {self.name}")
+
+        stdout = None if verbose else subprocess.PIPE
+        stderr = None if verbose else subprocess.PIPE
+        try:
+            p = subprocess.run(
+                cmd_str,
+                shell=True,
+                cwd=self.manifest.topdir,
+                env=env,
+                stdout=stdout,
+                stderr=stderr,
+                text=True,
+            )
+
+            if p.returncode != 0:
+                logger.error(
+                    "error: fetchcmd failed with exit code %d", p.returncode
+                )
+                if not verbose:
+                    logger.error("stderr: %s", p.stderr)
+                return False
+
+        except Exception as e:
+            logger.error("error: Failed to execute fetchcmd: %s", e)
+            return False
+
+        # Verify postconditions.
+        try:
+            self.bare_git.cat_file("-e", repo_trev)
+        except GitError:
+            logger.error(
+                "error: Postcondition failed: %s not found in object store",
+                repo_trev,
+            )
+            return False
+
+        try:
+            ref_name = self.GetRemote().ToLocal(self.revisionExpr)
+        except GitError as e:
+            logger.error("error: Failed to resolve tracking ref: %s", e)
+            return False
+        try:
+            resolved_ref = self.bare_git.rev_parse(ref_name)
+            if resolved_ref != repo_trev:
+                logger.error(
+                    "error: Postcondition failed: %s is %s, expected %s",
+                    ref_name,
+                    resolved_ref,
+                    repo_trev,
+                )
+                return False
+        except GitError:
+            logger.error("error: Postcondition failed: %s not found", ref_name)
+            return False
+
+        try:
+            fetch_head = self.bare_git.rev_parse("FETCH_HEAD")
+            if fetch_head != repo_trev:
+                logger.error(
+                    "error: Postcondition failed: FETCH_HEAD is %s, "
+                    "expected %s",
+                    fetch_head,
+                    repo_trev,
+                )
+                return False
+        except GitError:
+            logger.error("error: Postcondition failed: FETCH_HEAD not found")
+            return False
+
+        return True
+
     def _RemoteFetch(
         self,
         name=None,
@@ -4636,6 +4773,11 @@ class ManifestProject(MetaProject):
         """Whether we use local gitdirs."""
         return self.config.GetBoolean("repo.uselocalgitdirs")
 
+    @property
+    def fetch_cmd(self):
+        """The fetch command to use."""
+        return self.config.GetString("repo.fetchcmd")
+
     @property
     def clone_bundle(self):
         """Whether we use clone_bundle."""
@@ -5049,6 +5191,13 @@ class ManifestProject(MetaProject):
 
             self.config.SetBoolean("repo.uselocalgitdirs", use_local_gitdirs)
 
+        if self.fetch_cmd and not (use_local_gitdirs or self.use_local_gitdirs):
+            logger.error(
+                "fatal: repo.fetchcmd is set but repo.uselocalgitdirs is "
+                "not enabled"
+            )
+            return False
+
         if archive:
             if is_new:
                 self.config.SetBoolean("repo.archive", archive)