mirrors/git-repo
1
0
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2026-07-15 21:27:00 +00:00
Code Issues Projects Releases Wiki Activity

sync: Implement fetchcmd for standard Git layouts

Browse Source 
Introduce support for `repo.fetchcmd` configuration, allowing users to
specify a custom command to fetch project data instead of
`_RemoteFetch`.

When `repo.fetchcmd` is specified, `repo` will execute it instead of
`_RemoteFetch` during the network half of sync. Note that
`repo.fetchcmd` requires `repo.uselocalgitdirs` to be enabled.

The command is executed in a subshell with project-context environment
variables, including the new `REPO_TREV` (target revision resolved to a
commit hash) and `REPO_PROJECT_FETCH_URL`.

After execution, `repo` verifies that the target commit is available and
that the tracking ref and FETCH_HEAD are correctly updated.

Tested with:
```
> ~/git-repo/repo init -u https://android-review.googlesource.com/platform/manifest \
    --repo-url file:///usr/local/google/home/gavinmak/git-repo \
    --groups developers \
    --no-repo-verify \
    --use-local-gitdirs
...

> git config --file .repo/manifests.git/config repo.fetchcmd 'mkdir -p $REPO_PATH && cd $REPO_PATH && if [ ! -d .git ]; then git init && git remote add aosp $REPO_PROJECT_FETCH_URL; fi && git fetch aosp $REPO_TREV && git reset --hard $REPO_TREV && mkdir -p .git/refs/remotes/aosp && echo $REPO_TREV > .git/refs/remotes/aosp/main && echo $REPO_TREV > .git/FETCH_HEAD'

> ~/git-repo/repo sync -j32
warning: repo is not tracking a remote branch, so it will not receive updates; run `repo init --repo-rev=stable` to fix.
You are currently enrolled in Git submodules experiment (go/android-submodules-quickstart).  Use --no-use-superproject to override.

Syncing: 100% (4/4), done in 1m15.686s
Finalizing sync state...
repo sync has finished successfully.
```

Bug: 513329573
Change-Id: I754d3f3c78e86fdeee1a72115297a75b571bc497
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/583883
Reviewed-by: Becky Siegel <beckysiegel@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
This commit is contained in:
Gavin Mak
2026-06-15 23:23:04 +00:00
committed by gerrit-scoped@luci-project-accounts.iam.gserviceaccount.com
parent e7cac4bca6
commit ead4b2d7aa
3 changed files with 311 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/fetch-cmd.md
+63
View File
@@ -0,0 +1,63 @@
# Fetch Command Contract
The `repo.fetchcmd` configuration allows specifying a custom command to be
executed during `repo sync` to fetch objects, instead of using standard
`git fetch`. This is particularly useful in environments with virtualized
filesystems or lazy checkouts where fetching metadata and downloading file
contents should be decoupled.
## Configuration
To use this feature, set the following in `.repo/manifests.git/config`:
```ini
[repo]
fetchcmd = "your custom command here"
uselocalgitdirs = true
```
Setting `repo.fetchcmd` **requires** `repo.uselocalgitdirs` to be set to `true`.
## Environment Variables
The custom command is executed in a subshell populated with standard
project-context environment variables. For details on standard variables (such
as `REPO_PROJECT`, `REPO_PATH`, `REPO_PROJECT_FETCH_URL`, etc.), see the
Environment section in `repo help forall` or `subcmds/forall.py`.
The following environment variable is specific to `repo.fetchcmd`:
* `REPO_TREV`: The target revision resolved to a full commit hash.
## Contract
### Postconditions on exit 0
After the fetch command exits with status 0, `repo` expects the following
postconditions to be met:
1. `git cat-file -e REPO_TREV` succeeds (the commit must exist in the object
store).
2. The mapped local tracking ref (e.g. `refs/remotes/REPO_REMOTE/<branch>`
for a branch revision, or the tag ref itself for a tag) must point to
`REPO_TREV`.
3. `FETCH_HEAD` must point to `REPO_TREV`.
4. The commit graph from `REPO_TREV` must be reachable far enough to compute
merge bases with local branches.
### Invariants
* The command should be idempotent; fetching the same `REPO_TREV` twice should
be a no-op.
* Only `FETCH_HEAD` and `refs/remotes/*` should be modified to preserve
`repo sync --network-only` semantics. `HEAD` and local branches must not be
touched by the fetch command.
* Dirty worktree state must be preserved.
* The command is **not** executed for `MetaProject`s (i.e. the internal `repo`
repository itself at `.repo/repo` and the `manifests` repository at
`.repo/manifests`).
### Failure
* A non-zero exit status aborts the project's sync, and the command's stderr
is surfaced to the user.
* `repo` verifies the tracking ref and target reachability after exit 0. Any
mismatch is treated as a failure.
project.py
+180 -31
View File
@@ -1557,43 +1557,25 @@ class Project:
depth = None
clone_filter = clone_filter_for_depth
# See if we can skip the network fetch entirely.
custom_fetch_configured = (
self.manifest.manifestProject.use_local_gitdirs
and self.manifest.manifestProject.fetch_cmd
and not isinstance(self, MetaProject)
)
remote_fetched = False
has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow"))
if not (
optimized_fetch
and IsId(self.revisionExpr)
and self._CheckForImmutableRevision(
use_superproject=use_superproject
)
and (
has_shallow
or (not depth and not self._SharingProjectHasShallow())
)
):
if custom_fetch_configured:
# The custom fetch command is executed unconditionally on every sync
# because it is responsible for updating tracking refs and
# FETCH_HEAD.
remote_fetched = True
try:
if not self._RemoteFetch(
initial=is_new,
quiet=quiet,
verbose=verbose,
output_redir=output_redir,
alt_dir=alt_dir,
use_superproject=use_superproject,
current_branch_only=current_branch_only,
tags=tags,
prune=prune,
depth=depth,
submodules=submodules,
force_sync=force_sync,
ssh_proxy=ssh_proxy,
clone_filter=clone_filter,
retry_fetches=retry_fetches,
):
if not self._CustomFetch(verbose=verbose):
return SyncNetworkHalfResult(
remote_fetched,
SyncNetworkHalfError(
f"Unable to remote fetch project {self.name}",
f"Unable to fetch project {self.name} using "
"fetchcmd",
project=self.name,
),
)
@@ -1602,6 +1584,53 @@ class Project:
remote_fetched,
e,
)
else:
# See if we can skip the standard network fetch entirely.
has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow"))
skip_fetch = (
optimized_fetch
and IsId(self.revisionExpr)
and self._CheckForImmutableRevision(
use_superproject=use_superproject
)
and (
has_shallow
or (not depth and not self._SharingProjectHasShallow())
)
)
if not skip_fetch:
remote_fetched = True
try:
if not self._RemoteFetch(
initial=is_new,
quiet=quiet,
verbose=verbose,
output_redir=output_redir,
alt_dir=alt_dir,
use_superproject=use_superproject,
current_branch_only=current_branch_only,
tags=tags,
prune=prune,
depth=depth,
submodules=submodules,
force_sync=force_sync,
ssh_proxy=ssh_proxy,
clone_filter=clone_filter,
retry_fetches=retry_fetches,
):
return SyncNetworkHalfResult(
remote_fetched,
SyncNetworkHalfError(
f"Unable to remote fetch project {self.name}",
project=self.name,
),
)
except RepoError as e:
return SyncNetworkHalfResult(
remote_fetched,
e,
)
mp = self.manifest.manifestProject
dissociate = mp.dissociate
@@ -2710,6 +2739,114 @@ class Project:
)
command.Wait()
def _CustomFetch(self, verbose=False) -> bool:
"""Fetch using a custom command specified by repo.fetchcmd.
Populates the subshell with project-context environment variables,
including REPO_TREV (target revision resolved to a commit hash).
Expects the target commit to be reachable and tracking refs/FETCH_HEAD
to be updated upon a successful 0 exit code.
For a detailed contract, environment variables, and postconditions,
see docs/fetch-cmd.md.
"""
# Resolve REPO_TREV (target revision resolved to a full commit hash).
repo_trev = None
if self.revisionId and IsId(self.revisionId):
repo_trev = self.revisionId
if not repo_trev:
output = self._LsRemote(self.upstream or self.revisionExpr)
if output:
lines = output.splitlines()
if lines:
parts = lines[0].split()
if parts:
repo_trev = parts[0]
if not repo_trev:
logger.error("error: Cannot resolve REPO_TREV for %s", self.name)
return False
env = os.environ.copy()
env.update(self.GetEnvVars())
env["REPO_TREV"] = repo_trev
cmd_str = self.manifest.manifestProject.fetch_cmd
if verbose:
print(f"Running fetchcmd: {cmd_str} for {self.name}")
stdout = None if verbose else subprocess.PIPE
stderr = None if verbose else subprocess.PIPE
try:
p = subprocess.run(
cmd_str,
shell=True,
cwd=self.manifest.topdir,
env=env,
stdout=stdout,
stderr=stderr,
text=True,
)
if p.returncode != 0:
logger.error(
"error: fetchcmd failed with exit code %d", p.returncode
)
if not verbose:
logger.error("stderr: %s", p.stderr)
return False
except Exception as e:
logger.error("error: Failed to execute fetchcmd: %s", e)
return False
# Verify postconditions.
try:
self.bare_git.cat_file("-e", repo_trev)
except GitError:
logger.error(
"error: Postcondition failed: %s not found in object store",
repo_trev,
)
return False
try:
ref_name = self.GetRemote().ToLocal(self.revisionExpr)
except GitError as e:
logger.error("error: Failed to resolve tracking ref: %s", e)
return False
try:
resolved_ref = self.bare_git.rev_parse(ref_name)
if resolved_ref != repo_trev:
logger.error(
"error: Postcondition failed: %s is %s, expected %s",
ref_name,
resolved_ref,
repo_trev,
)
return False
except GitError:
logger.error("error: Postcondition failed: %s not found", ref_name)
return False
try:
fetch_head = self.bare_git.rev_parse("FETCH_HEAD")
if fetch_head != repo_trev:
logger.error(
"error: Postcondition failed: FETCH_HEAD is %s, "
"expected %s",
fetch_head,
repo_trev,
)
return False
except GitError:
logger.error("error: Postcondition failed: FETCH_HEAD not found")
return False
return True
def _RemoteFetch(
self,
name=None,
@@ -4636,6 +4773,11 @@ class ManifestProject(MetaProject):
"""Whether we use local gitdirs."""
return self.config.GetBoolean("repo.uselocalgitdirs")
@property
def fetch_cmd(self):
"""The fetch command to use."""
return self.config.GetString("repo.fetchcmd")
@property
def clone_bundle(self):
"""Whether we use clone_bundle."""
@@ -5049,6 +5191,13 @@ class ManifestProject(MetaProject):
self.config.SetBoolean("repo.uselocalgitdirs", use_local_gitdirs)
if self.fetch_cmd and not (use_local_gitdirs or self.use_local_gitdirs):
logger.error(
"fatal: repo.fetchcmd is set but repo.uselocalgitdirs is "
"not enabled"
)
return False
if archive:
if is_new:
self.config.SetBoolean("repo.archive", archive)
tests/test_project.py
+68
View File
@@ -1286,3 +1286,71 @@ class GetEnvVarsTests(unittest.TestCase):
env = proj.GetEnvVars()
self.assertEqual(env["REPO_LREV"], "")
class FetchCmdTests(unittest.TestCase):
"""Tests for fetch_cmd feature."""
def setUpManifest(self, tempdir):
repodir = os.path.join(tempdir, ".repo")
manifest_dir = os.path.join(repodir, "manifests")
manifest_file = os.path.join(repodir, manifest_xml.MANIFEST_FILE_NAME)
os.mkdir(repodir)
os.mkdir(manifest_dir)
manifest = manifest_xml.XmlManifest(repodir, manifest_file)
return project.ManifestProject(
manifest, "test/manifest", os.path.join(tempdir, ".git"), tempdir
)
def _get_project(self, tempdir):
proj = _create_mock_project(
tempdir, use_local_gitdirs=True, fetch_cmd="echo hi"
)
proj.GetRevisionId = mock.MagicMock(return_value="1234abcd")
return proj
def test_fetch_cmd_execution(self):
"""Test that fetch_cmd is executed with correct environment."""
with utils_for_test.TempGitTree() as tempdir:
proj = self._get_project(tempdir)
proj.bare_git.rev_parse.return_value = "1234abcd"
mock_remote = mock.MagicMock()
mock_remote.ToLocal.return_value = "refs/remotes/origin/main"
proj.GetRemote = mock.MagicMock(return_value=mock_remote)
with mock.patch("subprocess.run") as mock_run:
mock_run.return_value = mock.MagicMock(returncode=0, stderr="")
res = proj._CustomFetch()
self.assertTrue(res)
mock_run.assert_called_once()
args, kwargs = mock_run.call_args
self.assertEqual(args[0], "echo hi")
self.assertEqual(kwargs["shell"], True)
self.assertEqual(kwargs["cwd"], tempdir)
self.assertEqual(kwargs["env"]["REPO_TREV"], "1234abcd")
self.assertEqual(
kwargs["env"]["REPO_PROJECT_FETCH_URL"],
"http://example.com/repo",
)
def test_sync_fetch_cmd_requires_use_local_gitdirs(self):
"""Test that fetch_cmd requires use_local_gitdirs."""
with utils_for_test.TempGitTree() as tempdir:
fakeproj = self.setUpManifest(tempdir)
class DummyManifest:
is_submanifest = False
def GetDefaultGroupsStr(self, with_platform=False):
return ""
fakeproj.manifest = DummyManifest()
fakeproj.config.SetString("repo.fetchcmd", "echo hi")
fakeproj.config.SetBoolean("repo.uselocalgitdirs", False)
result = fakeproj.Sync(use_local_gitdirs=False)
self.assertFalse(result)