tests: e2e: add tests for partial clones & the Android setup

Cover the go/repo-init flags used across Android (superproject, `-c`,
`--partial-clone`, `--partial-clone-exclude` & `--clone-filter`)
followed by the go/sync flags, under Git 2 & Git 3 defaults: projects
are pinned to the superproject, filtered blobs stay on the server,
excluded projects are full clones, and incremental syncs skip projects
whose pin didn't move.  Also cover the default filter, and `sync -n`
then `sync -l` with a partial clone (where `-l` has to fetch blobs on
demand).

Add harness helpers to read a checkout's git config, list the repos a
client contacted, and create blobs that only exist in history.

Bug: 566270944
Change-Id: I77d2cabc1d4f5db2ea139c7825ba87638c6db1e7
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/648546
Commit-Queue: Brian Gan <brgan@google.com>
Reviewed-by: Gavin Mak <gavinmak@google.com>
Tested-by: Brian Gan <brgan@google.com>

tests/e2e/README.md

@@ -28,6 +28,8 @@ superproject support), or when not run from a git checkout.
 
 * `harness.py`: the git server, server-side repos & the repo client.
 * `test_init_sync.py`: core `repo init` & `repo sync` options.
+* `test_partial_clone.py`: partial clones, incl. the standard Android
+  configuration from go/repo-init & go/sync.
 * `test_sync_modes.py`: `sync -n`/`-l` & the other sync flags.
 * `test_workflows.py`: multi-step workflows.
 * `test_manifest_regressions.py`: manifests that broke users in the past.

tests/e2e/harness.py

@@ -74,6 +74,14 @@ REPO_SOURCE_BRANCH = "main"
 # the common `fetch=".."` resolves to the root of the host namespace.
 MANIFEST_REPO = "platform/manifest"
 
+# The git endpoints (relative to a repo URL) that clients request.
+REPO_ENDPOINTS = (
+    "info/refs",
+    "git-upload-pack",
+    "git-receive-pack",
+    "clone.bundle",
+)
+
 # Fixed identity used for all commits created by the harness (not by
 # the simulated user).
 HARNESS_IDENTITY = {
@@ -664,6 +672,21 @@ class ServerRepo:
         """Create an annotated tag."""
         self.git("tag", "-a", "-m", name, name, rev)
 
+    def commit_history_only_blob(
+        self, path: str, data: Union[str, bytes]
+    ) -> str:
+        """Commit `data` at `path`, then delete `path` in a second commit.
+
+        Partial clones only download such blobs if the filter allows it.
+
+        Returns:
+            The id of the blob (only reachable from history).
+        """
+        self.commit({path: data}, message=f"add {path}")
+        blob = self.git("rev-parse", f"{self.default_branch}:{path}")
+        self.commit({path: None}, message=f"delete {path}")
+        return blob
+
     def write_bundle(self) -> Path:
         """Write a clone.bundle (all branches & tags) served by the server."""
         bundle = self.path / "clone.bundle"
@@ -781,6 +804,42 @@ class GitHost:
         """All requests received for this namespace."""
         return self.server.requests_for(f"/{self.prefix}/")
 
+    def requests_for(
+        self, name: str, requests: Optional[List[Request]] = None
+    ) -> List[Request]:
+        """The requests for the repo `name` (out of `requests`, or all so far).
+
+        Args:
+            name: The repo name, e.g. `platform/foo`.
+            requests: Requests to filter, e.g. from `watch_requests`.
+        """
+        if requests is None:
+            requests = self.requests()
+        prefix = f"/{self.prefix}/{name}/"
+        return [r for r in requests if r.path.startswith(prefix)]
+
+    def requested_repos(
+        self, requests: Optional[List[Request]] = None
+    ) -> Set[str]:
+        """Return the names of the repos that received requests.
+
+        Args:
+            requests: Requests to inspect (e.g. from `watch_requests`), or all
+                requests received so far if None.
+        """
+        if requests is None:
+            requests = self.requests()
+        namespace = f"/{self.prefix}/"
+        names = set()
+        for request in requests:
+            if not request.path.startswith(namespace):
+                continue
+            for suffix in REPO_ENDPOINTS:
+                if request.path.endswith(f"/{suffix}"):
+                    names.add(request.path[len(namespace) : -len(suffix) - 1])
+                    break
+        return names
+
     @contextlib.contextmanager
     def watch_requests(self) -> Iterator[List[Request]]:
         """Collect the requests this namespace receives inside the block.
@@ -1006,6 +1065,34 @@ class RepoClient:
         """Whether HEAD in `path` is detached (not on a local branch)."""
         return self.git(path, "rev-parse", "--abbrev-ref", "HEAD") == "HEAD"
 
+    def config(self, path: str, key: str) -> Optional[str]:
+        """Return git config `key` of the checkout at `path` (None if unset).
+
+        Only the repo's own config is read (not e.g. the user's), and `path`
+        must be a git checkout.
+        """
+        proc = subprocess.run(
+            ["git", "config", "--local", "--get", key],
+            # See run_git for why this isn't a Path.
+            cwd=str(self.top / path),
+            env=self.env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=False,
+            timeout=COMMAND_TIMEOUT,
+        )
+        # git exits 1 for unset keys; anything else (e.g. 128 outside of a
+        # repo) is a real error.
+        if proc.returncode == 1:
+            return None
+        if proc.returncode:
+            raise AssertionError(
+                f"git config --get {key} failed in {path} "
+                f"({proc.returncode}):\n"
+                f"{proc.stderr.decode('utf-8', 'replace')}"
+            )
+        return proc.stdout.decode("utf-8").rstrip("\r\n")
+
     def remote_branches(self, path: str) -> Set[str]:
         """Return the names of the `origin` remote-tracking branches."""
         out = self.git(

tests/e2e/test_harness.py

@@ -16,13 +16,27 @@
 
 import http.client
 from pathlib import Path
-from typing import Mapping
+from typing import Mapping, Tuple
 import urllib.parse
 
 import harness
 import pytest
 
 
+def _http_request(
+    server: harness.GitHttpServer, method: str, path: str
+) -> Tuple[int, bytes]:
+    """Send a request to `server` & return the response status & body."""
+    url = urllib.parse.urlsplit(server.url)
+    conn = http.client.HTTPConnection(url.hostname, url.port, timeout=30)
+    try:
+        conn.request(method, path)
+        resp = conn.getresponse()
+        return resp.status, resp.read()
+    finally:
+        conn.close()
+
+
 @pytest.mark.parametrize(
     "key,value",
     [
@@ -135,3 +149,35 @@ def test_offline(git_server: harness.GitHttpServer) -> None:
         with host.offline():
             raise RuntimeError("boom")
     assert "refs/heads/main" in harness.run_git("ls-remote", repo.url)
+
+
+def test_requested_repos(git_server: harness.GitHttpServer) -> None:
+    """`requested_repos()` & `watch_requests()` track repo requests."""
+    host = harness.GitHost(git_server, "requested")
+    nested = host.create_repo("platform/nested", {"f": "x"})
+    top = host.create_repo("top", {"f": "x"})
+
+    harness.run_git("ls-remote", nested.url)
+    with host.watch_requests() as watched:
+        harness.run_git("ls-remote", top.url)
+        _http_request(git_server, "GET", f"/{host.prefix}/top/clone.bundle")
+
+    assert host.requested_repos() == {"platform/nested", "top"}
+    assert host.requested_repos(watched) == {"top"}
+    assert host.requests_for("platform/nested", watched) == []
+
+
+def test_client_config(tmp_path: Path) -> None:
+    """`RepoClient.config()` tells unset keys apart from errors."""
+    # Don't let git find a repo above tmp_path (e.g. with --basetemp).
+    env = dict(harness.clean_environ(), GIT_CEILING_DIRECTORIES=str(tmp_path))
+    client = harness.RepoClient(tmp_path / "client", harness.LAUNCHER, env, "")
+    repo = client.top / "repo"
+    harness.run_git("init", "-q", str(repo))
+    harness.run_git("config", "-f", str(repo / ".git/config"), "a.key", "val")
+    (client.top / "not-a-repo").mkdir()
+
+    assert client.config("repo", "a.key") == "val"
+    assert client.config("repo", "a.unset") is None
+    with pytest.raises(AssertionError, match="failed in not-a-repo"):
+        client.config("not-a-repo", "a.key")

tests/e2e/test_partial_clone.py

@@ -0,0 +1,294 @@
+# Copyright (C) 2026 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""E2E tests for partial clones, incl. the standard Android configuration."""
+
+import os
+from typing import Dict, List, NamedTuple, Tuple
+
+from harness import GIT_DEFAULTS_PARAMS
+from harness import GitHost
+from harness import Gitlink
+from harness import manifest_xml
+from harness import RepoClient
+from harness import ServerRepo
+import pytest
+
+
+# The go/repo-init flags.  The filter is 10M there; 10k keeps the test data
+# small without changing the semantics.
+ANDROID_INIT_ARGS = [
+    "-c",
+    "-b",
+    "main",
+    "--use-superproject",
+    "--partial-clone",
+    "--partial-clone-exclude=platform/frameworks/base",
+    "--clone-filter=blob:limit=10k",
+]
+
+# The go/sync flags.
+ANDROID_SYNC_ARGS = ["-c", "-j20", "--no-tags", "--optimized-fetch", "--prune"]
+
+# Larger than the 10k filter above, smaller than git's default big file
+# threshold so it's still delta compressed & packed normally.
+BIG = "x" * 20_000
+
+# The git config key holding a partial clone's filter.
+FILTER_KEY = "remote.origin.partialclonefilter"
+
+
+def create_with_big_history(host: GitHost, name: str) -> Tuple[ServerRepo, str]:
+    """Create `name` with a large blob that only exists in history.
+
+    Returns:
+        The repo and the id of the historical large blob.  The tip commit has
+        a different large blob (`big2.bin`) and a small file.
+    """
+    repo = host.create_repo(name, {"big.bin": BIG, "small": "s"})
+    big_blob = repo.git("rev-parse", "main:big.bin")
+    repo.commit({"big.bin": None, "big2.bin": BIG + "2"})
+    return repo, big_blob
+
+
+class AndroidTree(NamedTuple):
+    """The server side of `create_android_tree`, keyed by project path."""
+
+    repos: Dict[str, ServerRepo]
+    superproject: ServerRepo
+    # The commit the superproject pins each project to.
+    pins: Dict[str, str]
+    # A large blob that only exists in each project's history.
+    blobs: Dict[str, str]
+
+
+def create_android_tree(host: GitHost) -> AndroidTree:
+    """Create a small AOSP-like tree with a superproject.
+
+    `platform/foo` is a regular project and `platform/frameworks/base` is the
+    one go/repo-init excludes from the partial clone.  Both have moved past
+    the superproject's pins.
+    """
+    repos, pins, blobs = {}, {}, {}
+    for path in ("foo", "frameworks/base"):
+        name = f"platform/{path}"
+        repos[path], blobs[path] = create_with_big_history(host, name)
+        pins[path] = repos[path].rev_parse("main")
+    superproject = host.create_repo(
+        "platform/superproject",
+        {path: Gitlink(rev) for path, rev in pins.items()},
+    )
+    for repo in repos.values():
+        repo.commit({"small": "ahead"})
+        repo.commit({"small": "other"}, branch="other")
+    host.create_manifest(
+        manifest_xml(
+            '<superproject name="platform/superproject" />',
+            '<project name="platform/foo" path="foo" />',
+            '<project name="platform/frameworks/base" '
+            'path="frameworks/base" />',
+        )
+    )
+    return AndroidTree(repos, superproject, pins, blobs)
+
+
+def bundle_requests(host: GitHost) -> Dict[str, List[int]]:
+    """Map repo name to the status codes of its `clone.bundle` requests."""
+    result: Dict[str, List[int]] = {}
+    for r in host.requests():
+        if r.path.endswith("/clone.bundle"):
+            name = r.path[len(f"/{host.prefix}/") : -len("/clone.bundle")]
+            result.setdefault(name, []).append(r.status)
+    return result
+
+
+@pytest.mark.parametrize("git_defaults", GIT_DEFAULTS_PARAMS)
+def test_android_init_sync(host: GitHost, client: RepoClient) -> None:
+    """The go/repo-init flag combination followed by `repo sync -c -j32`.
+
+    Covers superproject pinning, `-c`, the clone filter on included projects,
+    no filter on the `--partial-clone-exclude`d project, and that historical
+    large blobs stay on the server while small historical blobs are fetched.
+    """
+    tree = create_android_tree(host)
+    small_history_blob = tree.repos["foo"].commit_history_only_blob(
+        "small_old.txt", b"small history"
+    )
+    new_foo_pin = tree.repos["foo"].rev_parse("main")
+    tree.superproject.commit({"foo": Gitlink(new_foo_pin)})
+    tree.repos["foo"].commit({"small": "unpinned"})
+    pins = dict(tree.pins, foo=new_foo_pin)
+
+    client.init(host.manifest_url, *ANDROID_INIT_ARGS)
+    assert client.sync("-c", "-j32").stderr == ""
+
+    # init persisted the settings later syncs depend on.
+    manifests = ".repo/manifests"
+    assert client.config(manifests, "repo.superproject") == "true"
+    assert client.config(manifests, "repo.partialclone") == "true"
+    assert client.config(manifests, "repo.clonefilter") == "blob:limit=10k"
+    assert (
+        client.config(manifests, "repo.partialcloneexclude")
+        == "platform/frameworks/base"
+    )
+    for path, pinned in pins.items():
+        assert client.head(path) == pinned
+        assert (client.top / path / "big2.bin").read_text() == BIG + "2"
+        assert client.remote_branches(path) == {"main"}
+    override = client.repodir / "exp-superproject/superproject_override.xml"
+    assert override.exists()
+    # The included project is a partial clone (git normalizes the size), so
+    # the large historical blob was never downloaded, while the small
+    # historical blob (under the 10k limit) was downloaded.
+    assert client.config("foo", "extensions.partialclone") == "origin"
+    assert client.config("foo", FILTER_KEY) == "blob:limit=10240"
+    assert not client.has_object("foo", tree.blobs["foo"])
+    assert client.has_object("foo", small_history_blob)
+    # The excluded project is a full clone.
+    assert client.config("frameworks/base", "extensions.partialclone") is None
+    assert client.config("frameworks/base", FILTER_KEY) is None
+    assert client.has_object("frameworks/base", tree.blobs["frameworks/base"])
+    # Partial clones disable clone bundles, but repo still requests one for
+    # excluded projects (this server has none).
+    assert bundle_requests(host) == {"platform/frameworks/base": [404]}
+
+
+@pytest.mark.parametrize("git_defaults", GIT_DEFAULTS_PARAMS)
+def test_android_sync_flags(host: GitHost, client: RepoClient) -> None:
+    """The go/sync flags on an initial & an incremental sync.
+
+    `-c -jN --no-tags --optimized-fetch --prune` on top of the go/repo-init
+    configuration.  The incremental sync must fetch a project whose pin moved
+    and skip one whose pin didn't.
+    """
+    tree = create_android_tree(host)
+    tree.repos["foo"].tag("v1", tree.pins["foo"])
+
+    client.init(host.manifest_url, *ANDROID_INIT_ARGS)
+    assert client.sync(*ANDROID_SYNC_ARGS).stderr == ""
+
+    for path, pinned in tree.pins.items():
+        assert client.head(path) == pinned
+        assert client.remote_branches(path) == {"main"}
+        assert client.tags(path) == set()
+
+    # Move foo's pin forward (on main) & tag it; frameworks/base keeps its pin.
+    new_pin = tree.repos["foo"].commit({"small": "new pin"})
+    tree.repos["foo"].tag("v2", new_pin)
+    tree.superproject.commit({"foo": Gitlink(new_pin)})
+    with host.watch_requests() as requests:
+        assert client.sync(*ANDROID_SYNC_ARGS).stderr == ""
+
+    assert client.head("foo") == new_pin
+    assert client.head("frameworks/base") == tree.pins["frameworks/base"]
+    assert client.tags("foo") == set()
+    assert (client.top / "foo/small").read_text() == "new pin"
+    # A project already at its (immutable) pin isn't fetched.
+    assert host.requested_repos(requests) == {
+        "platform/manifest",
+        "platform/superproject",
+        "platform/foo",
+    }
+
+
+def test_partial_clone_default_filter(
+    host: GitHost, client: RepoClient
+) -> None:
+    """`--partial-clone` alone uses `blob:none` & disables clone bundles."""
+    foo, big_blob = create_with_big_history(host, "platform/foo")
+    small_history = foo.commit_history_only_blob("old_small.txt", b"tiny")
+    small_tip = foo.git("rev-parse", "main:small")
+    foo.write_bundle()
+    host.create_manifest(manifest_xml('<project name="platform/foo" />'))
+
+    client.init(host.manifest_url, "--partial-clone")
+    client.sync()
+
+    path = "platform/foo"
+    assert client.config(".repo/manifests", "repo.partialclone") == "true"
+    assert client.config(".repo/manifests", "repo.clonefilter") == "blob:none"
+    assert client.config(".repo/manifests", FILTER_KEY) == "blob:none"
+    assert client.config(path, FILTER_KEY) == "blob:none"
+    assert client.config(path, "extensions.partialclone") == "origin"
+    # Blobs of the checked out tree were fetched on demand; history wasn't
+    # (even tiny blobs).
+    assert (client.top / path / "big2.bin").read_text() == BIG + "2"
+    assert client.has_object(path, small_tip)
+    assert not client.has_object(path, big_blob)
+    assert not client.has_object(path, small_history)
+    assert bundle_requests(host) == {}
+
+
+def test_partial_clone_two_phase_sync(
+    host: GitHost, client: RepoClient
+) -> None:
+    """`sync -n` then `sync -l` with a partial clone (initial & incremental).
+
+    The network phase fetches no blobs, so the local phase has to fetch the
+    checked out blobs on demand: `-l` isn't offline for partial clones.  When
+    the server is unreachable, `sync -l` fails cleanly & can be retried.
+    """
+    foo, big_blob = create_with_big_history(host, "platform/foo")
+    small_blob = foo.git("rev-parse", "main:small")
+    host.create_manifest(manifest_xml('<project name="platform/foo" />'))
+    client.init(host.manifest_url, "--partial-clone")
+
+    assert client.sync("-n").stderr == ""
+    path = "platform/foo"
+    assert not (client.top / path).exists()
+    gitdir = f"--git-dir={client.repodir / 'projects' / path}.git"
+    assert client.git(
+        ".", gitdir, "rev-parse", "refs/remotes/origin/main"
+    ) == foo.rev_parse("main")
+    objects = client.git(
+        ".",
+        gitdir,
+        "cat-file",
+        "--batch-check=%(objectname)",
+        "--batch-all-objects",
+    ).split()
+    assert foo.rev_parse("main") in objects
+    assert small_blob not in objects
+
+    # When the server is offline, `sync -l` fails cleanly without leaving a
+    # broken worktree behind.
+    with host.offline():
+        failed = client.sync("-l", check=False)
+    assert failed.returncode != 0
+    assert "Cannot initialize work tree for platform/foo" in failed.stderr
+    assert not os.path.lexists(client.top / path / ".git")
+
+    with host.watch_requests() as requests:
+        client.sync("-l")
+
+    assert client.head(path) == foo.rev_parse("main")
+    assert (client.top / path / "big2.bin").read_text() == BIG + "2"
+    assert client.has_object(path, small_blob)
+    assert not client.has_object(path, big_blob)
+    # The on-demand blob fetches are the only traffic of the local sync.
+    assert host.requested_repos(requests) == {"platform/foo"}
+
+    # Incremental `-n` then `-l` on the existing partial-clone checkout.
+    v2 = foo.commit({"small": "s2"})
+    s2_blob = foo.git("rev-parse", "main:small")
+    client.sync("-n")
+    assert (client.top / path / "small").read_text() == "s"
+    assert not client.has_object(path, s2_blob)
+
+    with host.watch_requests() as inc_requests:
+        client.sync("-l")
+    assert client.head(path) == v2
+    assert (client.top / path / "small").read_text() == "s2"
+    assert client.has_object(path, s2_blob)
+    assert host.requested_repos(inc_requests) == {"platform/foo"}