diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0f53d2c2c..1fd962797 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -28,6 +28,8 @@ superproject support), or when not run from a git checkout. * `harness.py`: the git server, server-side repos & the repo client. * `test_init_sync.py`: core `repo init` & `repo sync` options. +* `test_partial_clone.py`: partial clones, incl. the standard Android + configuration from go/repo-init & go/sync. * `test_sync_modes.py`: `sync -n`/`-l` & the other sync flags. * `test_workflows.py`: multi-step workflows. * `test_manifest_regressions.py`: manifests that broke users in the past. diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 07252ad0d..73cfc92fd 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -74,6 +74,14 @@ REPO_SOURCE_BRANCH = "main" # the common `fetch=".."` resolves to the root of the host namespace. MANIFEST_REPO = "platform/manifest" +# The git endpoints (relative to a repo URL) that clients request. +REPO_ENDPOINTS = ( + "info/refs", + "git-upload-pack", + "git-receive-pack", + "clone.bundle", +) + # Fixed identity used for all commits created by the harness (not by # the simulated user). HARNESS_IDENTITY = { @@ -664,6 +672,21 @@ class ServerRepo: """Create an annotated tag.""" self.git("tag", "-a", "-m", name, name, rev) + def commit_history_only_blob( + self, path: str, data: Union[str, bytes] + ) -> str: + """Commit `data` at `path`, then delete `path` in a second commit. + + Partial clones only download such blobs if the filter allows it. + + Returns: + The id of the blob (only reachable from history). + """ + self.commit({path: data}, message=f"add {path}") + blob = self.git("rev-parse", f"{self.default_branch}:{path}") + self.commit({path: None}, message=f"delete {path}") + return blob + def write_bundle(self) -> Path: """Write a clone.bundle (all branches & tags) served by the server.""" bundle = self.path / "clone.bundle" @@ -781,6 +804,42 @@ class GitHost: """All requests received for this namespace.""" return self.server.requests_for(f"/{self.prefix}/") + def requests_for( + self, name: str, requests: Optional[List[Request]] = None + ) -> List[Request]: + """The requests for the repo `name` (out of `requests`, or all so far). + + Args: + name: The repo name, e.g. `platform/foo`. + requests: Requests to filter, e.g. from `watch_requests`. + """ + if requests is None: + requests = self.requests() + prefix = f"/{self.prefix}/{name}/" + return [r for r in requests if r.path.startswith(prefix)] + + def requested_repos( + self, requests: Optional[List[Request]] = None + ) -> Set[str]: + """Return the names of the repos that received requests. + + Args: + requests: Requests to inspect (e.g. from `watch_requests`), or all + requests received so far if None. + """ + if requests is None: + requests = self.requests() + namespace = f"/{self.prefix}/" + names = set() + for request in requests: + if not request.path.startswith(namespace): + continue + for suffix in REPO_ENDPOINTS: + if request.path.endswith(f"/{suffix}"): + names.add(request.path[len(namespace) : -len(suffix) - 1]) + break + return names + @contextlib.contextmanager def watch_requests(self) -> Iterator[List[Request]]: """Collect the requests this namespace receives inside the block. @@ -1006,6 +1065,34 @@ class RepoClient: """Whether HEAD in `path` is detached (not on a local branch).""" return self.git(path, "rev-parse", "--abbrev-ref", "HEAD") == "HEAD" + def config(self, path: str, key: str) -> Optional[str]: + """Return git config `key` of the checkout at `path` (None if unset). + + Only the repo's own config is read (not e.g. the user's), and `path` + must be a git checkout. + """ + proc = subprocess.run( + ["git", "config", "--local", "--get", key], + # See run_git for why this isn't a Path. + cwd=str(self.top / path), + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=COMMAND_TIMEOUT, + ) + # git exits 1 for unset keys; anything else (e.g. 128 outside of a + # repo) is a real error. + if proc.returncode == 1: + return None + if proc.returncode: + raise AssertionError( + f"git config --get {key} failed in {path} " + f"({proc.returncode}):\n" + f"{proc.stderr.decode('utf-8', 'replace')}" + ) + return proc.stdout.decode("utf-8").rstrip("\r\n") + def remote_branches(self, path: str) -> Set[str]: """Return the names of the `origin` remote-tracking branches.""" out = self.git( diff --git a/tests/e2e/test_harness.py b/tests/e2e/test_harness.py index b25dc6763..d4da31cb9 100644 --- a/tests/e2e/test_harness.py +++ b/tests/e2e/test_harness.py @@ -16,13 +16,27 @@ import http.client from pathlib import Path -from typing import Mapping +from typing import Mapping, Tuple import urllib.parse import harness import pytest +def _http_request( + server: harness.GitHttpServer, method: str, path: str +) -> Tuple[int, bytes]: + """Send a request to `server` & return the response status & body.""" + url = urllib.parse.urlsplit(server.url) + conn = http.client.HTTPConnection(url.hostname, url.port, timeout=30) + try: + conn.request(method, path) + resp = conn.getresponse() + return resp.status, resp.read() + finally: + conn.close() + + @pytest.mark.parametrize( "key,value", [ @@ -135,3 +149,35 @@ def test_offline(git_server: harness.GitHttpServer) -> None: with host.offline(): raise RuntimeError("boom") assert "refs/heads/main" in harness.run_git("ls-remote", repo.url) + + +def test_requested_repos(git_server: harness.GitHttpServer) -> None: + """`requested_repos()` & `watch_requests()` track repo requests.""" + host = harness.GitHost(git_server, "requested") + nested = host.create_repo("platform/nested", {"f": "x"}) + top = host.create_repo("top", {"f": "x"}) + + harness.run_git("ls-remote", nested.url) + with host.watch_requests() as watched: + harness.run_git("ls-remote", top.url) + _http_request(git_server, "GET", f"/{host.prefix}/top/clone.bundle") + + assert host.requested_repos() == {"platform/nested", "top"} + assert host.requested_repos(watched) == {"top"} + assert host.requests_for("platform/nested", watched) == [] + + +def test_client_config(tmp_path: Path) -> None: + """`RepoClient.config()` tells unset keys apart from errors.""" + # Don't let git find a repo above tmp_path (e.g. with --basetemp). + env = dict(harness.clean_environ(), GIT_CEILING_DIRECTORIES=str(tmp_path)) + client = harness.RepoClient(tmp_path / "client", harness.LAUNCHER, env, "") + repo = client.top / "repo" + harness.run_git("init", "-q", str(repo)) + harness.run_git("config", "-f", str(repo / ".git/config"), "a.key", "val") + (client.top / "not-a-repo").mkdir() + + assert client.config("repo", "a.key") == "val" + assert client.config("repo", "a.unset") is None + with pytest.raises(AssertionError, match="failed in not-a-repo"): + client.config("not-a-repo", "a.key") diff --git a/tests/e2e/test_partial_clone.py b/tests/e2e/test_partial_clone.py new file mode 100644 index 000000000..fe6204007 --- /dev/null +++ b/tests/e2e/test_partial_clone.py @@ -0,0 +1,294 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""E2E tests for partial clones, incl. the standard Android configuration.""" + +import os +from typing import Dict, List, NamedTuple, Tuple + +from harness import GIT_DEFAULTS_PARAMS +from harness import GitHost +from harness import Gitlink +from harness import manifest_xml +from harness import RepoClient +from harness import ServerRepo +import pytest + + +# The go/repo-init flags. The filter is 10M there; 10k keeps the test data +# small without changing the semantics. +ANDROID_INIT_ARGS = [ + "-c", + "-b", + "main", + "--use-superproject", + "--partial-clone", + "--partial-clone-exclude=platform/frameworks/base", + "--clone-filter=blob:limit=10k", +] + +# The go/sync flags. +ANDROID_SYNC_ARGS = ["-c", "-j20", "--no-tags", "--optimized-fetch", "--prune"] + +# Larger than the 10k filter above, smaller than git's default big file +# threshold so it's still delta compressed & packed normally. +BIG = "x" * 20_000 + +# The git config key holding a partial clone's filter. +FILTER_KEY = "remote.origin.partialclonefilter" + + +def create_with_big_history(host: GitHost, name: str) -> Tuple[ServerRepo, str]: + """Create `name` with a large blob that only exists in history. + + Returns: + The repo and the id of the historical large blob. The tip commit has + a different large blob (`big2.bin`) and a small file. + """ + repo = host.create_repo(name, {"big.bin": BIG, "small": "s"}) + big_blob = repo.git("rev-parse", "main:big.bin") + repo.commit({"big.bin": None, "big2.bin": BIG + "2"}) + return repo, big_blob + + +class AndroidTree(NamedTuple): + """The server side of `create_android_tree`, keyed by project path.""" + + repos: Dict[str, ServerRepo] + superproject: ServerRepo + # The commit the superproject pins each project to. + pins: Dict[str, str] + # A large blob that only exists in each project's history. + blobs: Dict[str, str] + + +def create_android_tree(host: GitHost) -> AndroidTree: + """Create a small AOSP-like tree with a superproject. + + `platform/foo` is a regular project and `platform/frameworks/base` is the + one go/repo-init excludes from the partial clone. Both have moved past + the superproject's pins. + """ + repos, pins, blobs = {}, {}, {} + for path in ("foo", "frameworks/base"): + name = f"platform/{path}" + repos[path], blobs[path] = create_with_big_history(host, name) + pins[path] = repos[path].rev_parse("main") + superproject = host.create_repo( + "platform/superproject", + {path: Gitlink(rev) for path, rev in pins.items()}, + ) + for repo in repos.values(): + repo.commit({"small": "ahead"}) + repo.commit({"small": "other"}, branch="other") + host.create_manifest( + manifest_xml( + '', + '', + '', + ) + ) + return AndroidTree(repos, superproject, pins, blobs) + + +def bundle_requests(host: GitHost) -> Dict[str, List[int]]: + """Map repo name to the status codes of its `clone.bundle` requests.""" + result: Dict[str, List[int]] = {} + for r in host.requests(): + if r.path.endswith("/clone.bundle"): + name = r.path[len(f"/{host.prefix}/") : -len("/clone.bundle")] + result.setdefault(name, []).append(r.status) + return result + + +@pytest.mark.parametrize("git_defaults", GIT_DEFAULTS_PARAMS) +def test_android_init_sync(host: GitHost, client: RepoClient) -> None: + """The go/repo-init flag combination followed by `repo sync -c -j32`. + + Covers superproject pinning, `-c`, the clone filter on included projects, + no filter on the `--partial-clone-exclude`d project, and that historical + large blobs stay on the server while small historical blobs are fetched. + """ + tree = create_android_tree(host) + small_history_blob = tree.repos["foo"].commit_history_only_blob( + "small_old.txt", b"small history" + ) + new_foo_pin = tree.repos["foo"].rev_parse("main") + tree.superproject.commit({"foo": Gitlink(new_foo_pin)}) + tree.repos["foo"].commit({"small": "unpinned"}) + pins = dict(tree.pins, foo=new_foo_pin) + + client.init(host.manifest_url, *ANDROID_INIT_ARGS) + assert client.sync("-c", "-j32").stderr == "" + + # init persisted the settings later syncs depend on. + manifests = ".repo/manifests" + assert client.config(manifests, "repo.superproject") == "true" + assert client.config(manifests, "repo.partialclone") == "true" + assert client.config(manifests, "repo.clonefilter") == "blob:limit=10k" + assert ( + client.config(manifests, "repo.partialcloneexclude") + == "platform/frameworks/base" + ) + for path, pinned in pins.items(): + assert client.head(path) == pinned + assert (client.top / path / "big2.bin").read_text() == BIG + "2" + assert client.remote_branches(path) == {"main"} + override = client.repodir / "exp-superproject/superproject_override.xml" + assert override.exists() + # The included project is a partial clone (git normalizes the size), so + # the large historical blob was never downloaded, while the small + # historical blob (under the 10k limit) was downloaded. + assert client.config("foo", "extensions.partialclone") == "origin" + assert client.config("foo", FILTER_KEY) == "blob:limit=10240" + assert not client.has_object("foo", tree.blobs["foo"]) + assert client.has_object("foo", small_history_blob) + # The excluded project is a full clone. + assert client.config("frameworks/base", "extensions.partialclone") is None + assert client.config("frameworks/base", FILTER_KEY) is None + assert client.has_object("frameworks/base", tree.blobs["frameworks/base"]) + # Partial clones disable clone bundles, but repo still requests one for + # excluded projects (this server has none). + assert bundle_requests(host) == {"platform/frameworks/base": [404]} + + +@pytest.mark.parametrize("git_defaults", GIT_DEFAULTS_PARAMS) +def test_android_sync_flags(host: GitHost, client: RepoClient) -> None: + """The go/sync flags on an initial & an incremental sync. + + `-c -jN --no-tags --optimized-fetch --prune` on top of the go/repo-init + configuration. The incremental sync must fetch a project whose pin moved + and skip one whose pin didn't. + """ + tree = create_android_tree(host) + tree.repos["foo"].tag("v1", tree.pins["foo"]) + + client.init(host.manifest_url, *ANDROID_INIT_ARGS) + assert client.sync(*ANDROID_SYNC_ARGS).stderr == "" + + for path, pinned in tree.pins.items(): + assert client.head(path) == pinned + assert client.remote_branches(path) == {"main"} + assert client.tags(path) == set() + + # Move foo's pin forward (on main) & tag it; frameworks/base keeps its pin. + new_pin = tree.repos["foo"].commit({"small": "new pin"}) + tree.repos["foo"].tag("v2", new_pin) + tree.superproject.commit({"foo": Gitlink(new_pin)}) + with host.watch_requests() as requests: + assert client.sync(*ANDROID_SYNC_ARGS).stderr == "" + + assert client.head("foo") == new_pin + assert client.head("frameworks/base") == tree.pins["frameworks/base"] + assert client.tags("foo") == set() + assert (client.top / "foo/small").read_text() == "new pin" + # A project already at its (immutable) pin isn't fetched. + assert host.requested_repos(requests) == { + "platform/manifest", + "platform/superproject", + "platform/foo", + } + + +def test_partial_clone_default_filter( + host: GitHost, client: RepoClient +) -> None: + """`--partial-clone` alone uses `blob:none` & disables clone bundles.""" + foo, big_blob = create_with_big_history(host, "platform/foo") + small_history = foo.commit_history_only_blob("old_small.txt", b"tiny") + small_tip = foo.git("rev-parse", "main:small") + foo.write_bundle() + host.create_manifest(manifest_xml('')) + + client.init(host.manifest_url, "--partial-clone") + client.sync() + + path = "platform/foo" + assert client.config(".repo/manifests", "repo.partialclone") == "true" + assert client.config(".repo/manifests", "repo.clonefilter") == "blob:none" + assert client.config(".repo/manifests", FILTER_KEY) == "blob:none" + assert client.config(path, FILTER_KEY) == "blob:none" + assert client.config(path, "extensions.partialclone") == "origin" + # Blobs of the checked out tree were fetched on demand; history wasn't + # (even tiny blobs). + assert (client.top / path / "big2.bin").read_text() == BIG + "2" + assert client.has_object(path, small_tip) + assert not client.has_object(path, big_blob) + assert not client.has_object(path, small_history) + assert bundle_requests(host) == {} + + +def test_partial_clone_two_phase_sync( + host: GitHost, client: RepoClient +) -> None: + """`sync -n` then `sync -l` with a partial clone (initial & incremental). + + The network phase fetches no blobs, so the local phase has to fetch the + checked out blobs on demand: `-l` isn't offline for partial clones. When + the server is unreachable, `sync -l` fails cleanly & can be retried. + """ + foo, big_blob = create_with_big_history(host, "platform/foo") + small_blob = foo.git("rev-parse", "main:small") + host.create_manifest(manifest_xml('')) + client.init(host.manifest_url, "--partial-clone") + + assert client.sync("-n").stderr == "" + path = "platform/foo" + assert not (client.top / path).exists() + gitdir = f"--git-dir={client.repodir / 'projects' / path}.git" + assert client.git( + ".", gitdir, "rev-parse", "refs/remotes/origin/main" + ) == foo.rev_parse("main") + objects = client.git( + ".", + gitdir, + "cat-file", + "--batch-check=%(objectname)", + "--batch-all-objects", + ).split() + assert foo.rev_parse("main") in objects + assert small_blob not in objects + + # When the server is offline, `sync -l` fails cleanly without leaving a + # broken worktree behind. + with host.offline(): + failed = client.sync("-l", check=False) + assert failed.returncode != 0 + assert "Cannot initialize work tree for platform/foo" in failed.stderr + assert not os.path.lexists(client.top / path / ".git") + + with host.watch_requests() as requests: + client.sync("-l") + + assert client.head(path) == foo.rev_parse("main") + assert (client.top / path / "big2.bin").read_text() == BIG + "2" + assert client.has_object(path, small_blob) + assert not client.has_object(path, big_blob) + # The on-demand blob fetches are the only traffic of the local sync. + assert host.requested_repos(requests) == {"platform/foo"} + + # Incremental `-n` then `-l` on the existing partial-clone checkout. + v2 = foo.commit({"small": "s2"}) + s2_blob = foo.git("rev-parse", "main:small") + client.sync("-n") + assert (client.top / path / "small").read_text() == "s" + assert not client.has_object(path, s2_blob) + + with host.watch_requests() as inc_requests: + client.sync("-l") + assert client.head(path) == v2 + assert (client.top / path / "small").read_text() == "s2" + assert client.has_object(path, s2_blob) + assert host.requested_repos(inc_requests) == {"platform/foo"}