From e3e8e8a3b0509db51ef42c46412af2c59b17db95 Mon Sep 17 00:00:00 2001 From: Brian Gan Date: Fri, 23 Oct 2026 01:57:42 +0000 Subject: [PATCH] tests: e2e: add tests for more `repo init` options Cover `--reference` from a local mirror (with & without `--dissociate`), `--standalone-manifest` with a manifest file served over HTTP, and `--manifest-depth`. Teach the server to serve static files (`GitHost.publish_file`), let `manifest_xml` take an absolute `fetch` URL, and add `move_aside` to make git dirs disappear mid-test (deleting them fails on Windows). Bug: 566270944 Change-Id: Id60a19613a627f76d8d9125570dd5ce0cdfff9aa Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/648549 Reviewed-by: Gavin Mak Tested-by: Brian Gan Commit-Queue: Brian Gan --- tests/e2e/harness.py | 73 ++++++++++++++++++-- tests/e2e/test_harness.py | 26 ++++++++ tests/e2e/test_init_sync.py | 128 ++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 4 deletions(-) diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 0dda5111c..f295ce02c 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -17,8 +17,8 @@ The harness has three parts: * `GitHttpServer`: a local smart HTTP git server built on `git http-backend`. - It also serves static `clone.bundle` files like Gerrit/GoB do, and can - simulate outages. + It also serves static `clone.bundle` files like Gerrit/GoB do, other static + files (e.g. standalone manifests), and can simulate outages. * `ServerRepo`/`GitHost`: bare repositories hosted by the server, with helpers to create commits (including gitlinks for superprojects), branches, tags & bundles without needing a worktree. @@ -74,6 +74,11 @@ REPO_SOURCE_BRANCH = "main" # the common `fetch=".."` resolves to the root of the host namespace. MANIFEST_REPO = "platform/manifest" +# Directory (directly under a host namespace) whose files the server returns +# as-is, e.g. for `repo init --standalone-manifest`. The leading underscore +# keeps it from clashing with repo names used by tests. +STATIC_DIR = "_static" + # The git endpoints (relative to a repo URL) that clients request. REPO_ENDPOINTS = ( "info/refs", @@ -279,6 +284,17 @@ def write_config(path: Path, settings: Mapping[str, str]) -> None: fp.write(f"[{section}]\n\t{name} = {_quote_config(value)}\n") +def move_aside(path: Path) -> Path: + """Make `path` disappear by renaming it next to itself; return the new path. + + Use this rather than deleting git dirs mid-test: git's read-only pack + files make `shutil.rmtree` fail on Windows. + """ + dest = path.with_name(f"{path.name}.moved-aside") + path.rename(dest) + return dest + + class Request(NamedTuple): """A request received by GitHttpServer.""" @@ -302,6 +318,7 @@ class GitHttpServer: All bare repositories under `root` are exported. Requests for `/clone.bundle` are served from a `clone.bundle` file inside the repository directory (if it exists) to emulate Gerrit's bundle support. + Files under `/_static/` (see `STATIC_DIR`) are served as-is. """ def __init__(self, root: Path) -> None: @@ -494,6 +511,27 @@ class _GitHttpHandler(http.server.BaseHTTPRequestHandler): return self._send(404, {"Content-Type": "text/plain"}, b"no bundle\n") + @staticmethod + def _static_parts(path: str) -> Optional[List[str]]: + """Split `path` if it names a static file (see `STATIC_DIR`).""" + parts = path.strip("/").split("/") + if len(parts) >= 3 and parts[1] == STATIC_DIR: + return parts + return None + + def _serve_static(self, parts: List[str]) -> None: + # Never serve anything outside the static dir (e.g. via `..`). + if self.command in ("GET", "HEAD") and not {"", ".", ".."} & set(parts): + path = self.git_server.root.joinpath(*parts) + if path.is_file(): + self._send( + 200, + {"Content-Type": "application/octet-stream"}, + path.read_bytes(), + ) + return + self._send(404, {"Content-Type": "text/plain"}, b"not found\n") + def _handle(self) -> None: path = self._request_path query = self.path.partition("?")[2] @@ -505,6 +543,10 @@ class _GitHttpHandler(http.server.BaseHTTPRequestHandler): if path.endswith("/clone.bundle"): self._serve_bundle(path) return + static_parts = self._static_parts(path) + if static_parts: + self._serve_static(static_parts) + return body = self._read_body() output = self.git_server.run_backend( @@ -857,6 +899,25 @@ class GitHost: finally: requests.extend(self.server.requests_for(f"/{self.prefix}/", start)) + def publish_file(self, name: str, content: Union[str, bytes]) -> str: + """Serve `content` as a static file (replacing any old one). + + Returns: + The URL of the file. + """ + # The server refuses to serve such paths (see _serve_static). + assert not {"", ".", ".."} & set(name.split("/")), name + path = self.server.root / self.prefix / STATIC_DIR / name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, str): + content = content.encode("utf-8") + # Write to a temp file & rename so the server never serves a partial + # file. Write bytes so the content is the same on every platform. + tmp = path.with_name(f"{path.name}.tmp") + tmp.write_bytes(content) + os.replace(tmp, path) + return f"{self.url}/{STATIC_DIR}/{name}" + @contextlib.contextmanager def offline(self) -> Iterator[None]: """Fail all requests for this namespace during the context. @@ -867,12 +928,16 @@ class GitHost: yield -def manifest_xml(*elements: str, default: Optional[str] = None) -> str: +def manifest_xml( + *elements: str, default: Optional[str] = None, fetch: str = ".." +) -> str: """Build a manifest with an `origin` remote & a default element. Args: elements: Extra XML elements (projects, etc...) to include. default: Attributes for the element. + fetch: The `fetch` attribute of the remote; the default resolves to + the root of the host namespace for manifests served by `GitHost`. """ if default is None: default = 'remote="origin" revision="main"' @@ -880,7 +945,7 @@ def manifest_xml(*elements: str, default: Optional[str] = None) -> str: return ( '\n' "\n" - ' \n' + f' \n' f" \n" f"{body}" "\n" diff --git a/tests/e2e/test_harness.py b/tests/e2e/test_harness.py index d4da31cb9..63baf602b 100644 --- a/tests/e2e/test_harness.py +++ b/tests/e2e/test_harness.py @@ -130,6 +130,30 @@ def test_http_post_body( conn.close() +def test_static_files(git_server: harness.GitHttpServer) -> None: + """Published files are served as-is, and nothing outside of them.""" + host = harness.GitHost(git_server, "static-files") + host.create_repo("repo", {"f": "x"}) + url = host.publish_file("dir/default.xml", "\n") + path = urllib.parse.urlsplit(url).path + assert _http_request(git_server, "GET", path) == (200, b"\n") + + # Publishing again replaces the file. + assert host.publish_file("dir/default.xml", b"v2") == url + assert _http_request(git_server, "GET", path) == (200, b"v2") + + static = f"/{host.prefix}/{harness.STATIC_DIR}" + for bad_path in ( + f"{static}/missing.xml", + f"{static}/dir", + f"{static}/../repo.git/config", + f"{static}/%2e%2e/repo.git/config", + f"{static}//etc/passwd", + ): + assert _http_request(git_server, "GET", bad_path)[0] == 404, bad_path + assert _http_request(git_server, "POST", path)[0] == 404 + + def test_offline(git_server: harness.GitHttpServer) -> None: """Requests fail while a namespace is offline, and work again after.""" host = harness.GitHost(git_server, "offline") @@ -156,11 +180,13 @@ def test_requested_repos(git_server: harness.GitHttpServer) -> None: host = harness.GitHost(git_server, "requested") nested = host.create_repo("platform/nested", {"f": "x"}) top = host.create_repo("top", {"f": "x"}) + host.publish_file("file.xml", "x") harness.run_git("ls-remote", nested.url) with host.watch_requests() as watched: harness.run_git("ls-remote", top.url) _http_request(git_server, "GET", f"/{host.prefix}/top/clone.bundle") + _http_request(git_server, "GET", f"/{host.prefix}/_static/file.xml") assert host.requested_repos() == {"platform/nested", "top"} assert host.requested_repos(watched) == {"top"} diff --git a/tests/e2e/test_init_sync.py b/tests/e2e/test_init_sync.py index 5fb1db100..67c3d53af 100644 --- a/tests/e2e/test_init_sync.py +++ b/tests/e2e/test_init_sync.py @@ -14,11 +14,13 @@ """E2E tests for the core `repo init` & `repo sync` options.""" +from pathlib import Path from typing import List, Set from harness import GIT_DEFAULTS_PARAMS from harness import GitHost from harness import manifest_xml +from harness import move_aside from harness import RepoClient import pytest @@ -225,3 +227,129 @@ def test_manifest_name(host: GitHost, client: RepoClient) -> None: client.sync() assert client.list_projects() == {"platform/bar": "platform/bar"} + + +@pytest.mark.parametrize("dissociate", [False, True]) +def test_reference_mirror( + host: GitHost, client: RepoClient, dissociate: bool +) -> None: + """`init --reference` borrows objects from a local `--mirror` checkout. + + With `--dissociate` the objects are copied so the mirror can go away. + """ + foo = host.create_repo("platform/foo", {"f": "foo"}) + host.create_manifest(manifest_xml('')) + mirror = RepoClient( + client.top.parent / "mirror", + client.launcher, + client.env, + client.repo_url, + ) + mirror.init(host.manifest_url, "--mirror") + mirror.sync() + if dissociate: + # Dissociating the manifest repo itself currently fails (its + # preciousObjects setting blocks `git repack -a -d`), so only reference + # the mirror's projects. + move_aside(mirror.repodir / "manifests.git") + + init_args = ["--reference", str(mirror.top)] + if dissociate: + init_args.append("--dissociate") + client.init(host.manifest_url, *init_args) + client.sync() + + path = "platform/foo" + assert client.head(path) == foo.rev_parse("main") + assert (client.top / path / "f").read_text() == "foo" + assert client.has_object(path, foo.rev_parse("main")) + objdir = client.repodir / "project-objects/platform/foo.git" + alternates = objdir / "objects/info/alternates" + assert client.config(".repo/manifests", "repo.reference") == str(mirror.top) + if dissociate: + assert not alternates.exists() + assert client.config(".repo/manifests", "repo.dissociate") == "true" + else: + expected = mirror.top / "platform/foo.git/objects" + assert Path(alternates.read_text().strip()) == expected + # Nothing had to be downloaded or copied into the local object dir. + counts = client.git(".", f"--git-dir={objdir}", "count-objects", "-v") + assert "count: 0\n" in counts + assert "in-pack: 0\n" in counts + # Updates still come from the server (the mirror isn't synced). + latest = foo.commit({"f": "2"}) + client.sync() + assert client.head(path) == latest + + # If the mirror goes away, `--dissociate` checkouts still have all + # historical objects, whereas shared checkouts lost the borrowed ones. + move_aside(mirror.top) + if dissociate: + assert client.git(path, "rev-list", "--count", "HEAD") == "2" + client.git(path, "rev-list", "--objects", "HEAD") + else: + with pytest.raises(AssertionError, match="rev-list"): + client.git(path, "rev-list", "--objects", "HEAD") + + +def test_standalone_manifest(host: GitHost, client: RepoClient) -> None: + """`init --standalone-manifest` fetches a static manifest file over HTTP. + + There's no manifest checkout to update, so syncs only fetch the projects. + """ + foo = host.create_repo("platform/foo", {"f": "1"}) + host.create_repo("platform/bar", {"f": "bar"}) + xml = manifest_xml('', fetch=host.url) + manifest_url = host.publish_file("default.xml", xml) + + client.init(manifest_url, "--standalone-manifest") + client.sync() + + path = "platform/foo" + assert client.head(path) == foo.rev_parse("main") + assert (client.repodir / "manifests/default.xml").read_text() == xml + assert ( + client.config(".repo/manifests.git", "manifest.standalone") + == manifest_url + ) + # Updating the static file on the server does not affect `repo sync`, + # which only fetches the project. + host.publish_file( + "default.xml", + manifest_xml( + '', + '', + fetch=host.url, + ), + ) + latest = foo.commit({"f": "2"}) + with host.watch_requests() as requests: + client.sync() + assert client.head(path) == latest + assert client.list_projects() == {"platform/foo": "platform/foo"} + assert requests == host.requests_for("platform/foo", requests) + # Re-initializing without a manifest URL isn't allowed. + result = client.run("init", "--no-repo-verify", check=False) + assert result.returncode != 0 + assert "cannot be re-initialized without --manifest-url/-u" in result.stderr + + +def test_manifest_depth(host: GitHost, client: RepoClient) -> None: + """`init --manifest-depth` shallow-clones the manifest repo only.""" + foo = host.create_repo("platform/foo", {"f": "1"}) + foo.commit({"f": "2"}) + host.create_manifest(manifest_xml()) + host.update_manifest(manifest_xml('')) + + client.init(host.manifest_url, "--manifest-depth=1") + + manifests = ".repo/manifests" + assert client.is_shallow(manifests) + assert client.git(manifests, "rev-list", "--count", "HEAD") == "1" + + # `--nmu` skips updating the manifest project, keeping it shallow while + # syncing projects at full depth. + client.sync("--nmu") + assert client.is_shallow(manifests) + assert not client.is_shallow("platform/foo") + assert client.git("platform/foo", "rev-list", "--count", "HEAD") == "2"