tests: e2e: add tests for more repo init options

Cover `--reference` from a local mirror (with & without `--dissociate`),
`--standalone-manifest` with a manifest file served over HTTP, and
`--manifest-depth`.

Teach the server to serve static files (`GitHost.publish_file`), let
`manifest_xml` take an absolute `fetch` URL, and add `move_aside` to
make git dirs disappear mid-test (deleting them fails on Windows).

Bug: 566270944
Change-Id: Id60a19613a627f76d8d9125570dd5ce0cdfff9aa
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/648549
Reviewed-by: Gavin Mak <gavinmak@google.com>
Tested-by: Brian Gan <brgan@google.com>
Commit-Queue: Brian Gan <brgan@google.com>

tests/e2e/harness.py

@@ -17,8 +17,8 @@
 The harness has three parts:
 
 * `GitHttpServer`: a local smart HTTP git server built on `git http-backend`.
-  It also serves static `clone.bundle` files like Gerrit/GoB do, and can
-  simulate outages.
+  It also serves static `clone.bundle` files like Gerrit/GoB do, other static
+  files (e.g. standalone manifests), and can simulate outages.
 * `ServerRepo`/`GitHost`: bare repositories hosted by the server, with helpers
   to create commits (including gitlinks for superprojects), branches, tags &
   bundles without needing a worktree.
@@ -74,6 +74,11 @@ REPO_SOURCE_BRANCH = "main"
 # the common `fetch=".."` resolves to the root of the host namespace.
 MANIFEST_REPO = "platform/manifest"
 
+# Directory (directly under a host namespace) whose files the server returns
+# as-is, e.g. for `repo init --standalone-manifest`.  The leading underscore
+# keeps it from clashing with repo names used by tests.
+STATIC_DIR = "_static"
+
 # The git endpoints (relative to a repo URL) that clients request.
 REPO_ENDPOINTS = (
     "info/refs",
@@ -279,6 +284,17 @@ def write_config(path: Path, settings: Mapping[str, str]) -> None:
             fp.write(f"[{section}]\n\t{name} = {_quote_config(value)}\n")
 
 
+def move_aside(path: Path) -> Path:
+    """Make `path` disappear by renaming it next to itself; return the new path.
+
+    Use this rather than deleting git dirs mid-test: git's read-only pack
+    files make `shutil.rmtree` fail on Windows.
+    """
+    dest = path.with_name(f"{path.name}.moved-aside")
+    path.rename(dest)
+    return dest
+
+
 class Request(NamedTuple):
     """A request received by GitHttpServer."""
 
@@ -302,6 +318,7 @@ class GitHttpServer:
     All bare repositories under `root` are exported.  Requests for
     `<repo>/clone.bundle` are served from a `clone.bundle` file inside the
     repository directory (if it exists) to emulate Gerrit's bundle support.
+    Files under `<namespace>/_static/` (see `STATIC_DIR`) are served as-is.
     """
 
     def __init__(self, root: Path) -> None:
@@ -494,6 +511,27 @@ class _GitHttpHandler(http.server.BaseHTTPRequestHandler):
                     return
         self._send(404, {"Content-Type": "text/plain"}, b"no bundle\n")
 
+    @staticmethod
+    def _static_parts(path: str) -> Optional[List[str]]:
+        """Split `path` if it names a static file (see `STATIC_DIR`)."""
+        parts = path.strip("/").split("/")
+        if len(parts) >= 3 and parts[1] == STATIC_DIR:
+            return parts
+        return None
+
+    def _serve_static(self, parts: List[str]) -> None:
+        # Never serve anything outside the static dir (e.g. via `..`).
+        if self.command in ("GET", "HEAD") and not {"", ".", ".."} & set(parts):
+            path = self.git_server.root.joinpath(*parts)
+            if path.is_file():
+                self._send(
+                    200,
+                    {"Content-Type": "application/octet-stream"},
+                    path.read_bytes(),
+                )
+                return
+        self._send(404, {"Content-Type": "text/plain"}, b"not found\n")
+
     def _handle(self) -> None:
         path = self._request_path
         query = self.path.partition("?")[2]
@@ -505,6 +543,10 @@ class _GitHttpHandler(http.server.BaseHTTPRequestHandler):
         if path.endswith("/clone.bundle"):
             self._serve_bundle(path)
             return
+        static_parts = self._static_parts(path)
+        if static_parts:
+            self._serve_static(static_parts)
+            return
 
         body = self._read_body()
         output = self.git_server.run_backend(
@@ -857,6 +899,25 @@ class GitHost:
         finally:
             requests.extend(self.server.requests_for(f"/{self.prefix}/", start))
 
+    def publish_file(self, name: str, content: Union[str, bytes]) -> str:
+        """Serve `content` as a static file (replacing any old one).
+
+        Returns:
+            The URL of the file.
+        """
+        # The server refuses to serve such paths (see _serve_static).
+        assert not {"", ".", ".."} & set(name.split("/")), name
+        path = self.server.root / self.prefix / STATIC_DIR / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        # Write to a temp file & rename so the server never serves a partial
+        # file.  Write bytes so the content is the same on every platform.
+        tmp = path.with_name(f"{path.name}.tmp")
+        tmp.write_bytes(content)
+        os.replace(tmp, path)
+        return f"{self.url}/{STATIC_DIR}/{name}"
+
     @contextlib.contextmanager
     def offline(self) -> Iterator[None]:
         """Fail all requests for this namespace during the context.
@@ -867,12 +928,16 @@ class GitHost:
             yield
 
 
-def manifest_xml(*elements: str, default: Optional[str] = None) -> str:
+def manifest_xml(
+    *elements: str, default: Optional[str] = None, fetch: str = ".."
+) -> str:
     """Build a manifest with an `origin` remote & a default element.
 
     Args:
         elements: Extra XML elements (projects, etc...) to include.
         default: Attributes for the <default> element.
+        fetch: The `fetch` attribute of the remote; the default resolves to
+            the root of the host namespace for manifests served by `GitHost`.
     """
     if default is None:
         default = 'remote="origin" revision="main"'
@@ -880,7 +945,7 @@ def manifest_xml(*elements: str, default: Optional[str] = None) -> str:
     return (
         '<?xml version="1.0" encoding="UTF-8"?>\n'
         "<manifest>\n"
-        '  <remote name="origin" fetch=".." />\n'
+        f'  <remote name="origin" fetch="{fetch}" />\n'
         f"  <default {default} />\n"
         f"{body}"
         "</manifest>\n"

tests/e2e/test_harness.py

@@ -130,6 +130,30 @@ def test_http_post_body(
         conn.close()
 
 
+def test_static_files(git_server: harness.GitHttpServer) -> None:
+    """Published files are served as-is, and nothing outside of them."""
+    host = harness.GitHost(git_server, "static-files")
+    host.create_repo("repo", {"f": "x"})
+    url = host.publish_file("dir/default.xml", "<manifest />\n")
+    path = urllib.parse.urlsplit(url).path
+    assert _http_request(git_server, "GET", path) == (200, b"<manifest />\n")
+
+    # Publishing again replaces the file.
+    assert host.publish_file("dir/default.xml", b"v2") == url
+    assert _http_request(git_server, "GET", path) == (200, b"v2")
+
+    static = f"/{host.prefix}/{harness.STATIC_DIR}"
+    for bad_path in (
+        f"{static}/missing.xml",
+        f"{static}/dir",
+        f"{static}/../repo.git/config",
+        f"{static}/%2e%2e/repo.git/config",
+        f"{static}//etc/passwd",
+    ):
+        assert _http_request(git_server, "GET", bad_path)[0] == 404, bad_path
+    assert _http_request(git_server, "POST", path)[0] == 404
+
+
 def test_offline(git_server: harness.GitHttpServer) -> None:
     """Requests fail while a namespace is offline, and work again after."""
     host = harness.GitHost(git_server, "offline")
@@ -156,11 +180,13 @@ def test_requested_repos(git_server: harness.GitHttpServer) -> None:
     host = harness.GitHost(git_server, "requested")
     nested = host.create_repo("platform/nested", {"f": "x"})
     top = host.create_repo("top", {"f": "x"})
+    host.publish_file("file.xml", "x")
 
     harness.run_git("ls-remote", nested.url)
     with host.watch_requests() as watched:
         harness.run_git("ls-remote", top.url)
         _http_request(git_server, "GET", f"/{host.prefix}/top/clone.bundle")
+        _http_request(git_server, "GET", f"/{host.prefix}/_static/file.xml")
 
     assert host.requested_repos() == {"platform/nested", "top"}
     assert host.requested_repos(watched) == {"top"}

tests/e2e/test_init_sync.py

@@ -14,11 +14,13 @@
 
 """E2E tests for the core `repo init` & `repo sync` options."""
 
+from pathlib import Path
 from typing import List, Set
 
 from harness import GIT_DEFAULTS_PARAMS
 from harness import GitHost
 from harness import manifest_xml
+from harness import move_aside
 from harness import RepoClient
 import pytest
 
@@ -225,3 +227,129 @@ def test_manifest_name(host: GitHost, client: RepoClient) -> None:
     client.sync()
 
     assert client.list_projects() == {"platform/bar": "platform/bar"}
+
+
+@pytest.mark.parametrize("dissociate", [False, True])
+def test_reference_mirror(
+    host: GitHost, client: RepoClient, dissociate: bool
+) -> None:
+    """`init --reference` borrows objects from a local `--mirror` checkout.
+
+    With `--dissociate` the objects are copied so the mirror can go away.
+    """
+    foo = host.create_repo("platform/foo", {"f": "foo"})
+    host.create_manifest(manifest_xml('<project name="platform/foo" />'))
+    mirror = RepoClient(
+        client.top.parent / "mirror",
+        client.launcher,
+        client.env,
+        client.repo_url,
+    )
+    mirror.init(host.manifest_url, "--mirror")
+    mirror.sync()
+    if dissociate:
+        # Dissociating the manifest repo itself currently fails (its
+        # preciousObjects setting blocks `git repack -a -d`), so only reference
+        # the mirror's projects.
+        move_aside(mirror.repodir / "manifests.git")
+
+    init_args = ["--reference", str(mirror.top)]
+    if dissociate:
+        init_args.append("--dissociate")
+    client.init(host.manifest_url, *init_args)
+    client.sync()
+
+    path = "platform/foo"
+    assert client.head(path) == foo.rev_parse("main")
+    assert (client.top / path / "f").read_text() == "foo"
+    assert client.has_object(path, foo.rev_parse("main"))
+    objdir = client.repodir / "project-objects/platform/foo.git"
+    alternates = objdir / "objects/info/alternates"
+    assert client.config(".repo/manifests", "repo.reference") == str(mirror.top)
+    if dissociate:
+        assert not alternates.exists()
+        assert client.config(".repo/manifests", "repo.dissociate") == "true"
+    else:
+        expected = mirror.top / "platform/foo.git/objects"
+        assert Path(alternates.read_text().strip()) == expected
+        # Nothing had to be downloaded or copied into the local object dir.
+        counts = client.git(".", f"--git-dir={objdir}", "count-objects", "-v")
+        assert "count: 0\n" in counts
+        assert "in-pack: 0\n" in counts
+    # Updates still come from the server (the mirror isn't synced).
+    latest = foo.commit({"f": "2"})
+    client.sync()
+    assert client.head(path) == latest
+
+    # If the mirror goes away, `--dissociate` checkouts still have all
+    # historical objects, whereas shared checkouts lost the borrowed ones.
+    move_aside(mirror.top)
+    if dissociate:
+        assert client.git(path, "rev-list", "--count", "HEAD") == "2"
+        client.git(path, "rev-list", "--objects", "HEAD")
+    else:
+        with pytest.raises(AssertionError, match="rev-list"):
+            client.git(path, "rev-list", "--objects", "HEAD")
+
+
+def test_standalone_manifest(host: GitHost, client: RepoClient) -> None:
+    """`init --standalone-manifest` fetches a static manifest file over HTTP.
+
+    There's no manifest checkout to update, so syncs only fetch the projects.
+    """
+    foo = host.create_repo("platform/foo", {"f": "1"})
+    host.create_repo("platform/bar", {"f": "bar"})
+    xml = manifest_xml('<project name="platform/foo" />', fetch=host.url)
+    manifest_url = host.publish_file("default.xml", xml)
+
+    client.init(manifest_url, "--standalone-manifest")
+    client.sync()
+
+    path = "platform/foo"
+    assert client.head(path) == foo.rev_parse("main")
+    assert (client.repodir / "manifests/default.xml").read_text() == xml
+    assert (
+        client.config(".repo/manifests.git", "manifest.standalone")
+        == manifest_url
+    )
+    # Updating the static file on the server does not affect `repo sync`,
+    # which only fetches the project.
+    host.publish_file(
+        "default.xml",
+        manifest_xml(
+            '<project name="platform/foo" />',
+            '<project name="platform/bar" />',
+            fetch=host.url,
+        ),
+    )
+    latest = foo.commit({"f": "2"})
+    with host.watch_requests() as requests:
+        client.sync()
+    assert client.head(path) == latest
+    assert client.list_projects() == {"platform/foo": "platform/foo"}
+    assert requests == host.requests_for("platform/foo", requests)
+    # Re-initializing without a manifest URL isn't allowed.
+    result = client.run("init", "--no-repo-verify", check=False)
+    assert result.returncode != 0
+    assert "cannot be re-initialized without --manifest-url/-u" in result.stderr
+
+
+def test_manifest_depth(host: GitHost, client: RepoClient) -> None:
+    """`init --manifest-depth` shallow-clones the manifest repo only."""
+    foo = host.create_repo("platform/foo", {"f": "1"})
+    foo.commit({"f": "2"})
+    host.create_manifest(manifest_xml())
+    host.update_manifest(manifest_xml('<project name="platform/foo" />'))
+
+    client.init(host.manifest_url, "--manifest-depth=1")
+
+    manifests = ".repo/manifests"
+    assert client.is_shallow(manifests)
+    assert client.git(manifests, "rev-list", "--count", "HEAD") == "1"
+
+    # `--nmu` skips updating the manifest project, keeping it shallow while
+    # syncing projects at full depth.
+    client.sync("--nmu")
+    assert client.is_shallow(manifests)
+    assert not client.is_shallow("platform/foo")
+    assert client.git("platform/foo", "rev-list", "--count", "HEAD") == "2"