tests: e2e: add a local smart HTTP git server

Add `GitHttpServer`, which serves the bare repositories under a
directory over smart HTTP by running `git http-backend` behind a small
threaded HTTP server on 127.0.0.1.  Like Gerrit/GoB, it also serves
static `<repo>/clone.bundle` files.  It records the requests it receives
so tests can check what clients fetched.

Add `GitHost`, which gives each test its own namespace of repos on a
shared server, with helpers to create & update a manifest repo.

Bug: 566270944
Change-Id: I35ff0b9ae743ed7a8d3c775525986e662ef03de0

Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/647504
Reviewed-by: Gavin Mak <gavinmak@google.com>
Tested-by: Brian Gan <brgan@google.com>
Commit-Queue: Brian Gan <brgan@google.com>

tests/e2e/harness.py

@@ -14,21 +14,45 @@
 
 """Hermetic end-to-end test harness for repo.
 
-`ServerRepo` manages bare repositories with helpers to create commits
-(including gitlinks for superprojects), branches, tags & bundles without
-needing a worktree.
+The harness has two parts:
+
+* `GitHttpServer`: a local smart HTTP git server built on `git http-backend`.
+  It also serves static `clone.bundle` files like Gerrit/GoB do.
+* `ServerRepo`/`GitHost`: bare repositories hosted by the server, with helpers
+  to create commits (including gitlinks for superprojects), branches, tags &
+  bundles without needing a worktree.
 """
 
+import contextlib
 import functools
+import http.server
 import os
 from pathlib import Path
+import re
+import socketserver
 import subprocess
-from typing import Dict, Mapping, NamedTuple, Optional, Tuple, Union
+import sys
+import threading
+from typing import (
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+)
+import urllib.parse
 
 
 # Every repo/git subprocess gets this timeout so a hang can't wedge the CQ.
 COMMAND_TIMEOUT = 300
 
+# The manifest repo lives two levels deep (like AOSP's platform/manifest) so
+# the common `fetch=".."` resolves to the root of the host namespace.
+MANIFEST_REPO = "platform/manifest"
+
 # Fixed identity used for all commits created by the harness (not by
 # the simulated user).
 HARNESS_IDENTITY = {
@@ -128,6 +152,18 @@ def git_version() -> Tuple[int, ...]:
     return tuple(int(x) if x.isdigit() else 0 for x in ver.split(".")[:3])
 
 
+def has_http_backend() -> bool:
+    """Whether `git http-backend` is available."""
+    try:
+        exec_path = Path(run_git("--exec-path"))
+    except (OSError, AssertionError):
+        return False
+    return any(
+        (exec_path / name).exists()
+        for name in ("git-http-backend", "git-http-backend.exe")
+    )
+
+
 def _quote_config(value: str) -> str:
     """Quote `value` for use as a git config subsection name or value."""
     escaped = (
@@ -151,6 +187,218 @@ def write_config(path: Path, settings: Mapping[str, str]) -> None:
             fp.write(f"[{section}]\n\t{name} = {_quote_config(value)}\n")
 
 
+class Request(NamedTuple):
+    """A request received by GitHttpServer."""
+
+    method: str
+    path: str
+    status: int
+
+
+class _ThreadingHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
+    """Like `http.server.ThreadingHTTPServer`, which needs Python 3.7+."""
+
+    daemon_threads = True
+    # The default backlog of 5 can overflow when `repo sync` fetches many
+    # projects in parallel, stalling connections on SYN retries.
+    request_queue_size = 128
+
+
+class GitHttpServer:
+    """A local smart HTTP git server backed by `git http-backend`.
+
+    All bare repositories under `root` are exported.  Requests for
+    `<repo>/clone.bundle` are served from a `clone.bundle` file inside the
+    repository directory (if it exists) to emulate Gerrit's bundle support.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.requests: List[Request] = []
+        self._lock = threading.Lock()
+        self._httpd: Optional[_ThreadingHTTPServer] = None
+        self._thread: Optional[threading.Thread] = None
+        self._env = dict(
+            clean_environ(),
+            GIT_HTTP_EXPORT_ALL="1",
+            GIT_PROJECT_ROOT=str(root),
+            HOME=str(root),
+            USERPROFILE=str(root),
+        )
+
+    @property
+    def url(self) -> str:
+        """Base URL of the server (without trailing slash)."""
+        assert self._httpd, "server not started"
+        host, port = self._httpd.server_address[:2]
+        return f"http://{host}:{port}"
+
+    def start(self) -> None:
+        """Start serving in a background thread."""
+        server = self
+
+        class Handler(_GitHttpHandler):
+            git_server = server
+
+        self._httpd = _ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self._thread = threading.Thread(
+            target=self._httpd.serve_forever, name="git-http-server"
+        )
+        self._thread.daemon = True
+        self._thread.start()
+
+    def stop(self) -> None:
+        """Stop the server."""
+        if self._httpd:
+            self._httpd.shutdown()
+            self._httpd.server_close()
+            self._httpd = None
+        if self._thread:
+            self._thread.join()
+            self._thread = None
+
+    def record(self, method: str, path: str, status: int) -> None:
+        with self._lock:
+            self.requests.append(Request(method, path, status))
+
+    def requests_for(self, prefix: str) -> List[Request]:
+        """Return the requests whose path starts with `prefix`."""
+        with self._lock:
+            return [r for r in self.requests if r.path.startswith(prefix)]
+
+    def run_backend(
+        self,
+        method: str,
+        path: str,
+        query: str,
+        headers: Mapping[str, str],
+        body: bytes,
+    ) -> bytes:
+        """Run `git http-backend` as a CGI script and return its output."""
+        env = dict(
+            self._env,
+            REQUEST_METHOD=method,
+            PATH_INFO=path,
+            QUERY_STRING=query,
+            CONTENT_TYPE=headers.get("Content-Type", ""),
+            CONTENT_LENGTH=str(len(body)),
+            REMOTE_ADDR="127.0.0.1",
+        )
+        # Needed for git protocol v2.
+        if headers.get("Git-Protocol"):
+            env["GIT_PROTOCOL"] = headers["Git-Protocol"]
+        if headers.get("Content-Encoding"):
+            env["HTTP_CONTENT_ENCODING"] = headers["Content-Encoding"]
+        proc = subprocess.run(
+            ["git", "http-backend"],
+            input=body,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=False,
+            timeout=COMMAND_TIMEOUT,
+        )
+        if proc.stderr:
+            sys.stderr.write(proc.stderr.decode("utf-8", "replace"))
+        return proc.stdout
+
+
+class _GitHttpHandler(http.server.BaseHTTPRequestHandler):
+    """Request handler for GitHttpServer."""
+
+    git_server: GitHttpServer
+
+    # Keep the protocol simple: one request per connection.
+    protocol_version = "HTTP/1.0"
+
+    def log_message(self, *args: object) -> None:
+        """Silence the default stderr logging."""
+
+    def _read_body(self) -> bytes:
+        # Respond to libcurl's `Expect: 100-continue` immediately so chunked
+        # POSTs don't wait for curl's 1s expect timeout under HTTP/1.0.
+        if self.headers.get("Expect", "").lower() == "100-continue":
+            self.send_response_only(100)
+            self.end_headers()
+        if self.headers.get("Transfer-Encoding", "").lower() == "chunked":
+            chunks = []
+            while True:
+                line = self.rfile.readline()
+                if not line:
+                    break
+                size = int(line.split(b";")[0].strip(), 16)
+                if not size:
+                    # Consume the trailer section.
+                    while self.rfile.readline() not in (b"\r\n", b"\n", b""):
+                        pass
+                    break
+                chunks.append(self.rfile.read(size))
+                self.rfile.readline()
+            return b"".join(chunks)
+        length = int(self.headers.get("Content-Length") or 0)
+        return self.rfile.read(length) if length else b""
+
+    @property
+    def _request_path(self) -> str:
+        """The URL-decoded request path (like CGI's PATH_INFO)."""
+        return urllib.parse.unquote(self.path.partition("?")[0])
+
+    def _send(self, status: int, headers: Dict[str, str], body: bytes) -> None:
+        self.git_server.record(self.command, self._request_path, status)
+        self.send_response(status)
+        for key, value in headers.items():
+            self.send_header(key, value)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
+    def _serve_bundle(self, path: str) -> None:
+        repo = path[: -len("/clone.bundle")].strip("/")
+        if ".." not in repo.split("/"):
+            for candidate in (f"{repo}.git", repo):
+                bundle = self.git_server.root / candidate / "clone.bundle"
+                if bundle.is_file():
+                    self._send(
+                        200,
+                        {"Content-Type": "application/octet-stream"},
+                        bundle.read_bytes(),
+                    )
+                    return
+        self._send(404, {"Content-Type": "text/plain"}, b"no bundle\n")
+
+    def _handle(self) -> None:
+        path = self._request_path
+        query = self.path.partition("?")[2]
+        if path.endswith("/clone.bundle"):
+            self._serve_bundle(path)
+            return
+
+        body = self._read_body()
+        output = self.git_server.run_backend(
+            self.command, path, query, self.headers, body
+        )
+        # Split the CGI response into headers & body at the first blank line.
+        match = re.search(rb"\r?\n\r?\n", output)
+        if not match:
+            self._send(500, {}, b"invalid CGI response\n")
+            return
+        sep, body_start = match.span()
+        status = 200
+        headers = {}
+        for line in output[:sep].decode("latin-1").splitlines():
+            key, _, value = line.partition(":")
+            if key.lower() == "status":
+                status = int(value.split()[0])
+            elif key and key.lower() != "content-length":
+                headers[key] = value.strip()
+        self._send(status, headers, output[body_start:])
+
+    do_GET = _handle
+    do_POST = _handle
+    do_HEAD = _handle
+
+
 class Gitlink(NamedTuple):
     """A gitlink (submodule/superproject) tree entry pointing at a commit."""
 
@@ -170,7 +418,7 @@ FileContent = Optional[Union[str, bytes, Blob, Gitlink]]
 
 
 class ServerRepo:
-    """A bare repository."""
+    """A bare repository hosted by GitHttpServer."""
 
     # Fake commit timestamp; see commit().
     _clock = 1700000000
@@ -293,7 +541,149 @@ class ServerRepo:
         self.git("tag", "-a", "-m", name, name, rev)
 
     def write_bundle(self) -> Path:
-        """Write a clone.bundle (all branches & tags) in the repo."""
+        """Write a clone.bundle (all branches & tags) served by the server."""
         bundle = self.path / "clone.bundle"
         self.git("bundle", "create", str(bundle), "--branches", "--tags")
         return bundle
+
+
+def _manifest_files(
+    content: Union[str, Mapping[str, FileContent]],
+) -> Mapping[str, FileContent]:
+    """Normalize default.xml contents or a map of files to a map of files."""
+    return {"default.xml": content} if isinstance(content, str) else content
+
+
+class GitHost:
+    """A namespace of repositories on a GitHttpServer.
+
+    Each test gets its own namespace so tests don't interfere with each other
+    while sharing a single server.
+    """
+
+    def __init__(self, server: GitHttpServer, prefix: str) -> None:
+        self.server = server
+        self.prefix = prefix
+        (server.root / prefix).mkdir(parents=True, exist_ok=True)
+
+    @property
+    def url(self) -> str:
+        """Base URL for this namespace (without trailing slash)."""
+        return f"{self.server.url}/{self.prefix}"
+
+    def path_of(self, name: str) -> Path:
+        return self.server.root / self.prefix / f"{name}.git"
+
+    def url_of(self, name: str) -> str:
+        return f"{self.url}/{name}"
+
+    def create_repo(
+        self,
+        name: str,
+        files: Optional[Mapping[str, FileContent]] = None,
+        branch: str = "main",
+    ) -> ServerRepo:
+        """Create a repository, optionally with an initial commit."""
+        repo = ServerRepo.create(
+            self.path_of(name), self.url_of(name), default_branch=branch
+        )
+        if files is not None:
+            repo.commit(files, branch=branch, message=f"{name}: initial")
+        return repo
+
+    @property
+    def manifest_url(self) -> str:
+        """URL of the manifest repo created by create_manifest."""
+        return self.url_of(MANIFEST_REPO)
+
+    def create_manifest(
+        self,
+        content: Union[str, Mapping[str, FileContent]],
+        branch: str = "main",
+    ) -> ServerRepo:
+        """Create the manifest repo.
+
+        Args:
+            content: The default.xml contents, or a map of files.
+            branch: The manifest branch.
+        """
+        return self.create_repo(
+            MANIFEST_REPO, _manifest_files(content), branch=branch
+        )
+
+    def update_manifest(
+        self,
+        content: Union[str, Mapping[str, FileContent]],
+        branch: str = "main",
+    ) -> str:
+        """Commit to the manifest repo.
+
+        Args:
+            content: The new default.xml contents, or a map of files.
+            branch: The manifest branch.
+
+        Returns:
+            The new manifest commit id.
+        """
+        repo = ServerRepo(
+            self.path_of(MANIFEST_REPO),
+            self.manifest_url,
+            default_branch=branch,
+        )
+        return repo.commit(_manifest_files(content), branch=branch)
+
+    def fork(self, src: ServerRepo, name: str) -> ServerRepo:
+        """Create a cheap copy of `src` sharing its objects via alternates.
+
+        Use this before mutating a session-wide repo (e.g. the repo source).
+        """
+        repo = ServerRepo.create(
+            self.path_of(name),
+            self.url_of(name),
+            default_branch=src.default_branch,
+        )
+        alternates = repo.path / "objects" / "info" / "alternates"
+        # Use forward slashes & LF (write_text would emit CRLF on Windows),
+        # which git accepts on every platform.
+        objects = (src.path / "objects").as_posix()
+        alternates.write_bytes(f"{objects}\n".encode())
+        refs = src.git("for-each-ref", "--format=%(objectname) %(refname)")
+        for line in refs.splitlines():
+            oid, ref = line.split()
+            repo.set_ref(ref, oid)
+        return repo
+
+    def requests(self) -> List[Request]:
+        """All requests received for this namespace."""
+        return self.server.requests_for(f"/{self.prefix}/")
+
+
+def manifest_xml(*elements: str, default: Optional[str] = None) -> str:
+    """Build a manifest with an `origin` remote & a default element.
+
+    Args:
+        elements: Extra XML elements (projects, etc...) to include.
+        default: Attributes for the <default> element.
+    """
+    if default is None:
+        default = 'remote="origin" revision="main"'
+    body = "".join(f"  {x}\n" for x in elements)
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        "<manifest>\n"
+        '  <remote name="origin" fetch=".." />\n'
+        f"  <default {default} />\n"
+        f"{body}"
+        "</manifest>\n"
+    )
+
+
+@contextlib.contextmanager
+def serve(root: Path) -> Iterator[GitHttpServer]:
+    """Run a GitHttpServer for the duration of the context."""
+    server = GitHttpServer(root)
+    server.start()
+    try:
+        yield server
+    finally:
+        server.stop()

tests/e2e/test_harness.py

@@ -14,8 +14,10 @@
 
 """Tests for the E2E harness helpers themselves."""
 
+import http.client
 from pathlib import Path
-from typing import Mapping
+from typing import Iterator, Mapping
+import urllib.parse
 
 import harness
 import pytest
@@ -27,6 +29,15 @@ pytestmark = pytest.mark.skipif(
 )
 
 
+@pytest.fixture
+def git_server(tmp_path: Path) -> Iterator[harness.GitHttpServer]:
+    """A git smart HTTP server for a single test."""
+    if not harness.has_http_backend():
+        pytest.skip("git http-backend is not available")
+    with harness.serve(tmp_path) as server:
+        yield server
+
+
 @pytest.mark.parametrize(
     "key,value",
     [
@@ -58,3 +69,40 @@ def test_commit_replace_file_with_dir(
     repo.commit({"foo": "file"})
     repo.commit(files)
     assert repo.git("ls-tree", "-r", "--name-only", "main") == "foo/bar"
+
+
+@pytest.mark.parametrize(
+    "chunked,expect_continue",
+    [(False, False), (True, False), (True, True)],
+)
+def test_http_post_body(
+    git_server: harness.GitHttpServer, chunked: bool, expect_continue: bool
+) -> None:
+    """POST bodies reach git http-backend however curl encodes them."""
+    name = f"post-{int(chunked)}{int(expect_continue)}.git"
+    repo = harness.ServerRepo.create(git_server.root / name, "unused")
+    commit = repo.commit({"file": "x"})
+    # A protocol v2 ls-refs request: a command pkt-line & a flush-pkt.
+    body = b"0014command=ls-refs\n0000"
+    headers = {
+        "Content-Type": "application/x-git-upload-pack-request",
+        "Git-Protocol": "version=2",
+    }
+    if expect_continue:
+        headers["Expect"] = "100-continue"
+    url = urllib.parse.urlsplit(git_server.url)
+    conn = http.client.HTTPConnection(url.hostname, url.port, timeout=30)
+    try:
+        # http.client skips any interim `100 Continue` response.
+        conn.request(
+            "POST",
+            f"/{name}/git-upload-pack",
+            body=iter([body[:10], body[10:]]) if chunked else body,
+            headers=headers,
+            encode_chunked=chunked,
+        )
+        resp = conn.getresponse()
+        assert resp.status == 200
+        assert f"{commit} refs/heads/main".encode() in resp.read()
+    finally:
+        conn.close()