diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 4dee6d2e9..359d7eb56 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -14,21 +14,45 @@ """Hermetic end-to-end test harness for repo. -`ServerRepo` manages bare repositories with helpers to create commits -(including gitlinks for superprojects), branches, tags & bundles without -needing a worktree. +The harness has two parts: + +* `GitHttpServer`: a local smart HTTP git server built on `git http-backend`. + It also serves static `clone.bundle` files like Gerrit/GoB do. +* `ServerRepo`/`GitHost`: bare repositories hosted by the server, with helpers + to create commits (including gitlinks for superprojects), branches, tags & + bundles without needing a worktree. """ +import contextlib import functools +import http.server import os from pathlib import Path +import re +import socketserver import subprocess -from typing import Dict, Mapping, NamedTuple, Optional, Tuple, Union +import sys +import threading +from typing import ( + Dict, + Iterator, + List, + Mapping, + NamedTuple, + Optional, + Tuple, + Union, +) +import urllib.parse # Every repo/git subprocess gets this timeout so a hang can't wedge the CQ. COMMAND_TIMEOUT = 300 +# The manifest repo lives two levels deep (like AOSP's platform/manifest) so +# the common `fetch=".."` resolves to the root of the host namespace. +MANIFEST_REPO = "platform/manifest" + # Fixed identity used for all commits created by the harness (not by # the simulated user). HARNESS_IDENTITY = { @@ -128,6 +152,18 @@ def git_version() -> Tuple[int, ...]: return tuple(int(x) if x.isdigit() else 0 for x in ver.split(".")[:3]) +def has_http_backend() -> bool: + """Whether `git http-backend` is available.""" + try: + exec_path = Path(run_git("--exec-path")) + except (OSError, AssertionError): + return False + return any( + (exec_path / name).exists() + for name in ("git-http-backend", "git-http-backend.exe") + ) + + def _quote_config(value: str) -> str: """Quote `value` for use as a git config subsection name or value.""" escaped = ( @@ -151,6 +187,218 @@ def write_config(path: Path, settings: Mapping[str, str]) -> None: fp.write(f"[{section}]\n\t{name} = {_quote_config(value)}\n") +class Request(NamedTuple): + """A request received by GitHttpServer.""" + + method: str + path: str + status: int + + +class _ThreadingHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): + """Like `http.server.ThreadingHTTPServer`, which needs Python 3.7+.""" + + daemon_threads = True + # The default backlog of 5 can overflow when `repo sync` fetches many + # projects in parallel, stalling connections on SYN retries. + request_queue_size = 128 + + +class GitHttpServer: + """A local smart HTTP git server backed by `git http-backend`. + + All bare repositories under `root` are exported. Requests for + `/clone.bundle` are served from a `clone.bundle` file inside the + repository directory (if it exists) to emulate Gerrit's bundle support. + """ + + def __init__(self, root: Path) -> None: + self.root = root + self.requests: List[Request] = [] + self._lock = threading.Lock() + self._httpd: Optional[_ThreadingHTTPServer] = None + self._thread: Optional[threading.Thread] = None + self._env = dict( + clean_environ(), + GIT_HTTP_EXPORT_ALL="1", + GIT_PROJECT_ROOT=str(root), + HOME=str(root), + USERPROFILE=str(root), + ) + + @property + def url(self) -> str: + """Base URL of the server (without trailing slash).""" + assert self._httpd, "server not started" + host, port = self._httpd.server_address[:2] + return f"http://{host}:{port}" + + def start(self) -> None: + """Start serving in a background thread.""" + server = self + + class Handler(_GitHttpHandler): + git_server = server + + self._httpd = _ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread( + target=self._httpd.serve_forever, name="git-http-server" + ) + self._thread.daemon = True + self._thread.start() + + def stop(self) -> None: + """Stop the server.""" + if self._httpd: + self._httpd.shutdown() + self._httpd.server_close() + self._httpd = None + if self._thread: + self._thread.join() + self._thread = None + + def record(self, method: str, path: str, status: int) -> None: + with self._lock: + self.requests.append(Request(method, path, status)) + + def requests_for(self, prefix: str) -> List[Request]: + """Return the requests whose path starts with `prefix`.""" + with self._lock: + return [r for r in self.requests if r.path.startswith(prefix)] + + def run_backend( + self, + method: str, + path: str, + query: str, + headers: Mapping[str, str], + body: bytes, + ) -> bytes: + """Run `git http-backend` as a CGI script and return its output.""" + env = dict( + self._env, + REQUEST_METHOD=method, + PATH_INFO=path, + QUERY_STRING=query, + CONTENT_TYPE=headers.get("Content-Type", ""), + CONTENT_LENGTH=str(len(body)), + REMOTE_ADDR="127.0.0.1", + ) + # Needed for git protocol v2. + if headers.get("Git-Protocol"): + env["GIT_PROTOCOL"] = headers["Git-Protocol"] + if headers.get("Content-Encoding"): + env["HTTP_CONTENT_ENCODING"] = headers["Content-Encoding"] + proc = subprocess.run( + ["git", "http-backend"], + input=body, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=COMMAND_TIMEOUT, + ) + if proc.stderr: + sys.stderr.write(proc.stderr.decode("utf-8", "replace")) + return proc.stdout + + +class _GitHttpHandler(http.server.BaseHTTPRequestHandler): + """Request handler for GitHttpServer.""" + + git_server: GitHttpServer + + # Keep the protocol simple: one request per connection. + protocol_version = "HTTP/1.0" + + def log_message(self, *args: object) -> None: + """Silence the default stderr logging.""" + + def _read_body(self) -> bytes: + # Respond to libcurl's `Expect: 100-continue` immediately so chunked + # POSTs don't wait for curl's 1s expect timeout under HTTP/1.0. + if self.headers.get("Expect", "").lower() == "100-continue": + self.send_response_only(100) + self.end_headers() + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + chunks = [] + while True: + line = self.rfile.readline() + if not line: + break + size = int(line.split(b";")[0].strip(), 16) + if not size: + # Consume the trailer section. + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + break + chunks.append(self.rfile.read(size)) + self.rfile.readline() + return b"".join(chunks) + length = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(length) if length else b"" + + @property + def _request_path(self) -> str: + """The URL-decoded request path (like CGI's PATH_INFO).""" + return urllib.parse.unquote(self.path.partition("?")[0]) + + def _send(self, status: int, headers: Dict[str, str], body: bytes) -> None: + self.git_server.record(self.command, self._request_path, status) + self.send_response(status) + for key, value in headers.items(): + self.send_header(key, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _serve_bundle(self, path: str) -> None: + repo = path[: -len("/clone.bundle")].strip("/") + if ".." not in repo.split("/"): + for candidate in (f"{repo}.git", repo): + bundle = self.git_server.root / candidate / "clone.bundle" + if bundle.is_file(): + self._send( + 200, + {"Content-Type": "application/octet-stream"}, + bundle.read_bytes(), + ) + return + self._send(404, {"Content-Type": "text/plain"}, b"no bundle\n") + + def _handle(self) -> None: + path = self._request_path + query = self.path.partition("?")[2] + if path.endswith("/clone.bundle"): + self._serve_bundle(path) + return + + body = self._read_body() + output = self.git_server.run_backend( + self.command, path, query, self.headers, body + ) + # Split the CGI response into headers & body at the first blank line. + match = re.search(rb"\r?\n\r?\n", output) + if not match: + self._send(500, {}, b"invalid CGI response\n") + return + sep, body_start = match.span() + status = 200 + headers = {} + for line in output[:sep].decode("latin-1").splitlines(): + key, _, value = line.partition(":") + if key.lower() == "status": + status = int(value.split()[0]) + elif key and key.lower() != "content-length": + headers[key] = value.strip() + self._send(status, headers, output[body_start:]) + + do_GET = _handle + do_POST = _handle + do_HEAD = _handle + + class Gitlink(NamedTuple): """A gitlink (submodule/superproject) tree entry pointing at a commit.""" @@ -170,7 +418,7 @@ FileContent = Optional[Union[str, bytes, Blob, Gitlink]] class ServerRepo: - """A bare repository.""" + """A bare repository hosted by GitHttpServer.""" # Fake commit timestamp; see commit(). _clock = 1700000000 @@ -293,7 +541,149 @@ class ServerRepo: self.git("tag", "-a", "-m", name, name, rev) def write_bundle(self) -> Path: - """Write a clone.bundle (all branches & tags) in the repo.""" + """Write a clone.bundle (all branches & tags) served by the server.""" bundle = self.path / "clone.bundle" self.git("bundle", "create", str(bundle), "--branches", "--tags") return bundle + + +def _manifest_files( + content: Union[str, Mapping[str, FileContent]], +) -> Mapping[str, FileContent]: + """Normalize default.xml contents or a map of files to a map of files.""" + return {"default.xml": content} if isinstance(content, str) else content + + +class GitHost: + """A namespace of repositories on a GitHttpServer. + + Each test gets its own namespace so tests don't interfere with each other + while sharing a single server. + """ + + def __init__(self, server: GitHttpServer, prefix: str) -> None: + self.server = server + self.prefix = prefix + (server.root / prefix).mkdir(parents=True, exist_ok=True) + + @property + def url(self) -> str: + """Base URL for this namespace (without trailing slash).""" + return f"{self.server.url}/{self.prefix}" + + def path_of(self, name: str) -> Path: + return self.server.root / self.prefix / f"{name}.git" + + def url_of(self, name: str) -> str: + return f"{self.url}/{name}" + + def create_repo( + self, + name: str, + files: Optional[Mapping[str, FileContent]] = None, + branch: str = "main", + ) -> ServerRepo: + """Create a repository, optionally with an initial commit.""" + repo = ServerRepo.create( + self.path_of(name), self.url_of(name), default_branch=branch + ) + if files is not None: + repo.commit(files, branch=branch, message=f"{name}: initial") + return repo + + @property + def manifest_url(self) -> str: + """URL of the manifest repo created by create_manifest.""" + return self.url_of(MANIFEST_REPO) + + def create_manifest( + self, + content: Union[str, Mapping[str, FileContent]], + branch: str = "main", + ) -> ServerRepo: + """Create the manifest repo. + + Args: + content: The default.xml contents, or a map of files. + branch: The manifest branch. + """ + return self.create_repo( + MANIFEST_REPO, _manifest_files(content), branch=branch + ) + + def update_manifest( + self, + content: Union[str, Mapping[str, FileContent]], + branch: str = "main", + ) -> str: + """Commit to the manifest repo. + + Args: + content: The new default.xml contents, or a map of files. + branch: The manifest branch. + + Returns: + The new manifest commit id. + """ + repo = ServerRepo( + self.path_of(MANIFEST_REPO), + self.manifest_url, + default_branch=branch, + ) + return repo.commit(_manifest_files(content), branch=branch) + + def fork(self, src: ServerRepo, name: str) -> ServerRepo: + """Create a cheap copy of `src` sharing its objects via alternates. + + Use this before mutating a session-wide repo (e.g. the repo source). + """ + repo = ServerRepo.create( + self.path_of(name), + self.url_of(name), + default_branch=src.default_branch, + ) + alternates = repo.path / "objects" / "info" / "alternates" + # Use forward slashes & LF (write_text would emit CRLF on Windows), + # which git accepts on every platform. + objects = (src.path / "objects").as_posix() + alternates.write_bytes(f"{objects}\n".encode()) + refs = src.git("for-each-ref", "--format=%(objectname) %(refname)") + for line in refs.splitlines(): + oid, ref = line.split() + repo.set_ref(ref, oid) + return repo + + def requests(self) -> List[Request]: + """All requests received for this namespace.""" + return self.server.requests_for(f"/{self.prefix}/") + + +def manifest_xml(*elements: str, default: Optional[str] = None) -> str: + """Build a manifest with an `origin` remote & a default element. + + Args: + elements: Extra XML elements (projects, etc...) to include. + default: Attributes for the element. + """ + if default is None: + default = 'remote="origin" revision="main"' + body = "".join(f" {x}\n" for x in elements) + return ( + '\n' + "\n" + ' \n' + f" \n" + f"{body}" + "\n" + ) + + +@contextlib.contextmanager +def serve(root: Path) -> Iterator[GitHttpServer]: + """Run a GitHttpServer for the duration of the context.""" + server = GitHttpServer(root) + server.start() + try: + yield server + finally: + server.stop() diff --git a/tests/e2e/test_harness.py b/tests/e2e/test_harness.py index de73b2b25..30aae4372 100644 --- a/tests/e2e/test_harness.py +++ b/tests/e2e/test_harness.py @@ -14,8 +14,10 @@ """Tests for the E2E harness helpers themselves.""" +import http.client from pathlib import Path -from typing import Mapping +from typing import Iterator, Mapping +import urllib.parse import harness import pytest @@ -27,6 +29,15 @@ pytestmark = pytest.mark.skipif( ) +@pytest.fixture +def git_server(tmp_path: Path) -> Iterator[harness.GitHttpServer]: + """A git smart HTTP server for a single test.""" + if not harness.has_http_backend(): + pytest.skip("git http-backend is not available") + with harness.serve(tmp_path) as server: + yield server + + @pytest.mark.parametrize( "key,value", [ @@ -58,3 +69,40 @@ def test_commit_replace_file_with_dir( repo.commit({"foo": "file"}) repo.commit(files) assert repo.git("ls-tree", "-r", "--name-only", "main") == "foo/bar" + + +@pytest.mark.parametrize( + "chunked,expect_continue", + [(False, False), (True, False), (True, True)], +) +def test_http_post_body( + git_server: harness.GitHttpServer, chunked: bool, expect_continue: bool +) -> None: + """POST bodies reach git http-backend however curl encodes them.""" + name = f"post-{int(chunked)}{int(expect_continue)}.git" + repo = harness.ServerRepo.create(git_server.root / name, "unused") + commit = repo.commit({"file": "x"}) + # A protocol v2 ls-refs request: a command pkt-line & a flush-pkt. + body = b"0014command=ls-refs\n0000" + headers = { + "Content-Type": "application/x-git-upload-pack-request", + "Git-Protocol": "version=2", + } + if expect_continue: + headers["Expect"] = "100-continue" + url = urllib.parse.urlsplit(git_server.url) + conn = http.client.HTTPConnection(url.hostname, url.port, timeout=30) + try: + # http.client skips any interim `100 Continue` response. + conn.request( + "POST", + f"/{name}/git-upload-pack", + body=iter([body[:10], body[10:]]) if chunked else body, + headers=headers, + encode_chunked=chunked, + ) + resp = conn.getresponse() + assert resp.status == 200 + assert f"{commit} refs/heads/main".encode() in resp.read() + finally: + conn.close()