From e045aa19d850af3d45eb835b3f320cd5b3d99596 Mon Sep 17 00:00:00 2001 From: John Cai Date: Mon, 17 Aug 2026 17:15:33 -0500 Subject: [PATCH] Adding sparse checkout functionality Sparse checkout allows Git to check out only a portion of the repository. This is particularly helpful for monorepository workflows where CI jobs only need a portion of the repository to run tests. This speeds up the CI jobs, and puts less load on the Git server as it relieves it from having to pack up all the objects. In the Git project, though it is still labeled experimental, the feature has been stable for a number of years and is unlikely to change at this point. This change allows configuring Git to use sparse checkout and provide a list of directories for the sync command to retrieve. Projects with sparse paths default to a blob:none partial clone so the excluded paths are never fetched, unless a filter is already configured or the project is excluded from partial clones. Both the plain and the git-worktree checkout flows configure sparse-checkout before populating the working tree. Change-Id: I772c7db5209b345306be378771c8f2040832bca1 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/535061 Reviewed-by: Gavin Mak Tested-by: John Cai Commit-Queue: John Cai Reviewed-by: Mike Frysinger --- docs/manifest-format.md | 75 ++++- man/repo-manifest.1 | 75 ++++- manifest_xml.py | 33 +++ project.py | 200 ++++++++++--- tests/test_manifest_xml.py | 100 +++++++ tests/test_project.py | 576 ++++++++++++++++++++++++++++++++++++- 6 files changed, 1004 insertions(+), 55 deletions(-) diff --git a/docs/manifest-format.md b/docs/manifest-format.md index 18420b8ea..4cf5108c1 100644 --- a/docs/manifest-format.md +++ b/docs/manifest-format.md @@ -74,7 +74,8 @@ following DTD: + linkfile*, + sparse-path*)> @@ -102,9 +103,13 @@ following DTD: + + + + linkfile*, + sparse-path*)> @@ -424,10 +429,11 @@ attributes of an existing project without completely replacing the existing project definition. This makes the local manifest more robust against changes to the original manifest. -The `extend-project` element can also contain `annotation`, `copyfile`, and -`linkfile` child elements. These are added to the project's definition. A -`copyfile` or `linkfile` with a `dest` that already exists in the project -will overwrite the original. +The `extend-project` element can also contain `annotation`, `copyfile`, +`linkfile`, and `sparse-path` child elements. These are added to the project's +definition. A `copyfile` or `linkfile` with a `dest` that already exists in the +project will overwrite the original. There is no way to remove a `sparse-path` +that the original project definition declared. Attribute `path`: If specified, limit the change to projects checked out at the specified path, rather than all projects with the given name. @@ -471,6 +477,63 @@ attribute "keep" which accepts the case insensitive values "true" (default) or "false". This attribute determines whether or not the annotation will be kept when exported with the manifest subcommand. +### Element sparse-path + +Zero or more sparse-path elements may be specified as children of a +project element or an extend-project element. Each element specifies a +path to include in the working directory. + +Requires Git 2.25.0 or later, which is when `git sparse-checkout` and cone mode +were introduced. Declaring a sparse-path with an older Git fails the sync for +that project. + +Declaring at least one sparse-path is what enables Git sparse-checkout for +the project: the listed paths are checked out, and other directories are left +out of the working directory. Cone mode also checks out the files sitting +directly in the project's top-level directory and in each leading directory of +a sparse-path, so `src/backend` brings along the top-level files and the files +directly under `src/` too. A project with no sparse-path elements is checked +out in full. +The checkout uses [cone mode](https://git-scm.com/docs/git-sparse-checkout#_internalscone_mode_handling) +for better performance, which means only directory paths are supported; +individual files and globs are not. Cone mode does no pattern matching, so `*`, +`?`, `[`, `]`, and `\` are not glob syntax in a `path`. In fact, Git 2.36.0 and +later reject them with `specify directories rather than patterns`, which +surfaces as a sync failure for the project. Earlier versions starting with +2.25.0 have no such check and are treated as literals. + +Sparse-checkout is configured only when the project's working directory is +first created. Changes to `sparse-path` in the manifest (added, changed, or +removed) are not applied to a project that is already checked out. To change +the paths in an existing checkout, run `git sparse-checkout` in the project +directly, or remove the project's working directory and sync again. + +Because sparse-checkout only prunes the working directory, a project with at +least one sparse-path is fetched with `--filter=blob:none` so that blobs which +will never be checked out are not downloaded. Without the filter Git still +downloads every blob in the project and sparse-checkout merely declines to +write most of them to disk, so the filter is what turns sparse-checkout into a +saving in transfer size and server load. The filter from +`repo init --partial-clone --clone-filter` takes precedence, and projects listed +in `repo init --partial-clone-exclude` are fetched in full as usual. Like +sparse-checkout itself, the implied filter only applies when the project is +first fetched (or is already a partial clone), so adding a sparse-path to an +existing full checkout does not convert it to a partial clone. Mirrors are never +filtered. + +Because that filter is only useful if the objects it excludes are not fetched +by some other route, a project that takes the implied `--filter=blob:none` also +skips the `/clone.bundle` download: the bundle is an unfiltered pack of every +object, so fetching it would undo the filter. Projects that set the filter +explicitly, or that are listed in `repo init --partial-clone-exclude`, keep +using the clone bundle. + +Attribute `path`: A path relative to the project root to include in the +sparse-checkout. The path is validated like `` and `` +paths: it must stay inside the project, so absolute paths, `.` or `..` +components, `~`, and `.git` or `.repo*` components are all rejected. A trailing +`/` is allowed. + ### Element copyfile Zero or more copyfile elements may be specified as children of a diff --git a/man/repo-manifest.1 b/man/repo-manifest.1 index 49923b9d3..5665415fd 100644 --- a/man/repo-manifest.1 +++ b/man/repo-manifest.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man. -.TH REPO "1" "August 2026" "repo manifest" "Repo Manual" +.TH REPO "1" "October 2026" "repo manifest" "Repo Manual" .SH NAME repo \- repo manifest - manual page for repo manifest .SH SYNOPSIS @@ -149,7 +149,8 @@ include*)> +linkfile*, +sparse\-path*)> .TP @@ -201,10 +202,14 @@ CDATA #IMPLIED> +.IP + + .TP +linkfile*, +sparse\-path*)> .TP @@ -514,10 +519,11 @@ of an existing project without completely replacing the existing project definition. This makes the local manifest more robust against changes to the original manifest. .PP -The `extend\-project` element can also contain `annotation`, `copyfile`, and -`linkfile` child elements. These are added to the project's definition. A -`copyfile` or `linkfile` with a `dest` that already exists in the project will -overwrite the original. +The `extend\-project` element can also contain `annotation`, `copyfile`, +`linkfile`, and `sparse\-path` child elements. These are added to the project's +definition. A `copyfile` or `linkfile` with a `dest` that already exists in the +project will overwrite the original. There is no way to remove a `sparse\-path` +that the original project definition declared. .PP Attribute `path`: If specified, limit the change to projects checked out at the specified path, rather than all projects with the given name. @@ -561,6 +567,61 @@ insensitive values "true" (default) or "false". This attribute determines whether or not the annotation will be kept when exported with the manifest subcommand. .PP +Element sparse\-path +.PP +Zero or more sparse\-path elements may be specified as children of a project +element or an extend\-project element. Each element specifies a path to include +in the working directory. +.PP +Requires Git 2.25.0 or later, which is when `git sparse\-checkout` and cone mode +were introduced. Declaring a sparse\-path with an older Git fails the sync for +that project. +.PP +Declaring at least one sparse\-path is what enables Git sparse\-checkout for the +project: the listed paths are checked out, and other directories are left out of +the working directory. Cone mode also checks out the files sitting directly in +the project's top\-level directory and in each leading directory of a +sparse\-path, so `src/backend` brings along the top\-level files and the files +directly under `src/` too. A project with no sparse\-path elements is checked out +in full. The checkout uses [cone +mode](https://git\-scm.com/docs/git\-sparse\-checkout#_internalscone_mode_handling) +for better performance, which means only directory paths are supported; +individual files and globs are not. Cone mode does no pattern matching, so `*`, +`?`, `[`, `]`, and `\e` are not glob syntax in a `path`. In fact, Git 2.36.0 and +later reject them with `specify directories rather than patterns`, which +surfaces as a sync failure for the project. Earlier versions starting with +2.25.0 have no such check and are treated as literals. +.PP +Sparse\-checkout is configured only when the project's working directory is first +created. Changes to `sparse\-path` in the manifest (added, changed, or removed) +are not applied to a project that is already checked out. To change the paths in +an existing checkout, run `git sparse\-checkout` in the project directly, or +remove the project's working directory and sync again. +.PP +Because sparse\-checkout only prunes the working directory, a project with at +least one sparse\-path is fetched with `\-\-filter=blob:none` so that blobs which +will never be checked out are not downloaded. Without the filter Git still +downloads every blob in the project and sparse\-checkout merely declines to write +most of them to disk, so the filter is what turns sparse\-checkout into a saving +in transfer size and server load. The filter from `repo init \fB\-\-partial\-clone\fR +\fB\-\-clone\-filter\fR` takes precedence, and projects listed in `repo init +\fB\-\-partial\-clone\-exclude\fR` are fetched in full as usual. Like sparse\-checkout +itself, the implied filter only applies when the project is first fetched (or is +already a partial clone), so adding a sparse\-path to an existing full checkout +does not convert it to a partial clone. Mirrors are never filtered. +.PP +Because that filter is only useful if the objects it excludes are not fetched by +some other route, a project that takes the implied `\-\-filter=blob:none` also +skips the `/clone.bundle` download: the bundle is an unfiltered pack of every +object, so fetching it would undo the filter. Projects that set the filter +explicitly, or that are listed in `repo init \fB\-\-partial\-clone\-exclude\fR`, keep +using the clone bundle. +.PP +Attribute `path`: A path relative to the project root to include in the +sparse\-checkout. The path is validated like `` and `` paths: +it must stay inside the project, so absolute paths, `.` or `..` components, `~`, +and `.git` or `.repo*` components are all rejected. A trailing `/` is allowed. +.PP Element copyfile .PP Zero or more copyfile elements may be specified as children of a project diff --git a/manifest_xml.py b/manifest_xml.py index 58da71f1c..89ac258cc 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -754,6 +754,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md le.setAttribute("dest", lf.dest) e.appendChild(le) + for sp in p.sparse_paths: + se = doc.createElement("sparse-path") + se.setAttribute("path", sp) + e.appendChild(se) + groups = p.groups - {"all", f"name:{p.name}", f"path:{p.relpath}"} if groups: e.setAttribute("groups", ",".join(sorted(groups))) @@ -846,6 +851,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md "annotation", "copyfile", "linkfile", + "sparse-path", } doc = self.ToXml(**kwargs) @@ -1569,6 +1575,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._ParseLinkFile(p, n) elif n.nodeName == "annotation": self._ParseAnnotation(p, n) + elif n.nodeName == "sparse-path": + self._ParseSparsePath(p, n) if node.nodeName == "repo-hooks": # Only one project can be the hooks project @@ -2033,6 +2041,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._ParseLinkFile(project, n) elif n.nodeName == "annotation": self._ParseAnnotation(project, n) + elif n.nodeName == "sparse-path": + self._ParseSparsePath(project, n) elif n.nodeName == "project": project.subprojects.append( self._ParseProject(n, parent=project) @@ -2307,6 +2317,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md ) element.AddAnnotation(name, value, keep) + def _ParseSparsePath( + self, project: Project, node: xml.dom.minidom.Element + ) -> None: + """Parses a element and adds it to the project. + + Args: + project: The Project object to add the sparse path to. + node: The XML node representing the element. + + Raises: + ManifestInvalidPathError: |path| is not usable as a project + relative path. + """ + path = self._reqatt(node, "path") + # Sparse paths name directories inside the project checkout, so they + # get the same scrutiny as & sources. + msg = self._CheckLocalPath(path, dir_ok=True) + if msg: + raise error.ManifestInvalidPathError( + f' invalid "path": {path}: {msg}' + ) + project.sparse_paths.append(path) + def _get_remote(self, node): name = node.getAttribute("remote") if not name: diff --git a/project.py b/project.py index b1fd93108..d9575ae77 100644 --- a/project.py +++ b/project.py @@ -652,6 +652,7 @@ class Project: dest_branch=None, optimized_fetch=False, retry_fetches=0, + sparse_paths: Optional[List[str]] = None, ): """Init a Project object. @@ -686,6 +687,8 @@ class Project: only fetch from the remote if the sha1 is not present locally. retry_fetches: Retry remote fetches n times upon receiving transient error with exponential backoff and jitter. + sparse_paths: List of paths from manifest.xml's `sparse-path` child + elements. A non-empty list enables sparse checkout. """ self.client = self.manifest = manifest self.name = name @@ -710,6 +713,7 @@ class Project: self.gitlink_path = gitlink_path self.optimized_fetch = optimized_fetch self.retry_fetches = max(0, retry_fetches) + self.sparse_paths = sparse_paths if sparse_paths is not None else [] self.subprojects = [] self.snapshots = {} @@ -1878,6 +1882,27 @@ class Project: if clone_bundle and os.path.exists(self.objdir): clone_bundle = False + # Sparse checkout only prunes the working tree, so include a partial + # clone by default to avoid fetching blobs that will never be + # checked out. The clone bundle is an unfiltered pack of every object, + # so downloading it would undo the filter; skip it to match how + # `repo init` pairs the two. Explicitly configured partial clones or + # exclusions still take precedence. Only new projects or ones that are + # already partial clones get the default: sparse-checkout is not + # applied to existing checkouts, so converting a full one would gain + # nothing, and mirrors are never partial clones. + if ( + self.sparse_paths + and not clone_filter + and not self.manifest.IsMirror + and ( + not os.path.exists(self.objdir) + or self.config.GetString("extensions.partialclone") + ) + ): + clone_filter = "blob:none" + clone_bundle = False + if partial_clone_exclude is None: partial_clone_exclude = set() if self.name in partial_clone_exclude: @@ -4726,46 +4751,84 @@ class Project: self.bare_git.worktree( "add", "-ff", - "--checkout", + "--no-checkout" if self.sparse_paths else "--checkout", "--detach", "--lock", self.worktree, self.GetRevisionId(), ) - # Rewrite the internal state files to use relative paths between the - # checkouts & worktrees. - dotgit = os.path.join(self.worktree, ".git") - with open(dotgit) as fp: - # Figure out the checkout->worktree path. - setting = fp.read() - assert setting.startswith("gitdir:") - git_worktree_path = setting.split(":", 1)[1].strip() + # `worktree add` has registered and locked the worktree, so any + # failure from here on has to unwind it. Leaving a populated + # directory behind would wedge the project: `worktree prune` skips + # locked entries and `worktree add` refuses a non-empty path even + # with -ff, so every later sync would fail until the user cleaned + # up by hand. Catch BaseException so a Ctrl-C during the + # (potentially slow, blob-fetching) checkout unwinds too. + try: + # Rewrite the internal state files to use relative paths between the + # checkouts & worktrees. + dotgit = os.path.join(self.worktree, ".git") + with open(dotgit) as fp: + # Figure out the checkout->worktree path. + setting = fp.read() + assert setting.startswith("gitdir:") + git_worktree_path = setting.split(":", 1)[1].strip() - # `gitdir` maybe be either relative or absolute depending on the - # behavior of the local copy of git, so only convert the path to - # relative if it needs to be converted. - if os.path.isabs(git_worktree_path): - # Some platforms (e.g. Windows) won't let us update dotgit in situ - # because of file permissions. Delete it and recreate it from - # scratch to avoid. - platform_utils.remove(dotgit) - # Use relative path from checkout->worktree & maintain Unix line - # endings on all OS's to match git behavior. - with open(dotgit, "w", newline="\n") as fp: - print( - "gitdir:", - os.path.relpath(git_worktree_path, self.worktree), - file=fp, + # `gitdir` maybe be either relative or absolute depending on the + # behavior of the local copy of git, so only convert the path to + # relative if it needs to be converted. + if os.path.isabs(git_worktree_path): + # Some platforms (e.g. Windows) won't let us update + # dotgit in situ because of file permissions. Delete it + # and recreate it from scratch to avoid. + platform_utils.remove(dotgit) + # Use relative path from checkout->worktree & maintain Unix line + # endings on all OS's to match git behavior. + with open(dotgit, "w", newline="\n") as fp: + print( + "gitdir:", + os.path.relpath(git_worktree_path, self.worktree), + file=fp, + ) + # Use relative path from worktree->checkout & maintain Unix line + # endings on all OS's to match git behavior. + with open( + os.path.join(git_worktree_path, "gitdir"), "w", newline="\n" + ) as fp: + print(os.path.relpath(dotgit, git_worktree_path), file=fp) + + # Configure sparse-checkout before populating the worktree. This + # only runs when the worktree is first created, so later changes to + # `sparse-path` in the manifest are not applied to existing + # checkouts. See docs/manifest-format.md. There is no stale state + # to clear first: git keeps both core.sparseCheckout and the + # pattern file under the per-worktree directory that the + # `worktree add` above just recreated. + if self.sparse_paths: + self._ConfigureSparseCheckout(self.sparse_paths) + cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] + if git_command.GitCommand(self, cmd).Wait() != 0: + raise error.GitError( + "Cannot initialize work tree for " + self.name, + project=self.name, + ) + + self._InitMRef() + except BaseException: + try: + self.bare_git.worktree( + "remove", "--force", "--force", self.worktree ) - # Use relative path from worktree->checkout & maintain Unix line - # endings on all OS's to match git behavior. - with open( - os.path.join(git_worktree_path, "gitdir"), "w", newline="\n" - ) as fp: - print(os.path.relpath(dotgit, git_worktree_path), file=fp) - - self._InitMRef() + except Exception as e: + # Never let a cleanup failure mask the original error. + logger.warning( + "warn: %s: cannot remove work tree %s: %s", + self.name, + self.worktree, + e, + ) + raise def _InitWorkTree(self, force_sync=False, submodules=False): """Setup the worktree .git path. @@ -4820,25 +4883,82 @@ class Project: git_refs.HEAD, self.GetRevisionId(), detach=True ) - # Finish checking out the worktree. - cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] try: + # Configure sparse-checkout before checking out the + # worktree. This only runs when the worktree is first + # created, so later changes to `sparse-path` in the + # manifest are not applied to existing checkouts. See + # docs/manifest-format.md. + if self.sparse_paths: + self._ConfigureSparseCheckout(self.sparse_paths) + elif self._IsSparseCheckoutConfigured(): + # The gitdir outlives the worktree, so a project that + # dropped `sparse-path` would otherwise keep checking + # out a pruned tree from the stale config. + self.work_git.sparse_checkout("disable") + + # Finish checking out the worktree. + cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] if git_command.GitCommand(self, cmd).Wait() != 0: raise error.GitError( "Cannot initialize work tree for " + self.name, project=self.name, ) - except Exception as e: - # Something went wrong with read-tree (perhaps fetching - # missing blobs), so remove .git to avoid half initialized - # workspace from which repo can't recover on its own. - platform_utils.remove(dotgit) - raise e + except BaseException: + # Something went wrong with sparse-checkout or read-tree + # (perhaps fetching missing blobs), so remove .git to avoid + # a half initialized workspace from which repo can't recover + # on its own. BaseException so a Ctrl-C mid-checkout + # unwinds too. + try: + platform_utils.remove(dotgit) + except Exception as e: + # Never let a cleanup failure mask the original error. + logger.warning( + "warn: %s: cannot remove %s: %s", + self.name, + dotgit, + e, + ) + raise if submodules: self._SyncSubmodules(quiet=True) self._CopyAndLinkFiles() + def _IsSparseCheckoutConfigured(self) -> bool: + """Whether this checkout carries leftover sparse-checkout state. + + `git sparse-checkout` records core.sparseCheckout in worktree-scoped + config, which self.config cannot see: that reads $GITDIR/config + directly, while git keeps the setting in $GITDIR/config.worktree once + it turns on extensions.worktreeConfig. The pattern file sits at a + fixed path though, so use it as the signal. `sparse-checkout disable` + leaves the file behind, so this stays true afterwards; that is fine + because disabling twice is a no-op. + """ + if not git_command.git_require((2, 25, 0)): + # Without the builtin there is nothing that could have written + # the file, and no `sparse-checkout disable` to undo it with. + return False + + return os.path.exists( + os.path.join(self.gitdir, "info", "sparse-checkout") + ) + + def _ConfigureSparseCheckout(self, sparse_paths: List[str]) -> None: + """Configure sparse-checkout for the project. + + Args: + sparse_paths: List of paths to include in sparse-checkout. + """ + if not sparse_paths: + return + + git_command.git_require((2, 25, 0), fail=True, msg="Sparse Checkout") + self.work_git.sparse_checkout("init", "--cone") + self.work_git.sparse_checkout("set", "--", *sparse_paths) + def _createDotGit(self, dotgit): """Initialize .git path. diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index a04972a1d..2894e2df5 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py @@ -933,6 +933,106 @@ class TestProjectElement: with pytest.raises(error.ManifestInvalidPathError): parse("ok", path) + def test_sparse_checkout(self, repo_client: RepoClient) -> None: + """Check sparse-checkout settings.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + + +""" + ) + assert len(manifest.projects) == 2 + result = {p.name: p for p in manifest.projects} + + assert result["test-name"].sparse_paths == ["src/main", "docs"] + assert result["no-sparse"].sparse_paths == [] + + def test_sparse_checkout_bad_paths(self, repo_client: RepoClient) -> None: + """Check rejects paths that escape the checkout.""" + + def parse(path: str) -> None: + path = repo_client.encode_xml_attr(path) + manifest = repo_client.get_xml_manifest( + f""" + + + + + + + +""" + ) + # Force the manifest to be parsed. + manifest.ToXml() + + # Verify the parser is valid by default to avoid buggy tests below. + parse("src/main") + + # Sparse paths name directories, so a trailing slash is fine. + parse("src/main/") + + # A missing path is rejected by a different codepath. + with pytest.raises(error.ManifestParseError): + parse("") + + for path in INVALID_FS_PATHS: + if not path or path.endswith("/") or path.endswith(os.path.sep): + continue + + with pytest.raises(error.ManifestInvalidPathError): + parse(path) + + def test_sparse_checkout_extend_project( + self, repo_client: RepoClient + ) -> None: + """Check sparse-path elements added via extend-project.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + + + +""" + ) + assert len(manifest.projects) == 1 + assert manifest.projects[0].sparse_paths == ["src/main", "docs"] + + def test_sparse_checkout_roundtrip(self, repo_client: RepoClient) -> None: + """Check sparse-path elements survive a manifest round-trip.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + +""" + ) + roundtrip = repo_client.get_xml_manifest(manifest.ToXml().toxml()) + assert roundtrip.projects[0].sparse_paths == ["src/main", "docs"] + + assert manifest.ToDict()["project"][0]["sparse-path"] == [ + {"path": "src/main"}, + {"path": "docs"}, + ] + class TestSuperProjectElement: """Tests for .""" diff --git a/tests/test_project.py b/tests/test_project.py index 414d94205..d7ce0a641 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -42,9 +42,11 @@ import project class FakeProject: """A fake for Project for basic functionality.""" - def __init__(self, worktree): + def __init__(self, worktree: str, gitdir: Optional[str] = None) -> None: self.worktree = worktree - self.gitdir = os.path.join(worktree, ".git") + self.gitdir = ( + gitdir if gitdir is not None else os.path.join(worktree, ".git") + ) self.name = "fakeproject" self.work_git = project.Project._GitGetByExec( self, bare=False, gitdir=self.gitdir @@ -4994,3 +4996,573 @@ class InitGitDirConfigTests(unittest.TestCase): self.assertTrue( os.path.exists(os.path.join(hooks_dir, "post-checkout")) ) + + +class SparseCheckoutTests(unittest.TestCase): + """Check sparse-checkout handling.""" + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_configure_sparse_checkout_with_list(self) -> None: + """Test configuring sparse-checkout with list of paths.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = FakeProject(tempdir) + + sparse_paths = ["src/backend", "src/shared"] + fakeproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + fakeproj, FakeProject + ) + ) + fakeproj._ConfigureSparseCheckout(sparse_paths) + + result = fakeproj.work_git.sparse_checkout("list") + paths = result.strip().split("\n") + self.assertIn("src/backend", paths) + self.assertIn("src/shared", paths) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_configure_sparse_checkout_delimits_paths(self) -> None: + """Paths go after `--` so a leading dash isn't read as an option.""" + proj = mock.MagicMock() + project.Project._ConfigureSparseCheckout(proj, ["src/main", "docs"]) + + self.assertEqual( + [ + mock.call("init", "--cone"), + mock.call("set", "--", "src/main", "docs"), + ], + proj.work_git.sparse_checkout.call_args_list, + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_sparse_checkout_filters_worktree(self) -> None: + """Test that sparse-checkout filters files from worktree.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + os.makedirs(os.path.join(tempdir, "docs")) + + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + with open(os.path.join(tempdir, "docs/guide.md"), "w") as f: + f.write("# Guide\n") + + fakeproj.work_git.add(".") + fakeproj.work_git.commit("-mInitial commit") + + fakeproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + fakeproj, FakeProject + ) + ) + fakeproj._ConfigureSparseCheckout(["src/main", "docs"]) + + fakeproj.work_git.read_tree("-mu", "HEAD") + + self.assertTrue( + os.path.exists(os.path.join(tempdir, "src/main/app.py")) + ) + self.assertTrue( + os.path.exists(os.path.join(tempdir, "docs/guide.md")) + ) + self.assertFalse( + os.path.exists(os.path.join(tempdir, "src/tests/test.py")) + ) + + def _setup_git_worktree_project( + self, tempdir: str, wtdir: str, sparse_paths: List[str] + ) -> FakeProject: + """Build a project checked out from a real repo via `git worktree`.""" + srcproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + with open(os.path.join(tempdir, "README"), "w") as f: + f.write("readme\n") + with open(os.path.join(tempdir, "src/version.txt"), "w") as f: + f.write("1.0\n") + + srcproj.work_git.add(".") + srcproj.work_git.commit("-mInitial commit") + revid = srcproj.work_git.rev_parse("HEAD").strip() + + # Resolve symlinks (e.g. /var -> /private/var on macOS) so the + # worktree<->gitdir relative paths git records stay valid. + worktree = os.path.join(os.path.realpath(wtdir), "proj") + wtproj = FakeProject( + worktree, + gitdir=os.path.join(os.path.realpath(tempdir), ".git"), + ) + wtproj.sparse_paths = sparse_paths + wtproj.GetRevisionId = lambda: revid + wtproj._InitMRef = lambda: None + wtproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + wtproj, FakeProject + ) + ) + wtproj._InitGitWorktree = project.Project._InitGitWorktree.__get__( + wtproj, FakeProject + ) + return wtproj + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_applies_sparse_checkout(self) -> None: + """Test that git-worktree checkouts honor sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + wtproj._InitGitWorktree() + + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/main/app.py")) + ) + self.assertFalse( + os.path.exists(os.path.join(worktree, "src/tests/test.py")) + ) + # Cone mode also keeps the files directly in the top-level + # directory and in each leading directory of a sparse-path. + self.assertTrue( + os.path.exists(os.path.join(worktree, "README")) + ) + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/version.txt")) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_on_sparse_failure(self) -> None: + """A failed sparse checkout leaves nothing for the next sync to trip + over.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # Fail after `worktree add` has registered and locked the + # worktree, which is what makes the cleanup non-trivial. + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + # assertRaisesRegex, not assertRaises: `bare_git.worktree` + # raises GitError too, so a bare check could not tell the + # original failure from one thrown by the cleanup itself. + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + # Nothing is left behind: no half-populated tree, and no + # locked registration pointing at it. + self.assertFalse(os.path.exists(worktree)) + self.assertNotIn( + worktree, wtproj.bare_git.worktree("list", "--porcelain") + ) + + # The whole point of rolling back: the next sync recovers on + # its own, without the user cleaning up by hand. + wtproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + wtproj, FakeProject + ) + ) + wtproj._InitGitWorktree() + + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/main/app.py")) + ) + self.assertFalse( + os.path.exists(os.path.join(worktree, "src/tests/test.py")) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_a_populated_worktree(self) -> None: + """Rollback also handles a failure after files were written.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # Fail *after* dirtying the worktree. This is the case + # `worktree remove --force --force` exists for: a plain + # remove refuses to discard a non-empty tree. + def populate_then_fail(_paths: List[str]) -> None: + with open(os.path.join(worktree, "stray.txt"), "w") as fp: + fp.write("half written\n") + raise error.GitError("sparse-checkout failed") + + wtproj._ConfigureSparseCheckout = populate_then_fail + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + self.assertFalse(os.path.exists(worktree)) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_on_interrupt(self) -> None: + """Ctrl-C unwinds too, not just ordinary errors.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # A blob:none checkout can spend minutes fetching, so an + # interrupt here is the likeliest failure of all -- and + # KeyboardInterrupt is a BaseException, not an Exception. + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=KeyboardInterrupt() + ) + with self.assertRaises(KeyboardInterrupt): + wtproj._InitGitWorktree() + + self.assertFalse(os.path.exists(worktree)) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_cleanup_failure_keeps_original_error( + self, + ) -> None: + """A failing rollback warns but never hides why the sync failed.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + + real_worktree = wtproj.bare_git.worktree + + def failing_worktree(*args: str) -> str: + if args and args[0] == "remove": + raise error.GitError("worktree remove failed") + return real_worktree(*args) + + wtproj.bare_git.worktree = failing_worktree + + with mock.patch.object(project, "logger") as logger_mock: + # The original error survives, not the cleanup's. + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + logger_mock.warning.assert_called_once() + + def _setup_work_tree_project( + self, tempdir: str, wtdir: str, sparse_paths: List[str] + ) -> FakeProject: + """Build a project whose .git symlinks to an existing gitdir.""" + srcproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + + srcproj.work_git.add(".") + srcproj.work_git.commit("-mInitial commit") + revid = srcproj.work_git.rev_parse("HEAD").strip() + + # Resolve symlinks (e.g. /var -> /private/var on macOS) so the + # worktree<->gitdir paths git records stay valid. + worktree = os.path.join(os.path.realpath(wtdir), "proj") + os.makedirs(worktree) + gitdir = os.path.join(os.path.realpath(tempdir), ".git") + + proj = FakeProject(worktree, gitdir=gitdir) + proj.sparse_paths = sparse_paths + proj.parent = None + proj.use_git_worktrees = False + proj.manifest = mock.MagicMock() + proj.GetRevisionId = lambda: revid + proj._CopyAndLinkFiles = lambda: None + proj._createDotGit = lambda dotgit: platform_utils.symlink( + os.path.relpath(gitdir, worktree), dotgit + ) + proj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__(proj, FakeProject) + ) + proj._IsSparseCheckoutConfigured = ( + project.Project._IsSparseCheckoutConfigured.__get__( + proj, FakeProject + ) + ) + proj._InitWorkTree = project.Project._InitWorkTree.__get__( + proj, FakeProject + ) + return proj + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_work_tree_applies_sparse_checkout(self) -> None: + """Test that plain (non-worktree) checkouts honor sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + + proj._InitWorkTree() + + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/main/app.py") + ) + ) + self.assertFalse( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_work_tree_disables_stale_sparse_checkout(self) -> None: + """Dropping sparse-path restores a full checkout on the next sync.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + proj._InitWorkTree() + self.assertFalse( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + + # The manifest drops and the user removes the + # checkout and syncs again. The gitdir -- and the sparse + # state git recorded in it -- outlives the worktree. + shutil.rmtree(proj.worktree) + os.makedirs(proj.worktree) + proj.sparse_paths = [] + self.assertTrue(proj._IsSparseCheckoutConfigured()) + + proj._InitWorkTree() + + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/main/app.py") + ) + ) + + # No skipUnless: the answer must be False on old git too, via the + # version gate rather than the file check. + def test_is_sparse_checkout_configured_tracks_the_pattern_file( + self, + ) -> None: + """The probe follows the file git actually writes.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project(tempdir, wtdir, []) + self.assertFalse(proj._IsSparseCheckoutConfigured()) + + # Assert against the real path rather than just the False + # case, so the probe cannot pass by always returning False. + pattern_file = os.path.join( + proj.gitdir, "info", "sparse-checkout" + ) + os.makedirs(os.path.dirname(pattern_file), exist_ok=True) + with open(pattern_file, "w") as fp: + fp.write("/*\n") + + self.assertEqual( + git_command.git_require((2, 25, 0)), + proj._IsSparseCheckoutConfigured(), + ) + + def test_init_work_tree_removes_dotgit_on_sparse_failure(self) -> None: + """Test that a sparse-checkout failure cleans up .git.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + proj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + + with self.assertRaises(error.GitError): + proj._InitWorkTree() + + self.assertFalse( + os.path.lexists(os.path.join(proj.worktree, ".git")) + ) + + +class SparseCheckoutFetchFilterTests(unittest.TestCase): + """Check the partial clone filter used for sparse-checkout projects.""" + + def _get_project( + self, tempdir: str, sparse_paths: List[str] + ) -> project.Project: + # Point objdir somewhere that does not exist yet: Sync_NetworkHalf + # skips the clone bundle outright once the shared object dir is + # present, which would mask what these tests are checking. + proj = _create_mock_project( + tempdir, + revisionExpr="1234abcd", + objdir=os.path.join(tempdir, "no-such-objdir"), + ) + proj.sparse_paths = sparse_paths + # Project config falls back to the (mocked) global config, which would + # otherwise report every key, extensions.partialclone included, as set. + proj.manifest.globalConfig.GetString.return_value = None + proj._CheckForImmutableRevision = mock.MagicMock(return_value=False) + proj.bare_git.rev_parse.return_value = "5678abcd" + proj._InitGitDir = mock.MagicMock() + proj._InitRemote = mock.MagicMock() + proj._InitMRef = mock.MagicMock() + proj._ApplyCloneBundle = mock.MagicMock(return_value=False) + proj._RemoteFetch = mock.MagicMock(return_value=True) + return proj + + def _fetch_filter( + self, proj: project.Project, **kwargs: object + ) -> Optional[str]: + """Return the clone_filter Sync_NetworkHalf passes down to fetch.""" + res = proj.Sync_NetworkHalf(clone_bundle=False, **kwargs) + self.assertTrue(res.success) + proj._RemoteFetch.assert_called_once() + return proj._RemoteFetch.call_args.kwargs["clone_filter"] + + def test_no_sparse_paths_keeps_no_filter(self) -> None: + """A project without sparse-path is not filtered.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, []) + self.assertIsNone(self._fetch_filter(proj)) + + def test_sparse_paths_default_to_blob_none(self) -> None: + """sparse-path implies a blob:none partial clone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertEqual("blob:none", self._fetch_filter(proj)) + + def test_explicit_clone_filter_wins(self) -> None: + """An explicit filter is not replaced by the default.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertEqual( + "blob:limit=1m", + self._fetch_filter(proj, clone_filter="blob:limit=1m"), + ) + + def test_partial_clone_exclude_wins(self) -> None: + """An excluded project is fetched in full despite sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertIsNone( + self._fetch_filter(proj, partial_clone_exclude={proj.name}) + ) + + def test_existing_full_checkout_keeps_no_filter(self) -> None: + """An existing full checkout is not converted to a partial clone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + os.makedirs(proj.objdir) + self.assertIsNone(self._fetch_filter(proj, is_new=True)) + + def test_existing_partial_clone_keeps_blob_none(self) -> None: + """An existing partial clone keeps the blob:none default.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + os.makedirs(proj.objdir) + proj.config.SetString("extensions.partialclone", "origin") + self.assertEqual("blob:none", self._fetch_filter(proj, is_new=True)) + + def test_mirror_keeps_no_filter(self) -> None: + """Mirrors are never given the implied filter.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + proj.manifest.IsMirror = True + self.assertIsNone(self._fetch_filter(proj)) + + def _bundle_applied(self, proj: project.Project, **kwargs: object) -> bool: + """Return whether Sync_NetworkHalf reached for the clone bundle.""" + res = proj.Sync_NetworkHalf(clone_bundle=True, **kwargs) + self.assertTrue(res.success) + return proj._ApplyCloneBundle.called + + def test_no_sparse_paths_keeps_clone_bundle(self) -> None: + """A project without sparse-path still uses the clone bundle.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, []) + self.assertTrue(self._bundle_applied(proj)) + + def test_sparse_paths_skip_clone_bundle(self) -> None: + """The bundle is unfiltered, so the blob:none default drops it.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertFalse(self._bundle_applied(proj)) + + def test_explicit_clone_filter_keeps_clone_bundle(self) -> None: + """Only the implied filter drops the bundle, not an explicit one.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertTrue( + self._bundle_applied(proj, clone_filter="blob:limit=1m") + ) + + def test_partial_clone_exclude_restores_clone_bundle(self) -> None: + """An excluded project gets its bundle back despite sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertTrue( + self._bundle_applied(proj, partial_clone_exclude={proj.name}) + )