diff --git a/docs/manifest-format.md b/docs/manifest-format.md index 18420b8ea..4cf5108c1 100644 --- a/docs/manifest-format.md +++ b/docs/manifest-format.md @@ -74,7 +74,8 @@ following DTD: + linkfile*, + sparse-path*)> @@ -102,9 +103,13 @@ following DTD: + + + + linkfile*, + sparse-path*)> @@ -424,10 +429,11 @@ attributes of an existing project without completely replacing the existing project definition. This makes the local manifest more robust against changes to the original manifest. -The `extend-project` element can also contain `annotation`, `copyfile`, and -`linkfile` child elements. These are added to the project's definition. A -`copyfile` or `linkfile` with a `dest` that already exists in the project -will overwrite the original. +The `extend-project` element can also contain `annotation`, `copyfile`, +`linkfile`, and `sparse-path` child elements. These are added to the project's +definition. A `copyfile` or `linkfile` with a `dest` that already exists in the +project will overwrite the original. There is no way to remove a `sparse-path` +that the original project definition declared. Attribute `path`: If specified, limit the change to projects checked out at the specified path, rather than all projects with the given name. @@ -471,6 +477,63 @@ attribute "keep" which accepts the case insensitive values "true" (default) or "false". This attribute determines whether or not the annotation will be kept when exported with the manifest subcommand. +### Element sparse-path + +Zero or more sparse-path elements may be specified as children of a +project element or an extend-project element. Each element specifies a +path to include in the working directory. + +Requires Git 2.25.0 or later, which is when `git sparse-checkout` and cone mode +were introduced. Declaring a sparse-path with an older Git fails the sync for +that project. + +Declaring at least one sparse-path is what enables Git sparse-checkout for +the project: the listed paths are checked out, and other directories are left +out of the working directory. Cone mode also checks out the files sitting +directly in the project's top-level directory and in each leading directory of +a sparse-path, so `src/backend` brings along the top-level files and the files +directly under `src/` too. A project with no sparse-path elements is checked +out in full. +The checkout uses [cone mode](https://git-scm.com/docs/git-sparse-checkout#_internalscone_mode_handling) +for better performance, which means only directory paths are supported; +individual files and globs are not. Cone mode does no pattern matching, so `*`, +`?`, `[`, `]`, and `\` are not glob syntax in a `path`. In fact, Git 2.36.0 and +later reject them with `specify directories rather than patterns`, which +surfaces as a sync failure for the project. Earlier versions starting with +2.25.0 have no such check and are treated as literals. + +Sparse-checkout is configured only when the project's working directory is +first created. Changes to `sparse-path` in the manifest (added, changed, or +removed) are not applied to a project that is already checked out. To change +the paths in an existing checkout, run `git sparse-checkout` in the project +directly, or remove the project's working directory and sync again. + +Because sparse-checkout only prunes the working directory, a project with at +least one sparse-path is fetched with `--filter=blob:none` so that blobs which +will never be checked out are not downloaded. Without the filter Git still +downloads every blob in the project and sparse-checkout merely declines to +write most of them to disk, so the filter is what turns sparse-checkout into a +saving in transfer size and server load. The filter from +`repo init --partial-clone --clone-filter` takes precedence, and projects listed +in `repo init --partial-clone-exclude` are fetched in full as usual. Like +sparse-checkout itself, the implied filter only applies when the project is +first fetched (or is already a partial clone), so adding a sparse-path to an +existing full checkout does not convert it to a partial clone. Mirrors are never +filtered. + +Because that filter is only useful if the objects it excludes are not fetched +by some other route, a project that takes the implied `--filter=blob:none` also +skips the `/clone.bundle` download: the bundle is an unfiltered pack of every +object, so fetching it would undo the filter. Projects that set the filter +explicitly, or that are listed in `repo init --partial-clone-exclude`, keep +using the clone bundle. + +Attribute `path`: A path relative to the project root to include in the +sparse-checkout. The path is validated like `` and `` +paths: it must stay inside the project, so absolute paths, `.` or `..` +components, `~`, and `.git` or `.repo*` components are all rejected. A trailing +`/` is allowed. + ### Element copyfile Zero or more copyfile elements may be specified as children of a diff --git a/man/repo-manifest.1 b/man/repo-manifest.1 index 49923b9d3..5665415fd 100644 --- a/man/repo-manifest.1 +++ b/man/repo-manifest.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man. -.TH REPO "1" "August 2026" "repo manifest" "Repo Manual" +.TH REPO "1" "October 2026" "repo manifest" "Repo Manual" .SH NAME repo \- repo manifest - manual page for repo manifest .SH SYNOPSIS @@ -149,7 +149,8 @@ include*)> +linkfile*, +sparse\-path*)> .TP @@ -201,10 +202,14 @@ CDATA #IMPLIED> +.IP + + .TP +linkfile*, +sparse\-path*)> .TP @@ -514,10 +519,11 @@ of an existing project without completely replacing the existing project definition. This makes the local manifest more robust against changes to the original manifest. .PP -The `extend\-project` element can also contain `annotation`, `copyfile`, and -`linkfile` child elements. These are added to the project's definition. A -`copyfile` or `linkfile` with a `dest` that already exists in the project will -overwrite the original. +The `extend\-project` element can also contain `annotation`, `copyfile`, +`linkfile`, and `sparse\-path` child elements. These are added to the project's +definition. A `copyfile` or `linkfile` with a `dest` that already exists in the +project will overwrite the original. There is no way to remove a `sparse\-path` +that the original project definition declared. .PP Attribute `path`: If specified, limit the change to projects checked out at the specified path, rather than all projects with the given name. @@ -561,6 +567,61 @@ insensitive values "true" (default) or "false". This attribute determines whether or not the annotation will be kept when exported with the manifest subcommand. .PP +Element sparse\-path +.PP +Zero or more sparse\-path elements may be specified as children of a project +element or an extend\-project element. Each element specifies a path to include +in the working directory. +.PP +Requires Git 2.25.0 or later, which is when `git sparse\-checkout` and cone mode +were introduced. Declaring a sparse\-path with an older Git fails the sync for +that project. +.PP +Declaring at least one sparse\-path is what enables Git sparse\-checkout for the +project: the listed paths are checked out, and other directories are left out of +the working directory. Cone mode also checks out the files sitting directly in +the project's top\-level directory and in each leading directory of a +sparse\-path, so `src/backend` brings along the top\-level files and the files +directly under `src/` too. A project with no sparse\-path elements is checked out +in full. The checkout uses [cone +mode](https://git\-scm.com/docs/git\-sparse\-checkout#_internalscone_mode_handling) +for better performance, which means only directory paths are supported; +individual files and globs are not. Cone mode does no pattern matching, so `*`, +`?`, `[`, `]`, and `\e` are not glob syntax in a `path`. In fact, Git 2.36.0 and +later reject them with `specify directories rather than patterns`, which +surfaces as a sync failure for the project. Earlier versions starting with +2.25.0 have no such check and are treated as literals. +.PP +Sparse\-checkout is configured only when the project's working directory is first +created. Changes to `sparse\-path` in the manifest (added, changed, or removed) +are not applied to a project that is already checked out. To change the paths in +an existing checkout, run `git sparse\-checkout` in the project directly, or +remove the project's working directory and sync again. +.PP +Because sparse\-checkout only prunes the working directory, a project with at +least one sparse\-path is fetched with `\-\-filter=blob:none` so that blobs which +will never be checked out are not downloaded. Without the filter Git still +downloads every blob in the project and sparse\-checkout merely declines to write +most of them to disk, so the filter is what turns sparse\-checkout into a saving +in transfer size and server load. The filter from `repo init \fB\-\-partial\-clone\fR +\fB\-\-clone\-filter\fR` takes precedence, and projects listed in `repo init +\fB\-\-partial\-clone\-exclude\fR` are fetched in full as usual. Like sparse\-checkout +itself, the implied filter only applies when the project is first fetched (or is +already a partial clone), so adding a sparse\-path to an existing full checkout +does not convert it to a partial clone. Mirrors are never filtered. +.PP +Because that filter is only useful if the objects it excludes are not fetched by +some other route, a project that takes the implied `\-\-filter=blob:none` also +skips the `/clone.bundle` download: the bundle is an unfiltered pack of every +object, so fetching it would undo the filter. Projects that set the filter +explicitly, or that are listed in `repo init \fB\-\-partial\-clone\-exclude\fR`, keep +using the clone bundle. +.PP +Attribute `path`: A path relative to the project root to include in the +sparse\-checkout. The path is validated like `` and `` paths: +it must stay inside the project, so absolute paths, `.` or `..` components, `~`, +and `.git` or `.repo*` components are all rejected. A trailing `/` is allowed. +.PP Element copyfile .PP Zero or more copyfile elements may be specified as children of a project diff --git a/manifest_xml.py b/manifest_xml.py index 58da71f1c..89ac258cc 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -754,6 +754,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md le.setAttribute("dest", lf.dest) e.appendChild(le) + for sp in p.sparse_paths: + se = doc.createElement("sparse-path") + se.setAttribute("path", sp) + e.appendChild(se) + groups = p.groups - {"all", f"name:{p.name}", f"path:{p.relpath}"} if groups: e.setAttribute("groups", ",".join(sorted(groups))) @@ -846,6 +851,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md "annotation", "copyfile", "linkfile", + "sparse-path", } doc = self.ToXml(**kwargs) @@ -1569,6 +1575,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._ParseLinkFile(p, n) elif n.nodeName == "annotation": self._ParseAnnotation(p, n) + elif n.nodeName == "sparse-path": + self._ParseSparsePath(p, n) if node.nodeName == "repo-hooks": # Only one project can be the hooks project @@ -2033,6 +2041,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._ParseLinkFile(project, n) elif n.nodeName == "annotation": self._ParseAnnotation(project, n) + elif n.nodeName == "sparse-path": + self._ParseSparsePath(project, n) elif n.nodeName == "project": project.subprojects.append( self._ParseProject(n, parent=project) @@ -2307,6 +2317,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md ) element.AddAnnotation(name, value, keep) + def _ParseSparsePath( + self, project: Project, node: xml.dom.minidom.Element + ) -> None: + """Parses a element and adds it to the project. + + Args: + project: The Project object to add the sparse path to. + node: The XML node representing the element. + + Raises: + ManifestInvalidPathError: |path| is not usable as a project + relative path. + """ + path = self._reqatt(node, "path") + # Sparse paths name directories inside the project checkout, so they + # get the same scrutiny as & sources. + msg = self._CheckLocalPath(path, dir_ok=True) + if msg: + raise error.ManifestInvalidPathError( + f' invalid "path": {path}: {msg}' + ) + project.sparse_paths.append(path) + def _get_remote(self, node): name = node.getAttribute("remote") if not name: diff --git a/project.py b/project.py index b1fd93108..d9575ae77 100644 --- a/project.py +++ b/project.py @@ -652,6 +652,7 @@ class Project: dest_branch=None, optimized_fetch=False, retry_fetches=0, + sparse_paths: Optional[List[str]] = None, ): """Init a Project object. @@ -686,6 +687,8 @@ class Project: only fetch from the remote if the sha1 is not present locally. retry_fetches: Retry remote fetches n times upon receiving transient error with exponential backoff and jitter. + sparse_paths: List of paths from manifest.xml's `sparse-path` child + elements. A non-empty list enables sparse checkout. """ self.client = self.manifest = manifest self.name = name @@ -710,6 +713,7 @@ class Project: self.gitlink_path = gitlink_path self.optimized_fetch = optimized_fetch self.retry_fetches = max(0, retry_fetches) + self.sparse_paths = sparse_paths if sparse_paths is not None else [] self.subprojects = [] self.snapshots = {} @@ -1878,6 +1882,27 @@ class Project: if clone_bundle and os.path.exists(self.objdir): clone_bundle = False + # Sparse checkout only prunes the working tree, so include a partial + # clone by default to avoid fetching blobs that will never be + # checked out. The clone bundle is an unfiltered pack of every object, + # so downloading it would undo the filter; skip it to match how + # `repo init` pairs the two. Explicitly configured partial clones or + # exclusions still take precedence. Only new projects or ones that are + # already partial clones get the default: sparse-checkout is not + # applied to existing checkouts, so converting a full one would gain + # nothing, and mirrors are never partial clones. + if ( + self.sparse_paths + and not clone_filter + and not self.manifest.IsMirror + and ( + not os.path.exists(self.objdir) + or self.config.GetString("extensions.partialclone") + ) + ): + clone_filter = "blob:none" + clone_bundle = False + if partial_clone_exclude is None: partial_clone_exclude = set() if self.name in partial_clone_exclude: @@ -4726,46 +4751,84 @@ class Project: self.bare_git.worktree( "add", "-ff", - "--checkout", + "--no-checkout" if self.sparse_paths else "--checkout", "--detach", "--lock", self.worktree, self.GetRevisionId(), ) - # Rewrite the internal state files to use relative paths between the - # checkouts & worktrees. - dotgit = os.path.join(self.worktree, ".git") - with open(dotgit) as fp: - # Figure out the checkout->worktree path. - setting = fp.read() - assert setting.startswith("gitdir:") - git_worktree_path = setting.split(":", 1)[1].strip() + # `worktree add` has registered and locked the worktree, so any + # failure from here on has to unwind it. Leaving a populated + # directory behind would wedge the project: `worktree prune` skips + # locked entries and `worktree add` refuses a non-empty path even + # with -ff, so every later sync would fail until the user cleaned + # up by hand. Catch BaseException so a Ctrl-C during the + # (potentially slow, blob-fetching) checkout unwinds too. + try: + # Rewrite the internal state files to use relative paths between the + # checkouts & worktrees. + dotgit = os.path.join(self.worktree, ".git") + with open(dotgit) as fp: + # Figure out the checkout->worktree path. + setting = fp.read() + assert setting.startswith("gitdir:") + git_worktree_path = setting.split(":", 1)[1].strip() - # `gitdir` maybe be either relative or absolute depending on the - # behavior of the local copy of git, so only convert the path to - # relative if it needs to be converted. - if os.path.isabs(git_worktree_path): - # Some platforms (e.g. Windows) won't let us update dotgit in situ - # because of file permissions. Delete it and recreate it from - # scratch to avoid. - platform_utils.remove(dotgit) - # Use relative path from checkout->worktree & maintain Unix line - # endings on all OS's to match git behavior. - with open(dotgit, "w", newline="\n") as fp: - print( - "gitdir:", - os.path.relpath(git_worktree_path, self.worktree), - file=fp, + # `gitdir` maybe be either relative or absolute depending on the + # behavior of the local copy of git, so only convert the path to + # relative if it needs to be converted. + if os.path.isabs(git_worktree_path): + # Some platforms (e.g. Windows) won't let us update + # dotgit in situ because of file permissions. Delete it + # and recreate it from scratch to avoid. + platform_utils.remove(dotgit) + # Use relative path from checkout->worktree & maintain Unix line + # endings on all OS's to match git behavior. + with open(dotgit, "w", newline="\n") as fp: + print( + "gitdir:", + os.path.relpath(git_worktree_path, self.worktree), + file=fp, + ) + # Use relative path from worktree->checkout & maintain Unix line + # endings on all OS's to match git behavior. + with open( + os.path.join(git_worktree_path, "gitdir"), "w", newline="\n" + ) as fp: + print(os.path.relpath(dotgit, git_worktree_path), file=fp) + + # Configure sparse-checkout before populating the worktree. This + # only runs when the worktree is first created, so later changes to + # `sparse-path` in the manifest are not applied to existing + # checkouts. See docs/manifest-format.md. There is no stale state + # to clear first: git keeps both core.sparseCheckout and the + # pattern file under the per-worktree directory that the + # `worktree add` above just recreated. + if self.sparse_paths: + self._ConfigureSparseCheckout(self.sparse_paths) + cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] + if git_command.GitCommand(self, cmd).Wait() != 0: + raise error.GitError( + "Cannot initialize work tree for " + self.name, + project=self.name, + ) + + self._InitMRef() + except BaseException: + try: + self.bare_git.worktree( + "remove", "--force", "--force", self.worktree ) - # Use relative path from worktree->checkout & maintain Unix line - # endings on all OS's to match git behavior. - with open( - os.path.join(git_worktree_path, "gitdir"), "w", newline="\n" - ) as fp: - print(os.path.relpath(dotgit, git_worktree_path), file=fp) - - self._InitMRef() + except Exception as e: + # Never let a cleanup failure mask the original error. + logger.warning( + "warn: %s: cannot remove work tree %s: %s", + self.name, + self.worktree, + e, + ) + raise def _InitWorkTree(self, force_sync=False, submodules=False): """Setup the worktree .git path. @@ -4820,25 +4883,82 @@ class Project: git_refs.HEAD, self.GetRevisionId(), detach=True ) - # Finish checking out the worktree. - cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] try: + # Configure sparse-checkout before checking out the + # worktree. This only runs when the worktree is first + # created, so later changes to `sparse-path` in the + # manifest are not applied to existing checkouts. See + # docs/manifest-format.md. + if self.sparse_paths: + self._ConfigureSparseCheckout(self.sparse_paths) + elif self._IsSparseCheckoutConfigured(): + # The gitdir outlives the worktree, so a project that + # dropped `sparse-path` would otherwise keep checking + # out a pruned tree from the stale config. + self.work_git.sparse_checkout("disable") + + # Finish checking out the worktree. + cmd = ["read-tree", "--reset", "-u", "-v", git_refs.HEAD] if git_command.GitCommand(self, cmd).Wait() != 0: raise error.GitError( "Cannot initialize work tree for " + self.name, project=self.name, ) - except Exception as e: - # Something went wrong with read-tree (perhaps fetching - # missing blobs), so remove .git to avoid half initialized - # workspace from which repo can't recover on its own. - platform_utils.remove(dotgit) - raise e + except BaseException: + # Something went wrong with sparse-checkout or read-tree + # (perhaps fetching missing blobs), so remove .git to avoid + # a half initialized workspace from which repo can't recover + # on its own. BaseException so a Ctrl-C mid-checkout + # unwinds too. + try: + platform_utils.remove(dotgit) + except Exception as e: + # Never let a cleanup failure mask the original error. + logger.warning( + "warn: %s: cannot remove %s: %s", + self.name, + dotgit, + e, + ) + raise if submodules: self._SyncSubmodules(quiet=True) self._CopyAndLinkFiles() + def _IsSparseCheckoutConfigured(self) -> bool: + """Whether this checkout carries leftover sparse-checkout state. + + `git sparse-checkout` records core.sparseCheckout in worktree-scoped + config, which self.config cannot see: that reads $GITDIR/config + directly, while git keeps the setting in $GITDIR/config.worktree once + it turns on extensions.worktreeConfig. The pattern file sits at a + fixed path though, so use it as the signal. `sparse-checkout disable` + leaves the file behind, so this stays true afterwards; that is fine + because disabling twice is a no-op. + """ + if not git_command.git_require((2, 25, 0)): + # Without the builtin there is nothing that could have written + # the file, and no `sparse-checkout disable` to undo it with. + return False + + return os.path.exists( + os.path.join(self.gitdir, "info", "sparse-checkout") + ) + + def _ConfigureSparseCheckout(self, sparse_paths: List[str]) -> None: + """Configure sparse-checkout for the project. + + Args: + sparse_paths: List of paths to include in sparse-checkout. + """ + if not sparse_paths: + return + + git_command.git_require((2, 25, 0), fail=True, msg="Sparse Checkout") + self.work_git.sparse_checkout("init", "--cone") + self.work_git.sparse_checkout("set", "--", *sparse_paths) + def _createDotGit(self, dotgit): """Initialize .git path. diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index a04972a1d..2894e2df5 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py @@ -933,6 +933,106 @@ class TestProjectElement: with pytest.raises(error.ManifestInvalidPathError): parse("ok", path) + def test_sparse_checkout(self, repo_client: RepoClient) -> None: + """Check sparse-checkout settings.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + + +""" + ) + assert len(manifest.projects) == 2 + result = {p.name: p for p in manifest.projects} + + assert result["test-name"].sparse_paths == ["src/main", "docs"] + assert result["no-sparse"].sparse_paths == [] + + def test_sparse_checkout_bad_paths(self, repo_client: RepoClient) -> None: + """Check rejects paths that escape the checkout.""" + + def parse(path: str) -> None: + path = repo_client.encode_xml_attr(path) + manifest = repo_client.get_xml_manifest( + f""" + + + + + + + +""" + ) + # Force the manifest to be parsed. + manifest.ToXml() + + # Verify the parser is valid by default to avoid buggy tests below. + parse("src/main") + + # Sparse paths name directories, so a trailing slash is fine. + parse("src/main/") + + # A missing path is rejected by a different codepath. + with pytest.raises(error.ManifestParseError): + parse("") + + for path in INVALID_FS_PATHS: + if not path or path.endswith("/") or path.endswith(os.path.sep): + continue + + with pytest.raises(error.ManifestInvalidPathError): + parse(path) + + def test_sparse_checkout_extend_project( + self, repo_client: RepoClient + ) -> None: + """Check sparse-path elements added via extend-project.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + + + +""" + ) + assert len(manifest.projects) == 1 + assert manifest.projects[0].sparse_paths == ["src/main", "docs"] + + def test_sparse_checkout_roundtrip(self, repo_client: RepoClient) -> None: + """Check sparse-path elements survive a manifest round-trip.""" + manifest = repo_client.get_xml_manifest( + """ + + + + + + + + +""" + ) + roundtrip = repo_client.get_xml_manifest(manifest.ToXml().toxml()) + assert roundtrip.projects[0].sparse_paths == ["src/main", "docs"] + + assert manifest.ToDict()["project"][0]["sparse-path"] == [ + {"path": "src/main"}, + {"path": "docs"}, + ] + class TestSuperProjectElement: """Tests for .""" diff --git a/tests/test_project.py b/tests/test_project.py index 414d94205..d7ce0a641 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -42,9 +42,11 @@ import project class FakeProject: """A fake for Project for basic functionality.""" - def __init__(self, worktree): + def __init__(self, worktree: str, gitdir: Optional[str] = None) -> None: self.worktree = worktree - self.gitdir = os.path.join(worktree, ".git") + self.gitdir = ( + gitdir if gitdir is not None else os.path.join(worktree, ".git") + ) self.name = "fakeproject" self.work_git = project.Project._GitGetByExec( self, bare=False, gitdir=self.gitdir @@ -4994,3 +4996,573 @@ class InitGitDirConfigTests(unittest.TestCase): self.assertTrue( os.path.exists(os.path.join(hooks_dir, "post-checkout")) ) + + +class SparseCheckoutTests(unittest.TestCase): + """Check sparse-checkout handling.""" + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_configure_sparse_checkout_with_list(self) -> None: + """Test configuring sparse-checkout with list of paths.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = FakeProject(tempdir) + + sparse_paths = ["src/backend", "src/shared"] + fakeproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + fakeproj, FakeProject + ) + ) + fakeproj._ConfigureSparseCheckout(sparse_paths) + + result = fakeproj.work_git.sparse_checkout("list") + paths = result.strip().split("\n") + self.assertIn("src/backend", paths) + self.assertIn("src/shared", paths) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_configure_sparse_checkout_delimits_paths(self) -> None: + """Paths go after `--` so a leading dash isn't read as an option.""" + proj = mock.MagicMock() + project.Project._ConfigureSparseCheckout(proj, ["src/main", "docs"]) + + self.assertEqual( + [ + mock.call("init", "--cone"), + mock.call("set", "--", "src/main", "docs"), + ], + proj.work_git.sparse_checkout.call_args_list, + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_sparse_checkout_filters_worktree(self) -> None: + """Test that sparse-checkout filters files from worktree.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + os.makedirs(os.path.join(tempdir, "docs")) + + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + with open(os.path.join(tempdir, "docs/guide.md"), "w") as f: + f.write("# Guide\n") + + fakeproj.work_git.add(".") + fakeproj.work_git.commit("-mInitial commit") + + fakeproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + fakeproj, FakeProject + ) + ) + fakeproj._ConfigureSparseCheckout(["src/main", "docs"]) + + fakeproj.work_git.read_tree("-mu", "HEAD") + + self.assertTrue( + os.path.exists(os.path.join(tempdir, "src/main/app.py")) + ) + self.assertTrue( + os.path.exists(os.path.join(tempdir, "docs/guide.md")) + ) + self.assertFalse( + os.path.exists(os.path.join(tempdir, "src/tests/test.py")) + ) + + def _setup_git_worktree_project( + self, tempdir: str, wtdir: str, sparse_paths: List[str] + ) -> FakeProject: + """Build a project checked out from a real repo via `git worktree`.""" + srcproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + with open(os.path.join(tempdir, "README"), "w") as f: + f.write("readme\n") + with open(os.path.join(tempdir, "src/version.txt"), "w") as f: + f.write("1.0\n") + + srcproj.work_git.add(".") + srcproj.work_git.commit("-mInitial commit") + revid = srcproj.work_git.rev_parse("HEAD").strip() + + # Resolve symlinks (e.g. /var -> /private/var on macOS) so the + # worktree<->gitdir relative paths git records stay valid. + worktree = os.path.join(os.path.realpath(wtdir), "proj") + wtproj = FakeProject( + worktree, + gitdir=os.path.join(os.path.realpath(tempdir), ".git"), + ) + wtproj.sparse_paths = sparse_paths + wtproj.GetRevisionId = lambda: revid + wtproj._InitMRef = lambda: None + wtproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + wtproj, FakeProject + ) + ) + wtproj._InitGitWorktree = project.Project._InitGitWorktree.__get__( + wtproj, FakeProject + ) + return wtproj + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_applies_sparse_checkout(self) -> None: + """Test that git-worktree checkouts honor sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + wtproj._InitGitWorktree() + + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/main/app.py")) + ) + self.assertFalse( + os.path.exists(os.path.join(worktree, "src/tests/test.py")) + ) + # Cone mode also keeps the files directly in the top-level + # directory and in each leading directory of a sparse-path. + self.assertTrue( + os.path.exists(os.path.join(worktree, "README")) + ) + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/version.txt")) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_on_sparse_failure(self) -> None: + """A failed sparse checkout leaves nothing for the next sync to trip + over.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # Fail after `worktree add` has registered and locked the + # worktree, which is what makes the cleanup non-trivial. + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + # assertRaisesRegex, not assertRaises: `bare_git.worktree` + # raises GitError too, so a bare check could not tell the + # original failure from one thrown by the cleanup itself. + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + # Nothing is left behind: no half-populated tree, and no + # locked registration pointing at it. + self.assertFalse(os.path.exists(worktree)) + self.assertNotIn( + worktree, wtproj.bare_git.worktree("list", "--porcelain") + ) + + # The whole point of rolling back: the next sync recovers on + # its own, without the user cleaning up by hand. + wtproj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__( + wtproj, FakeProject + ) + ) + wtproj._InitGitWorktree() + + self.assertTrue( + os.path.exists(os.path.join(worktree, "src/main/app.py")) + ) + self.assertFalse( + os.path.exists(os.path.join(worktree, "src/tests/test.py")) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_a_populated_worktree(self) -> None: + """Rollback also handles a failure after files were written.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # Fail *after* dirtying the worktree. This is the case + # `worktree remove --force --force` exists for: a plain + # remove refuses to discard a non-empty tree. + def populate_then_fail(_paths: List[str]) -> None: + with open(os.path.join(worktree, "stray.txt"), "w") as fp: + fp.write("half written\n") + raise error.GitError("sparse-checkout failed") + + wtproj._ConfigureSparseCheckout = populate_then_fail + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + self.assertFalse(os.path.exists(worktree)) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_rolls_back_on_interrupt(self) -> None: + """Ctrl-C unwinds too, not just ordinary errors.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + worktree = wtproj.worktree + + # A blob:none checkout can spend minutes fetching, so an + # interrupt here is the likeliest failure of all -- and + # KeyboardInterrupt is a BaseException, not an Exception. + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=KeyboardInterrupt() + ) + with self.assertRaises(KeyboardInterrupt): + wtproj._InitGitWorktree() + + self.assertFalse(os.path.exists(worktree)) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_git_worktree_cleanup_failure_keeps_original_error( + self, + ) -> None: + """A failing rollback warns but never hides why the sync failed.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + wtproj = self._setup_git_worktree_project( + tempdir, wtdir, ["src/main"] + ) + wtproj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + + real_worktree = wtproj.bare_git.worktree + + def failing_worktree(*args: str) -> str: + if args and args[0] == "remove": + raise error.GitError("worktree remove failed") + return real_worktree(*args) + + wtproj.bare_git.worktree = failing_worktree + + with mock.patch.object(project, "logger") as logger_mock: + # The original error survives, not the cleanup's. + with self.assertRaisesRegex( + error.GitError, "sparse-checkout failed" + ): + wtproj._InitGitWorktree() + + logger_mock.warning.assert_called_once() + + def _setup_work_tree_project( + self, tempdir: str, wtdir: str, sparse_paths: List[str] + ) -> FakeProject: + """Build a project whose .git symlinks to an existing gitdir.""" + srcproj = FakeProject(tempdir) + + os.makedirs(os.path.join(tempdir, "src/main")) + os.makedirs(os.path.join(tempdir, "src/tests")) + with open(os.path.join(tempdir, "src/main/app.py"), "w") as f: + f.write("print('Hello')\n") + with open(os.path.join(tempdir, "src/tests/test.py"), "w") as f: + f.write("def test(): pass\n") + + srcproj.work_git.add(".") + srcproj.work_git.commit("-mInitial commit") + revid = srcproj.work_git.rev_parse("HEAD").strip() + + # Resolve symlinks (e.g. /var -> /private/var on macOS) so the + # worktree<->gitdir paths git records stay valid. + worktree = os.path.join(os.path.realpath(wtdir), "proj") + os.makedirs(worktree) + gitdir = os.path.join(os.path.realpath(tempdir), ".git") + + proj = FakeProject(worktree, gitdir=gitdir) + proj.sparse_paths = sparse_paths + proj.parent = None + proj.use_git_worktrees = False + proj.manifest = mock.MagicMock() + proj.GetRevisionId = lambda: revid + proj._CopyAndLinkFiles = lambda: None + proj._createDotGit = lambda dotgit: platform_utils.symlink( + os.path.relpath(gitdir, worktree), dotgit + ) + proj._ConfigureSparseCheckout = ( + project.Project._ConfigureSparseCheckout.__get__(proj, FakeProject) + ) + proj._IsSparseCheckoutConfigured = ( + project.Project._IsSparseCheckoutConfigured.__get__( + proj, FakeProject + ) + ) + proj._InitWorkTree = project.Project._InitWorkTree.__get__( + proj, FakeProject + ) + return proj + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_work_tree_applies_sparse_checkout(self) -> None: + """Test that plain (non-worktree) checkouts honor sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + + proj._InitWorkTree() + + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/main/app.py") + ) + ) + self.assertFalse( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + + @unittest.skipUnless( + git_command.git_require((2, 25, 0)), + "git 2.25.0+ is required for sparse-checkout", + ) + def test_init_work_tree_disables_stale_sparse_checkout(self) -> None: + """Dropping sparse-path restores a full checkout on the next sync.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + proj._InitWorkTree() + self.assertFalse( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + + # The manifest drops and the user removes the + # checkout and syncs again. The gitdir -- and the sparse + # state git recorded in it -- outlives the worktree. + shutil.rmtree(proj.worktree) + os.makedirs(proj.worktree) + proj.sparse_paths = [] + self.assertTrue(proj._IsSparseCheckoutConfigured()) + + proj._InitWorkTree() + + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/tests/test.py") + ) + ) + self.assertTrue( + os.path.exists( + os.path.join(proj.worktree, "src/main/app.py") + ) + ) + + # No skipUnless: the answer must be False on old git too, via the + # version gate rather than the file check. + def test_is_sparse_checkout_configured_tracks_the_pattern_file( + self, + ) -> None: + """The probe follows the file git actually writes.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project(tempdir, wtdir, []) + self.assertFalse(proj._IsSparseCheckoutConfigured()) + + # Assert against the real path rather than just the False + # case, so the probe cannot pass by always returning False. + pattern_file = os.path.join( + proj.gitdir, "info", "sparse-checkout" + ) + os.makedirs(os.path.dirname(pattern_file), exist_ok=True) + with open(pattern_file, "w") as fp: + fp.write("/*\n") + + self.assertEqual( + git_command.git_require((2, 25, 0)), + proj._IsSparseCheckoutConfigured(), + ) + + def test_init_work_tree_removes_dotgit_on_sparse_failure(self) -> None: + """Test that a sparse-checkout failure cleans up .git.""" + with utils_for_test.TempGitTree() as tempdir: + with tempfile.TemporaryDirectory() as wtdir: + proj = self._setup_work_tree_project( + tempdir, wtdir, ["src/main"] + ) + proj._ConfigureSparseCheckout = mock.MagicMock( + side_effect=error.GitError("sparse-checkout failed") + ) + + with self.assertRaises(error.GitError): + proj._InitWorkTree() + + self.assertFalse( + os.path.lexists(os.path.join(proj.worktree, ".git")) + ) + + +class SparseCheckoutFetchFilterTests(unittest.TestCase): + """Check the partial clone filter used for sparse-checkout projects.""" + + def _get_project( + self, tempdir: str, sparse_paths: List[str] + ) -> project.Project: + # Point objdir somewhere that does not exist yet: Sync_NetworkHalf + # skips the clone bundle outright once the shared object dir is + # present, which would mask what these tests are checking. + proj = _create_mock_project( + tempdir, + revisionExpr="1234abcd", + objdir=os.path.join(tempdir, "no-such-objdir"), + ) + proj.sparse_paths = sparse_paths + # Project config falls back to the (mocked) global config, which would + # otherwise report every key, extensions.partialclone included, as set. + proj.manifest.globalConfig.GetString.return_value = None + proj._CheckForImmutableRevision = mock.MagicMock(return_value=False) + proj.bare_git.rev_parse.return_value = "5678abcd" + proj._InitGitDir = mock.MagicMock() + proj._InitRemote = mock.MagicMock() + proj._InitMRef = mock.MagicMock() + proj._ApplyCloneBundle = mock.MagicMock(return_value=False) + proj._RemoteFetch = mock.MagicMock(return_value=True) + return proj + + def _fetch_filter( + self, proj: project.Project, **kwargs: object + ) -> Optional[str]: + """Return the clone_filter Sync_NetworkHalf passes down to fetch.""" + res = proj.Sync_NetworkHalf(clone_bundle=False, **kwargs) + self.assertTrue(res.success) + proj._RemoteFetch.assert_called_once() + return proj._RemoteFetch.call_args.kwargs["clone_filter"] + + def test_no_sparse_paths_keeps_no_filter(self) -> None: + """A project without sparse-path is not filtered.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, []) + self.assertIsNone(self._fetch_filter(proj)) + + def test_sparse_paths_default_to_blob_none(self) -> None: + """sparse-path implies a blob:none partial clone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertEqual("blob:none", self._fetch_filter(proj)) + + def test_explicit_clone_filter_wins(self) -> None: + """An explicit filter is not replaced by the default.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertEqual( + "blob:limit=1m", + self._fetch_filter(proj, clone_filter="blob:limit=1m"), + ) + + def test_partial_clone_exclude_wins(self) -> None: + """An excluded project is fetched in full despite sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertIsNone( + self._fetch_filter(proj, partial_clone_exclude={proj.name}) + ) + + def test_existing_full_checkout_keeps_no_filter(self) -> None: + """An existing full checkout is not converted to a partial clone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + os.makedirs(proj.objdir) + self.assertIsNone(self._fetch_filter(proj, is_new=True)) + + def test_existing_partial_clone_keeps_blob_none(self) -> None: + """An existing partial clone keeps the blob:none default.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + os.makedirs(proj.objdir) + proj.config.SetString("extensions.partialclone", "origin") + self.assertEqual("blob:none", self._fetch_filter(proj, is_new=True)) + + def test_mirror_keeps_no_filter(self) -> None: + """Mirrors are never given the implied filter.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + proj.manifest.IsMirror = True + self.assertIsNone(self._fetch_filter(proj)) + + def _bundle_applied(self, proj: project.Project, **kwargs: object) -> bool: + """Return whether Sync_NetworkHalf reached for the clone bundle.""" + res = proj.Sync_NetworkHalf(clone_bundle=True, **kwargs) + self.assertTrue(res.success) + return proj._ApplyCloneBundle.called + + def test_no_sparse_paths_keeps_clone_bundle(self) -> None: + """A project without sparse-path still uses the clone bundle.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, []) + self.assertTrue(self._bundle_applied(proj)) + + def test_sparse_paths_skip_clone_bundle(self) -> None: + """The bundle is unfiltered, so the blob:none default drops it.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertFalse(self._bundle_applied(proj)) + + def test_explicit_clone_filter_keeps_clone_bundle(self) -> None: + """Only the implied filter drops the bundle, not an explicit one.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertTrue( + self._bundle_applied(proj, clone_filter="blob:limit=1m") + ) + + def test_partial_clone_exclude_restores_clone_bundle(self) -> None: + """An excluded project gets its bundle back despite sparse-path.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir, ["src/main"]) + self.assertTrue( + self._bundle_applied(proj, partial_clone_exclude={proj.name}) + )