diff --git a/docs/fetch-cmd.md b/docs/fetch-cmd.md index 32fa8d156..b125ed344 100644 --- a/docs/fetch-cmd.md +++ b/docs/fetch-cmd.md @@ -6,6 +6,9 @@ executed during `repo sync` to fetch objects, instead of using standard filesystems or lazy checkouts where fetching metadata and downloading file contents should be decoupled. +The checkout half of a sync has a counterpart, `repo.reprojectcmd`; see +`docs/reproject-cmd.md`. + ## Configuration To use this feature, set the following in `.repo/manifests.git/config`: diff --git a/docs/reproject-cmd.md b/docs/reproject-cmd.md new file mode 100644 index 000000000..76e1de822 --- /dev/null +++ b/docs/reproject-cmd.md @@ -0,0 +1,126 @@ +# Reproject Command Contract + +The `repo.reprojectcmd` configuration names a command that `repo sync` runs +instead of Git to move a project's index and worktree to the tree of the +target commit. It is the checkout-side counterpart of `repo.fetchcmd` (see +`docs/fetch-cmd.md`): together they let an external tool take over both the +network fetch and the materialization of a project. This is useful on +virtualized filesystems that address content by hash, where a tree can be +materialized far faster than `git checkout` can write every file. + +The command only materializes the tree. `repo` then makes the ref write that +Git would have made, using `git update-ref`. + +## Configuration + +To use this feature, set the following in `.repo/manifests.git/config`: +```ini +[repo] + reprojectcmd = "your custom command here" + uselocalgitdirs = true +``` +Setting `repo.reprojectcmd` **requires** `repo.uselocalgitdirs` to be set to +`true`. + +For reference, this command does with Git what `repo` would otherwise do +itself: +```ini +[repo] + reprojectcmd = "git -C $REPO_PATH read-tree -m -u $REPO_TREV" + uselocalgitdirs = true +``` +The one-tree merge applies the change to the target, keeps local changes to +every other path, and refuses to overwrite a modified or untracked file, so it +enforces the preconditions below by itself. It also works for a project that +has nothing checked out yet. + +## Environment Variables + +The command is executed in a subshell, from the root of the client, populated +with standard project-context environment variables. For details on standard +variables (such as `REPO_PROJECT`, `REPO_PATH`, `REPO_REMOTE`, etc.), see the +Environment section in `repo help forall` or `subcmds/forall.py`. + +The variables the command typically needs are: + +* `REPO_PATH`: The project path relative to the root of the client. +* `REPO_TREV`: The target revision resolved to a full commit hash. Match this + commit's tree. + +There is no force mode: a project that would need one never reaches the +command (see the preconditions below). + +## When the command runs + +`repo sync` already classifies each project and picks a Git operation. The +command replaces the three that are a materialization of a target tree: + +1. The checkout that detaches HEAD at the target. This is the common case: a + project on a detached HEAD, a project on a branch that does not track + upstream, and `repo sync -d`. +2. The fast-forward of the checked out branch to the target. +3. The hard reset of the checked out branch to the target, when the commits + it carried were dropped upstream. + +After the command exits 0, `repo` writes the ref itself: it detaches `HEAD` at +`REPO_TREV`, or moves the checked out branch to `REPO_TREV`. + +The command is **not** run: + +* When `HEAD` already names `REPO_TREV`. +* At the fast-forward step when `HEAD` is ahead of `REPO_TREV`, where Git's + merge would be a no-op. +* For a rebase. A branch carrying local commits has them replayed onto the + target by `git rebase`, which is not a materialization of a target tree. +* For `MetaProject`s (i.e. the internal `repo` repository itself at + `.repo/repo` and the `manifests` repository at `.repo/manifests`). + +## Contract + +### Preconditions + +Before invoking the command, `repo` ensures that: + +* The index has no staged changes (the index matches `HEAD`, or is empty on an unborn `HEAD`). +* No rebase, cherry-pick, merge, or revert is in progress. + +Detecting collisions with untracked files or unstaged working-tree modifications is the responsibility of the reproject command itself (e.g. via `git read-tree -m -u $REPO_TREV` or a custom virtual filesystem checkout tool). If local changes collide with the target tree, the command must abort with a non-zero exit code. Local modifications and untracked files outside the diff between `HEAD` and `REPO_TREV` must be preserved. + +### Postconditions on exit 0 + +After the command exits with status 0, `repo` expects the following +postconditions to be met: + +1. `git diff-index --quiet --cached REPO_TREV^{tree}` exits 0 (the index + matches the target tree). +2. `HEAD` still names what it did before the command, and its resolved commit + object ID has not changed. + +### Invariants + +The command may modify the worktree and the index, and may write project-local +Git config. The command must: + +* Apply the change from `HEAD`'s tree to `REPO_TREV`'s tree and leave every + other path alone. Local modifications and untracked files outside that + change must survive: the command applies a diff, it does not reset the + tree. +* Not write any ref, including `HEAD` and `ORIG_HEAD`. `repo` owns every ref + write. +* Not create or replace `.git/`, and not touch anything under `.repo/`. +* Not require the Git remote, to preserve `repo sync --local-only`. +* Be idempotent. Running it twice on the same target is a no-op. + +### Failure + +* A non-zero exit status, a failed precondition or a failed postcondition + fails that project's sync, and the command's or Git's output is surfaced + to the user. +* Other projects continue, and `repo sync` exits non-zero. + +## Limitations + +Nested projects are out of scope: a project whose path lies inside another +project's path, a `` nested in another `` in the manifest, +and a submodule discovered with `sync-s` or `--recurse-submodules`. `repo sync` +fails if the manifest has one while `repo.reprojectcmd` is set. diff --git a/project.py b/project.py index a084d8db8..783aead06 100644 --- a/project.py +++ b/project.py @@ -110,6 +110,14 @@ RETRY_JITTER_PERCENT = 0.1 _ALTERNATES = os.environ.get("REPO_USE_ALTERNATES") == "1" +def _FirstLines(lines: List[str], limit: int = 10) -> str: + """Join |lines|, eliding all but the first |limit| of them.""" + shown = list(lines[:limit]) + if len(lines) > limit: + shown.append(f"... and {len(lines) - limit} more") + return "\n".join(shown) + + def _lwrite(path, content): lock = "%s.lock" % path @@ -811,6 +819,17 @@ class Project: """Returns True if a cherry-pick is in progress.""" return os.path.exists(self.work_git.GetDotgitPath("CHERRY_PICK_HEAD")) + def _OperationInProgress(self) -> Optional[str]: + """Return the name of the Git operation in progress, if any.""" + if self.IsRebaseInProgress(): + return "rebase" + if self.IsCherryPickInProgress(): + return "cherry-pick" + for state, name in (("MERGE_HEAD", "merge"), ("REVERT_HEAD", "revert")): + if os.path.exists(self.work_git.GetDotgitPath(state)): + return name + return None + def _AbortRebase(self): """Abort ongoing rebase, cherry-pick or patch apply (am). @@ -1829,6 +1848,18 @@ class Project: self.revisionId = revisionId + @property + def UseReprojectCmd(self) -> bool: + """Whether repo.reprojectcmd materializes this project's tree. + + MetaProjects (repo itself and the manifests) always use Git. See + docs/reproject-cmd.md. + """ + if isinstance(self, MetaProject): + return False + mp = self.manifest.manifestProject + return bool(mp.use_local_gitdirs and mp.reproject_cmd) + def Sync_LocalHalf( self, syncbuf, @@ -1856,6 +1887,29 @@ class Project: ) return + if not isinstance(self, MetaProject): + mp = self.manifest.manifestProject + if mp.reproject_cmd and not mp.use_local_gitdirs: + fail( + LocalSyncFail( + "repo.reprojectcmd requires repo.uselocalgitdirs to be " + "enabled", + project=self.name, + ) + ) + return + + reproject = self.UseReprojectCmd + if reproject and self.parent: + fail( + LocalSyncFail( + "repo.reprojectcmd does not support nested projects or " + "submodules", + project=self.name, + ) + ) + return + self._InitWorkTree(force_sync=force_sync, submodules=submodules) # TODO(https://git-scm.com/docs/git-worktree#_bugs): Re-evaluate if # submodules can be init when using worktrees once its support is @@ -1886,8 +1940,30 @@ class Project: ) return + head = self._GetHead() + if head and head.startswith(R_HEADS): + branch = head[len(R_HEADS) :] + try: + head = all_refs[head] + except KeyError: + head = None + else: + branch = None + + def _checkout() -> None: + """Detach HEAD at revid, like `git checkout `.""" + if reproject: + self._ReprojectCheckout(revid, head, verbose=verbose) + else: + self._Checkout(revid, force_checkout=force_checkout, quiet=True) + def _doff(): - self._FastForward(revid) + if reproject: + self._ReprojectBranch( + revid, head, f"merge {revid}: Fast-forward", verbose=verbose + ) + else: + self._FastForward(revid) self._CopyAndLinkFiles() def _dorebase(): @@ -1913,16 +1989,6 @@ class Project: if p.Wait() != 0: logger.warning("warn: %s: stateless gc failed", self.name) - head = self._GetHead() - if head and head.startswith(R_HEADS): - branch = head[len(R_HEADS) :] - try: - head = all_refs[head] - except KeyError: - head = None - else: - branch = None - if branch is None or syncbuf.detach_head: # Currently on a detached HEAD. The user is assumed to # not have any local modifications worth worrying about. @@ -1951,10 +2017,10 @@ class Project: syncbuf.info(self, "discarding %d commits", len(lost)) try: - self._Checkout(revid, force_checkout=force_checkout, quiet=True) + _checkout() if submodules: self._SyncSubmodules(quiet=True) - except GitError as e: + except (GitError, LocalSyncFail) as e: fail(e) return self._CopyAndLinkFiles() @@ -1978,10 +2044,10 @@ class Project: self, "leaving %s; does not track upstream", branch.name ) try: - self._Checkout(revid, force_checkout=force_checkout, quiet=True) + _checkout() if submodules: self._SyncSubmodules(quiet=True) - except GitError as e: + except (GitError, LocalSyncFail) as e: fail(e) return self._CopyAndLinkFiles() @@ -2022,7 +2088,12 @@ class Project: ) ) return - syncbuf.later1(self, _doff, not verbose) + if reproject: + # HEAD is ahead of revid, so there is no tree to + # materialize: Git's fast-forward would be a no-op. + self._CopyAndLinkFiles() + else: + syncbuf.later1(self, _doff, not verbose) return elif pub == head: # All published commits are merged, and thus we are a @@ -2048,7 +2119,9 @@ class Project: self._CopyAndLinkFiles() return - if self.IsDirty(consider_untracked=False): + if (not reproject or (cnt_mine > 0 and self.rebase)) and self.IsDirty( + consider_untracked=False + ): fail(_DirtyError(project=self.name)) return @@ -2094,11 +2167,19 @@ class Project: syncbuf.later2(self, _docopyandlink, not verbose) elif local_changes: try: - self._ResetHard(revid) + if reproject: + self._ReprojectBranch( + revid, + head, + f"reset: moving to {revid}", + verbose=verbose, + ) + else: + self._ResetHard(revid) if submodules: self._SyncSubmodules(quiet=True) self._CopyAndLinkFiles() - except GitError as e: + except (GitError, LocalSyncFail) as e: fail(e) return else: @@ -3696,6 +3777,184 @@ class Project: if GitCommand(self, cmd).Wait() != 0: raise GitError(f"{self.name} merge {head} ", project=self.name) + def _ReprojectCheckout( + self, revid: str, head: Optional[str], verbose: bool = False + ) -> None: + """Detach HEAD at |revid| with repo.reprojectcmd. + + This stands in for `git checkout `. |head| is the commit HEAD + names now, or None. The command is not run when that is already + |revid|, since there is then nothing to materialize. + """ + old = self._GetHead() + if old and old.startswith(R_HEADS): + old = old[len(R_HEADS) :] + if head != revid: + self._Reproject(revid, verbose=verbose) + self.work_git.DetachHead( + revid, message=f"checkout: moving from {old or revid} to {revid}" + ) + + def _ReprojectBranch( + self, + revid: str, + head: Optional[str], + message: str, + verbose: bool = False, + ) -> None: + """Move the checked out branch to |revid| with repo.reprojectcmd. + + This stands in for a fast-forward merge or a hard reset. |head| is the + commit the branch is at; the ref write fails if it moved meanwhile. + """ + self._Reproject(revid, verbose=verbose) + self.work_git.UpdateRef(HEAD, revid, old=head, message=message) + + def _Reproject(self, revid: str, verbose: bool = False) -> None: + """Make the index and worktree match |revid| with repo.reprojectcmd. + + The command stands in for the tree materialization of a checkout, a + fast-forward or a hard reset. It must leave every ref alone; the + caller writes the ref Git would have written. + + For the contract the command has to honor, see docs/reproject-cmd.md. + + Raises: + LocalSyncFail: An operation is in progress, the index has staged + changes, the command failed, or it left the project in a state + that breaks the contract. + """ + in_progress = self._OperationInProgress() + if in_progress: + raise LocalSyncFail( + f"{in_progress} in progress; reprojectcmd cannot run", + project=self.name, + ) + + try: + head_tree = self.work_git.rev_parse( + "-q", "--verify", "HEAD^{tree}", log_as_error=False + ) + except GitError: + head_tree = None + + if head_tree: + p = GitCommand( + self, + ["diff-index", "-z", "--cached", "--name-only", head_tree], + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + raise LocalSyncFail( + f"cannot check the index for staged changes: " + f"{p.stderr.strip()}", + project=self.name, + ) + staged = p.stdout.split("\0")[:-1] + else: + p = GitCommand( + self, + ["ls-files", "-z", "--cached"], + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + raise LocalSyncFail( + f"cannot check the index for staged changes: " + f"{p.stderr.strip()}", + project=self.name, + ) + staged = p.stdout.split("\0")[:-1] + + if staged: + raise LocalSyncFail( + "reprojectcmd cannot run with staged changes:\n" + + _FirstLines(staged), + project=self.name, + ) + + head = self.work_git.GetHead() + try: + head_oid = self.work_git.rev_parse( + "-q", "--verify", "HEAD", log_as_error=False + ) + except GitError: + head_oid = None + + env = os.environ.copy() + env.update(self.GetEnvVars()) + env["REPO_TREV"] = revid + cmd_str = self.manifest.manifestProject.reproject_cmd + if verbose: + print(f"Running reprojectcmd: {cmd_str} for {self.name}") + + output = None if verbose else subprocess.PIPE + try: + p = subprocess.run( + cmd_str, + shell=True, + cwd=self.manifest.topdir, + env=env, + stdout=output, + stderr=None if verbose else subprocess.STDOUT, + text=True, + ) + except OSError as e: + raise LocalSyncFail( + f"failed to run reprojectcmd: {e}", project=self.name + ) + if p.returncode != 0: + msg = f"reprojectcmd exited with {p.returncode}" + if p.stdout: + msg += ":\n" + p.stdout.rstrip() + raise LocalSyncFail(msg, project=self.name) + + new_head = self.work_git.GetHead() + try: + new_head_oid = self.work_git.rev_parse( + "-q", "--verify", "HEAD", log_as_error=False + ) + except GitError: + new_head_oid = None + + if new_head != head or new_head_oid != head_oid: + from_desc = ( + f"{head} ({head_oid})" + if head_oid and head != head_oid + else f"{head}" + ) + to_desc = ( + f"{new_head} ({new_head_oid})" + if new_head_oid and new_head != new_head_oid + else f"{new_head}" + ) + raise LocalSyncFail( + f"reprojectcmd moved HEAD from {from_desc} to {to_desc}; repo " + "owns every ref write", + project=self.name, + ) + + p = GitCommand( + self, + ["diff-index", "--cached", "--name-only", f"{revid}^{{tree}}"], + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + raise LocalSyncFail( + f"cannot compare the index against {revid}: " + f"{p.stderr.strip()}", + project=self.name, + ) + mismatched = p.stdout.splitlines() + if mismatched: + raise LocalSyncFail( + f"reprojectcmd left the index different from {revid}:\n" + + _FirstLines(mismatched), + project=self.name, + ) + def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False): # Prefix for temporary directories created during gitdir initialization. TMP_GITDIR_PREFIX = ".tmp-project-initgitdir-" @@ -4806,7 +5065,7 @@ class _Later: if not self.quiet: out.nl() return True - except GitError as e: + except (GitError, LocalSyncFail) as e: syncbuf.fail(self.project, e) out.nl() return False @@ -5085,6 +5344,11 @@ class ManifestProject(MetaProject): """The fetch command to use.""" return self.config.GetString("repo.fetchcmd") + @property + def reproject_cmd(self) -> Optional[str]: + """The command that materializes a project's tree instead of Git.""" + return self.config.GetString("repo.reprojectcmd") + @property def clone_bundle(self): """Whether we use clone_bundle.""" @@ -5505,6 +5769,15 @@ class ManifestProject(MetaProject): ) return False + if self.reproject_cmd and not ( + use_local_gitdirs or self.use_local_gitdirs + ): + logger.error( + "fatal: repo.reprojectcmd is set but repo.uselocalgitdirs is " + "not enabled" + ) + return False + if archive: if is_new: self.config.SetBoolean("repo.archive", archive) diff --git a/subcmds/sync.py b/subcmds/sync.py index d88ec883e..c51ecf385 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -72,6 +72,7 @@ from git_refs import HEAD from git_refs import R_HEADS import git_superproject from hooks import RepoHook +from manifest_xml import XmlManifest import platform_utils from progress import elapsed_str from progress import jobs_str @@ -156,6 +157,11 @@ def _SafeCheckoutOrder(checkouts: List[Project]) -> List[List[Project]]: return res +def _NestedProjects(projects: List[Project]) -> List[Project]: + """Return the projects in |projects| living inside another one's path.""" + return [p for level in _SafeCheckoutOrder(projects)[1:] for p in level] + + def _ParentFirstBatches(projects: List[Project]) -> List[List[Project]]: """Group |projects| so that a parent is fetched before its submodules. @@ -2478,6 +2484,7 @@ later is required to fix a server side protocol bug. manifest=manifest, all_manifests=not opt.this_manifest_only, ) + self._CheckReprojectCmdNesting(opt, args, manifest, all_projects) # Log the repo projects by existing and new. existing = [x for x in all_projects if x.Exists] @@ -2541,6 +2548,46 @@ later is required to fix a server side protocol bug. if not opt.quiet: print("repo sync has finished successfully.") + def _CheckReprojectCmdNesting( + self, + opt: optparse.Values, + args: List[str], + manifest: XmlManifest, + all_projects: List[Project], + ) -> None: + """Fail when repo.reprojectcmd is used on a manifest nesting projects. + + The command materializes a project's tree without Git, so nothing + keeps it from clobbering a project or submodule checked out inside + that tree. See docs/reproject-cmd.md. + """ + if not any(p.UseReprojectCmd for p in all_projects): + return + projects = all_projects + if args: + # Nesting is a property of the manifest, not of the projects + # picked on the command line. + projects = self.GetProjects( + [], + groups=opt.groups, + missing_ok=True, + submodules_ok=opt.recurse_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only, + ) + nested = _NestedProjects(projects) + if not nested: + return + e = SyncError( + "error: repo.reprojectcmd does not support nested projects or " + "submodules; found:\n" + + "\n".join( + f" - {p.RelPath(local=opt.this_manifest_only)}" for p in nested + ) + ) + logger.error(e) + raise e + def _CreateSyncProgressThread( self, pm: Progress, stop_event: _threading.Event ) -> _threading.Thread: diff --git a/tests/test_project.py b/tests/test_project.py index 911bf7285..764fbc469 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -20,7 +20,7 @@ from pathlib import Path import shutil import subprocess import tempfile -from typing import Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import unittest from unittest import mock @@ -1866,6 +1866,7 @@ def _create_mock_project( tempdir, use_local_gitdirs=False, fetch_cmd=None, + reproject_cmd: Optional[str] = None, depth=None, gitdir=None, objdir=None, @@ -1875,6 +1876,7 @@ def _create_mock_project( manifest = mock.MagicMock() manifest.manifestProject.use_local_gitdirs = use_local_gitdirs manifest.manifestProject.fetch_cmd = fetch_cmd + manifest.manifestProject.reproject_cmd = reproject_cmd manifest.manifestProject.depth = depth manifest.manifestProject.dissociate = False manifest.manifestProject.clone_filter = None @@ -2839,20 +2841,25 @@ class GetEnvVarsTests(unittest.TestCase): self.assertEqual(env["REPO_LREV"], "") +def _create_manifest_project(tempdir: str) -> project.ManifestProject: + """Return a ManifestProject for a new .repo/ under |tempdir|.""" + repodir = os.path.join(tempdir, ".repo") + manifest_dir = os.path.join(repodir, "manifests") + manifest_file = os.path.join(repodir, manifest_xml.MANIFEST_FILE_NAME) + os.mkdir(repodir) + os.mkdir(manifest_dir) + manifest = manifest_xml.XmlManifest(repodir, manifest_file) + + return project.ManifestProject( + manifest, "test/manifest", os.path.join(tempdir, ".git"), tempdir + ) + + class FetchCmdTests(unittest.TestCase): """Tests for fetch_cmd feature.""" def setUpManifest(self, tempdir): - repodir = os.path.join(tempdir, ".repo") - manifest_dir = os.path.join(repodir, "manifests") - manifest_file = os.path.join(repodir, manifest_xml.MANIFEST_FILE_NAME) - os.mkdir(repodir) - os.mkdir(manifest_dir) - manifest = manifest_xml.XmlManifest(repodir, manifest_file) - - return project.ManifestProject( - manifest, "test/manifest", os.path.join(tempdir, ".git"), tempdir - ) + return _create_manifest_project(tempdir) def _get_project(self, tempdir): proj = _create_mock_project( @@ -2905,3 +2912,752 @@ class FetchCmdTests(unittest.TestCase): result = fakeproj.Sync(use_local_gitdirs=False) self.assertFalse(result) + + +class ReprojectCmdTests(unittest.TestCase): + """Tests for the repo.reprojectcmd feature.""" + + REVID = "1234abcd" * 5 + HEAD_ID = "5678abcd" * 5 + HEAD_TREE = "cafe0000" * 5 + OTHER_ID = "9abcdef0" * 5 + PUB_ID = "0fedcba9" * 5 + + def _get_project(self, tempdir: str) -> project.Project: + proj = _create_mock_project( + tempdir, use_local_gitdirs=True, reproject_cmd="echo reproject" + ) + proj.manifest.path_prefix = "" + proj.GetRevisionId = mock.MagicMock(return_value=self.REVID) + proj.IsRebaseInProgress = mock.MagicMock(return_value=False) + proj.IsCherryPickInProgress = mock.MagicMock(return_value=False) + proj.work_git = mock.MagicMock() + proj.work_git.GetHead.return_value = self.HEAD_ID + proj.work_git.rev_parse.return_value = self.HEAD_TREE + proj.work_git.GetDotgitPath.side_effect = lambda subpath: os.path.join( + tempdir, ".git", subpath + ) + return proj + + @staticmethod + def _z(*items: str) -> str: + """Return |items| as NUL-delimited Git output.""" + return "".join(item + "\0" for item in items) + + @staticmethod + def _git_command( + staged: str = "", diff: str = "", returncode: int = 0 + ) -> Callable[..., mock.MagicMock]: + """Return a GitCommand stand-in answering the reproject queries. + + Args: + staged: `git diff-index -z --cached` or `git ls-files -z` output. + diff: `git diff-index --cached` postcondition output. + returncode: exit code of GitCommand. + """ + + def make( + project: project.Project, cmdv: List[str], **kwargs: Any + ) -> mock.MagicMock: + cmd = mock.MagicMock() + cmd.stderr = "" + if cmdv[0] == "diff-index": + if any("^{tree}" in arg for arg in cmdv): + cmd.stdout = diff + else: + cmd.stdout = staged + elif cmdv[0] == "ls-files": + cmd.stdout = staged + else: + cmd.stdout = "" + cmd.Wait.return_value = returncode + return cmd + + return make + + def _reproject( + self, proj: project.Project, **outputs: Any + ) -> mock.MagicMock: + """Run _Reproject against mocked Git; return the subprocess mock.""" + with mock.patch( + "project.GitCommand", side_effect=self._git_command(**outputs) + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="") + proj._Reproject(self.REVID) + return mock_run + + def test_reproject_runs_the_command_with_project_env(self) -> None: + """Test the command runs from the client root with REPO_TREV set.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + with mock.patch( + "project.GitCommand", side_effect=self._git_command() + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="") + proj._Reproject(self.REVID) + + mock_run.assert_called_once() + args, kwargs = mock_run.call_args + self.assertEqual(args[0], "echo reproject") + self.assertTrue(kwargs["shell"]) + self.assertEqual(kwargs["cwd"], tempdir) + self.assertEqual(kwargs["env"]["REPO_TREV"], self.REVID) + self.assertEqual(kwargs["env"]["REPO_PATH"], "test-project") + + def test_reproject_rejects_a_staged_change(self) -> None: + """Test a staged change fails before the command runs.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + with self.assertRaises(project.LocalSyncFail) as e: + self._reproject(proj, staged=self._z("lib/a.c")) + self.assertIn( + "reprojectcmd cannot run with staged changes", str(e.exception) + ) + self.assertIn("lib/a.c", str(e.exception)) + + def test_reproject_unborn_head_rejects_a_staged_change(self) -> None: + """Test a staged change on unborn HEAD fails before command runs.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + proj.work_git.rev_parse.side_effect = error.GitError("unborn") + with self.assertRaises(project.LocalSyncFail) as e: + self._reproject(proj, staged=self._z("lib/a.c")) + self.assertIn( + "reprojectcmd cannot run with staged changes", str(e.exception) + ) + self.assertIn("lib/a.c", str(e.exception)) + + def test_reproject_unborn_head_runs_clean(self) -> None: + """Test a clean unborn HEAD allows the command to run.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + proj.work_git.rev_parse.side_effect = error.GitError("unborn") + mock_run = self._reproject(proj) + mock_run.assert_called_once() + + def test_reproject_rejects_an_operation_in_progress(self) -> None: + """Test an unfinished rebase fails the sync before the command runs.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + proj.IsRebaseInProgress.return_value = True + with mock.patch("subprocess.run") as mock_run: + with self.assertRaises(project.LocalSyncFail) as e: + proj._Reproject(self.REVID) + self.assertIn("rebase in progress", str(e.exception)) + mock_run.assert_not_called() + + def test_reproject_surfaces_a_failed_command(self) -> None: + """Test a non-zero exit fails the sync with the command's output.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + with mock.patch( + "project.GitCommand", side_effect=self._git_command() + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock( + returncode=3, stdout="disk on fire\n" + ) + with self.assertRaises(project.LocalSyncFail) as e: + proj._Reproject(self.REVID) + self.assertIn("exited with 3", str(e.exception)) + self.assertIn("disk on fire", str(e.exception)) + + def test_reproject_rejects_a_moved_head_ref(self) -> None: + """Test a command that wrote HEAD fails the sync.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + proj.work_git.GetHead.side_effect = [self.HEAD_ID, self.REVID] + with mock.patch( + "project.GitCommand", side_effect=self._git_command() + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="") + with self.assertRaises(project.LocalSyncFail) as e: + proj._Reproject(self.REVID) + self.assertIn("moved HEAD", str(e.exception)) + + def test_reproject_rejects_a_moved_head_oid_on_branch(self) -> None: + """Test a command that moved a branch pointer fails the sync.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + proj.work_git.GetHead.return_value = "refs/heads/main" + proj.work_git.rev_parse.side_effect = [ + "tree123", # HEAD^{tree} + self.HEAD_ID, # HEAD before command + self.REVID, # HEAD after command + ] + with mock.patch( + "project.GitCommand", side_effect=self._git_command() + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="") + with self.assertRaises(project.LocalSyncFail) as e: + proj._Reproject(self.REVID) + self.assertIn("moved HEAD", str(e.exception)) + + def test_reproject_rejects_an_index_mismatch(self) -> None: + """Test a command that left the index off the target fails the sync.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + with mock.patch( + "project.GitCommand", + side_effect=self._git_command(diff="lib/a.c\n"), + ), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="") + with self.assertRaises(project.LocalSyncFail) as e: + proj._Reproject(self.REVID) + self.assertIn(self.REVID, str(e.exception)) + self.assertIn("lib/a.c", str(e.exception)) + + def _get_synced_project( + self, + tempdir: str, + head: Optional[str], + branch: Optional[str] = None, + upstream_gain: Sequence[str] = (), + local_changes: Sequence[str] = (), + ) -> project.Project: + """Return a project ready for Sync_LocalHalf with Git mocked out. + + Args: + head: The commit HEAD is at, or None for an unborn branch. + branch: The name of the checked out branch, or None if detached. + upstream_gain: The commits the target has that HEAD lacks. + local_changes: The " " lines HEAD has that the target + lacks. + """ + proj = self._get_project(tempdir) + for name in ( + "_InitWorkTree", + "CleanPublishedCache", + "_CopyAndLinkFiles", + "_Reproject", + "_Checkout", + "_FastForward", + "_ResetHard", + "_Rebase", + ): + setattr(proj, name, mock.MagicMock()) + proj.IsDirty = mock.MagicMock(return_value=False) + proj._userident_name = "Me" + proj._userident_email = "me@example.com" + + proj.bare_ref = mock.MagicMock() + if branch: + proj.bare_ref.head = project.R_HEADS + branch + proj.bare_ref.all = {project.R_HEADS + branch: head} if head else {} + # A branch that does not track upstream; tests that need one + # tracking upstream replace this with _tracking_branch(). + proj.GetBranch = mock.MagicMock( + return_value=self._tracking_branch(branch, merge=None) + ) + else: + proj.bare_ref.head = head + proj.bare_ref.all = {} + + def _revlist(*args: Any, **kwargs: Any) -> List[str]: + if kwargs.get("format"): + return list(local_changes) + if args[0] == project.not_rev(project.HEAD): + return list(upstream_gain) + if args[1] == self.PUB_ID: + return [self.PUB_ID] + return [] + + proj._revlist = mock.MagicMock(side_effect=_revlist) + return proj + + @staticmethod + def _tracking_branch( + name: str = "topic", merge: Optional[str] = "main" + ) -> mock.MagicMock: + """Return a branch tracking |merge| upstream, or nothing if None.""" + branch = mock.MagicMock() + branch.name = name + branch.merge = merge + branch.LocalMerge = f"refs/remotes/origin/{merge}" if merge else None + return branch + + def test_sync_local_half_materializes_a_fresh_project(self) -> None: + """Test a project with an unborn HEAD is checked out by the command.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project(tempdir, head=None, branch="main") + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_called_once_with(self.REVID, verbose=False) + proj.work_git.DetachHead.assert_called_once_with( + self.REVID, + message=f"checkout: moving from main to {self.REVID}", + ) + proj._Checkout.assert_not_called() + proj._CopyAndLinkFiles.assert_called_once_with() + + def test_sync_local_half_detached_head_uses_the_command(self) -> None: + """Test a detached HEAD is moved to the target by the command.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project(tempdir, head=self.HEAD_ID) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_called_once_with(self.REVID, verbose=False) + proj.work_git.DetachHead.assert_called_once_with( + self.REVID, + message=f"checkout: moving from {self.HEAD_ID} to {self.REVID}", + ) + proj._Checkout.assert_not_called() + + def test_sync_local_half_head_at_target_skips_the_command(self) -> None: + """Test `repo sync -d` at the target only detaches HEAD.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project( + tempdir, head=self.REVID, branch="topic" + ) + syncbuf = project.SyncBuffer(proj.config, detach_head=True) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_not_called() + proj.work_git.DetachHead.assert_called_once() + self.assertEqual( + proj.work_git.DetachHead.call_args[0][0], self.REVID + ) + + def test_sync_local_half_command_failure_fails_the_project(self) -> None: + """Test a failed command is reported and leaves the ref alone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project(tempdir, head=self.HEAD_ID) + proj._Reproject.side_effect = project.LocalSyncFail( + "reprojectcmd exited with 1", project=proj.name + ) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertFalse(syncbuf.Finish()) + self.assertEqual(len(syncbuf.errors), 1) + self.assertIn("exited with 1", str(syncbuf.errors[0])) + proj.work_git.DetachHead.assert_not_called() + proj._CopyAndLinkFiles.assert_not_called() + + def test_sync_local_half_fast_forward_uses_the_command(self) -> None: + """Test a branch behind the target is fast-forwarded by the command.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project( + tempdir, + head=self.HEAD_ID, + branch="topic", + upstream_gain=[self.REVID], + ) + proj.GetBranch = mock.MagicMock( + return_value=self._tracking_branch() + ) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_called_once_with(self.REVID, verbose=False) + proj.work_git.UpdateRef.assert_called_once_with( + project.HEAD, + self.REVID, + old=self.HEAD_ID, + message=f"merge {self.REVID}: Fast-forward", + ) + proj._FastForward.assert_not_called() + proj._CopyAndLinkFiles.assert_called_once_with() + + def test_sync_local_half_head_ahead_skips_the_command(self) -> None: + """Test a published branch ahead of the target is left alone.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project( + tempdir, head=self.HEAD_ID, branch="topic" + ) + proj.GetBranch = mock.MagicMock( + return_value=self._tracking_branch() + ) + proj.work_git.merge_base.side_effect = error.GitError("no") + proj.WasPublished = mock.MagicMock(return_value=self.PUB_ID) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_not_called() + proj.work_git.UpdateRef.assert_not_called() + proj._FastForward.assert_not_called() + proj._CopyAndLinkFiles.assert_called_once_with() + + def test_sync_local_half_hard_reset_uses_the_command(self) -> None: + """Test a branch whose commits upstream dropped is reset by it.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project( + tempdir, + head=self.HEAD_ID, + branch="topic", + upstream_gain=[self.REVID], + local_changes=[f"{self.OTHER_ID} other@example.com"], + ) + proj.GetBranch = mock.MagicMock( + return_value=self._tracking_branch() + ) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Reproject.assert_called_once_with(self.REVID, verbose=False) + proj.work_git.UpdateRef.assert_called_once_with( + project.HEAD, + self.REVID, + old=self.HEAD_ID, + message=f"reset: moving to {self.REVID}", + ) + proj._ResetHard.assert_not_called() + + def test_sync_local_half_rebase_is_not_delegated(self) -> None: + """Test a branch carrying the user's commits is rebased by Git.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project( + tempdir, + head=self.HEAD_ID, + branch="topic", + upstream_gain=[self.REVID], + local_changes=[f"{self.OTHER_ID} me@example.com"], + ) + proj.GetBranch = mock.MagicMock( + return_value=self._tracking_branch() + ) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Rebase.assert_called_once_with( + upstream=f"{self.OTHER_ID}^1", onto=self.REVID + ) + proj._Reproject.assert_not_called() + proj.work_git.UpdateRef.assert_not_called() + + def test_sync_local_half_rejects_a_nested_project(self) -> None: + """Test a submodule or nested project fails before any checkout.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project(tempdir, head=self.HEAD_ID) + proj.parent = mock.MagicMock() + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertFalse(syncbuf.Finish()) + self.assertIn("nested", str(syncbuf.errors[0])) + proj._InitWorkTree.assert_not_called() + proj._Reproject.assert_not_called() + + def test_sync_local_half_without_the_command_uses_git(self) -> None: + """Test Git keeps doing the checkout when the command is not set.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_synced_project(tempdir, head=self.HEAD_ID) + proj.manifest.manifestProject.reproject_cmd = None + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + + self.assertTrue(syncbuf.Finish()) + proj._Checkout.assert_called_once_with( + self.REVID, force_checkout=False, quiet=True + ) + proj._Reproject.assert_not_called() + proj.work_git.DetachHead.assert_not_called() + + def test_metaproject_never_uses_the_command(self) -> None: + """Test .repo/manifests and .repo/repo are checked out by Git.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = _create_manifest_project(tempdir) + fakeproj.config.SetString("repo.reprojectcmd", "echo hi") + fakeproj.config.SetBoolean("repo.uselocalgitdirs", True) + self.assertFalse(fakeproj.UseReprojectCmd) + + def test_sync_reproject_cmd_requires_use_local_gitdirs(self) -> None: + """Test that repo.reprojectcmd requires repo.uselocalgitdirs.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = _create_manifest_project(tempdir) + + class DummyManifest: + is_submanifest = False + + def GetDefaultGroupsStr( + self, with_platform: bool = False + ) -> str: + return "" + + fakeproj.manifest = DummyManifest() + + fakeproj.config.SetString("repo.reprojectcmd", "echo hi") + fakeproj.config.SetBoolean("repo.uselocalgitdirs", False) + + result = fakeproj.Sync(use_local_gitdirs=False) + self.assertFalse(result) + + +class ReprojectCmdGitTests(unittest.TestCase): + """Tests running the reprojectcmd contract against a real Git checkout.""" + + READ_TREE = "git -C $REPO_PATH read-tree -m -u $REPO_TREV" + + @staticmethod + def _git(cwd: str, *args: str) -> str: + return subprocess.run( + ["git", "-C", cwd] + list(args), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stdout.rstrip("\n") + + def _make_client( + self, topdir: str, reproject_cmd: Optional[str] + ) -> Tuple[project.Project, str]: + """Set up a fetched, never checked out project under |topdir|. + + Returns: + The project and the commit its manifest revision names. + """ + # A remote holding the history the project fetches. + remote = os.path.join(topdir, "remote") + os.mkdir(remote) + self._git(remote, "init", "-q") + self._git(remote, "symbolic-ref", "HEAD", "refs/heads/main") + for msg, files in ( + ("one", {"one.txt": "one\n", "keep.txt": "keep\n"}), + ("two", {"one.txt": "one, revised\n", "two.txt": "two\n"}), + ): + for name, content in files.items(): + with open(os.path.join(remote, name), "w") as fp: + fp.write(content) + self._git(remote, "add", name) + self._git( + remote, + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-q", + "-m", + msg, + ) + revid = self._git(remote, "rev-parse", "HEAD") + + # The project as repo.fetchcmd leaves it: objects fetched, HEAD on an + # unborn branch, nothing in the index or the worktree. + worktree = os.path.join(topdir, "proj") + os.mkdir(worktree) + self._git(worktree, "init", "-q") + self._git(worktree, "symbolic-ref", "HEAD", "refs/heads/main") + self._git(worktree, "fetch", "-q", remote, "refs/heads/main") + + manifest = mock.MagicMock() + manifest.manifestProject.use_local_gitdirs = True + manifest.manifestProject.reproject_cmd = reproject_cmd + manifest.UseLocalGitDirs = True + manifest.IsMirror = False + manifest.is_multimanifest = False + manifest.topdir = topdir + manifest.path_prefix = "" + manifest.globalConfig = None + + remote_spec = mock.MagicMock() + remote_spec.name = "origin" + remote_spec.url = remote + + proj = project.Project( + manifest=manifest, + name="proj", + remote=remote_spec, + gitdir=os.path.join(worktree, ".git"), + objdir=os.path.join(worktree, ".git"), + worktree=worktree, + relpath="proj", + revisionExpr="main", + revisionId=revid, + ) + proj._Checkout = mock.MagicMock( + side_effect=AssertionError("Git must not do the checkout") + ) + return proj, revid + + def _sync(self, proj: project.Project) -> Tuple[bool, List[Any]]: + """Run Sync_LocalHalf; return whether it succeeded and its errors.""" + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + return syncbuf.Finish(), syncbuf.errors + + def _step_back(self, proj: project.Project, revid: str) -> str: + """Put the project cleanly at the commit before |revid|, with Git.""" + parent = self._git(proj.worktree, "rev-parse", revid + "~1") + self._git(proj.worktree, "update-ref", "--no-deref", "HEAD", parent) + self._git(proj.worktree, "read-tree", "-u", "--reset", "HEAD") + return parent + + def test_sync_local_half_checks_out_a_fresh_project(self) -> None: + """Test the read-tree command materializes a project like Git would.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + marker = os.path.join(topdir, "ran") + proj, revid = self._make_client( + topdir, f"{self.READ_TREE} && touch {marker}" + ) + worktree = proj.worktree + + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + self.assertTrue(syncbuf.Finish(), syncbuf.errors) + + self.assertTrue(os.path.exists(marker)) + self.assertEqual(self._git(worktree, "rev-parse", "HEAD"), revid) + with self.assertRaises(subprocess.CalledProcessError): + self._git(worktree, "symbolic-ref", "-q", "HEAD") + self.assertEqual(self._git(worktree, "status", "--porcelain"), "") + with open(os.path.join(worktree, "one.txt")) as fp: + self.assertEqual(fp.read(), "one, revised\n") + with open(os.path.join(worktree, "two.txt")) as fp: + self.assertEqual(fp.read(), "two\n") + + # Syncing again finds HEAD at the target and leaves it alone. + os.remove(marker) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + self.assertTrue(syncbuf.Finish(), syncbuf.errors) + self.assertFalse(os.path.exists(marker)) + self.assertEqual(self._git(worktree, "rev-parse", "HEAD"), revid) + + def test_sync_local_half_leaves_an_untracked_file_in_the_way_alone( + self, + ) -> None: + """Test an untracked file the target adds fails without any change.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client(topdir, self.READ_TREE) + worktree = proj.worktree + with open(os.path.join(worktree, "two.txt"), "w") as fp: + fp.write("mine\n") + + clean, errors = self._sync(proj) + self.assertFalse(clean) + self.assertIn("two.txt", str(errors[0])) + + self.assertEqual( + self._git(worktree, "symbolic-ref", "HEAD"), "refs/heads/main" + ) + self.assertEqual(sorted(os.listdir(worktree)), [".git", "two.txt"]) + with open(os.path.join(worktree, "two.txt")) as fp: + self.assertEqual(fp.read(), "mine\n") + + def test_sync_local_half_keeps_local_changes_out_of_the_way(self) -> None: + """Test edits and untracked files the target leaves alone survive.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client(topdir, self.READ_TREE) + worktree = proj.worktree + self.assertTrue(self._sync(proj)[0]) + self._step_back(proj, revid) + with open(os.path.join(worktree, "keep.txt"), "w") as fp: + fp.write("edited\n") + with open(os.path.join(worktree, "junk"), "w") as fp: + fp.write("junk\n") + + clean, errors = self._sync(proj) + self.assertTrue(clean, errors) + + self.assertEqual(self._git(worktree, "rev-parse", "HEAD"), revid) + with open(os.path.join(worktree, "one.txt")) as fp: + self.assertEqual(fp.read(), "one, revised\n") + with open(os.path.join(worktree, "keep.txt")) as fp: + self.assertEqual(fp.read(), "edited\n") + self.assertEqual( + self._git(worktree, "status", "--porcelain").splitlines(), + [" M keep.txt", "?? junk"], + ) + + def test_sync_local_half_rejects_a_local_change_in_the_way(self) -> None: + """Test an edit to a file the target changes fails without a change.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client(topdir, self.READ_TREE) + worktree = proj.worktree + self.assertTrue(self._sync(proj)[0]) + parent = self._step_back(proj, revid) + with open(os.path.join(worktree, "one.txt"), "w") as fp: + fp.write("edited\n") + + clean, errors = self._sync(proj) + self.assertFalse(clean) + self.assertIn("one.txt", str(errors[0])) + + self.assertEqual(self._git(worktree, "rev-parse", "HEAD"), parent) + with open(os.path.join(worktree, "one.txt")) as fp: + self.assertEqual(fp.read(), "edited\n") + self.assertNotIn("two.txt", os.listdir(worktree)) + + def test_sync_local_half_rejects_a_command_that_moves_head(self) -> None: + """Test a command that writes HEAD fails the postcondition.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client( + topdir, + f"{self.READ_TREE} && git -C $REPO_PATH update-ref " + "--no-deref HEAD $REPO_TREV", + ) + syncbuf = project.SyncBuffer(proj.config) + proj.Sync_LocalHalf(syncbuf) + self.assertFalse(syncbuf.Finish()) + self.assertIn("moved HEAD", str(syncbuf.errors[0])) + + def test_sync_local_half_rejects_staged_changes(self) -> None: + """Test a staged change fails before the command runs.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client(topdir, self.READ_TREE) + worktree = proj.worktree + self.assertTrue(self._sync(proj)[0]) + self._step_back(proj, revid) + with open(os.path.join(worktree, "keep.txt"), "w") as fp: + fp.write("staged\n") + self._git(worktree, "add", "keep.txt") + + clean, errors = self._sync(proj) + self.assertFalse(clean) + self.assertIn("staged changes", str(errors[0])) + self.assertIn("keep.txt", str(errors[0])) + + def test_sync_local_half_rejects_a_command_that_moves_branch_tip( + self, + ) -> None: + """Test a command that moves the branch commit fails postcondition.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client( + topdir, + f"{self.READ_TREE} && git -C $REPO_PATH update-ref " + "refs/heads/main $REPO_TREV", + ) + clean, errors = self._sync(proj) + self.assertFalse(clean) + self.assertIn("moved HEAD", str(errors[0])) + + def test_sync_local_half_keeps_local_changes_on_tracking_branch( + self, + ) -> None: + """Test benign edits survive fast-forward on a tracking branch.""" + with tempfile.TemporaryDirectory(prefix="repo-tests") as topdir: + proj, revid = self._make_client(topdir, self.READ_TREE) + worktree = proj.worktree + self.assertTrue(self._sync(proj)[0]) + parent = self._step_back(proj, revid) + self._git(worktree, "checkout", "-q", "-b", "main", parent) + self._git(worktree, "config", "branch.main.remote", "origin") + self._git( + worktree, "config", "branch.main.merge", "refs/heads/main" + ) + with open(os.path.join(worktree, "keep.txt"), "w") as fp: + fp.write("edited\n") + with open(os.path.join(worktree, "junk"), "w") as fp: + fp.write("junk\n") + + clean, errors = self._sync(proj) + self.assertTrue(clean, errors) + + self.assertEqual(self._git(worktree, "rev-parse", "HEAD"), revid) + with open(os.path.join(worktree, "one.txt")) as fp: + self.assertEqual(fp.read(), "one, revised\n") + with open(os.path.join(worktree, "keep.txt")) as fp: + self.assertEqual(fp.read(), "edited\n") + self.assertEqual( + self._git(worktree, "status", "--porcelain").splitlines(), + [" M keep.txt", "?? junk"], + ) diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index e929f22b9..39d9f03b1 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py @@ -13,6 +13,7 @@ # limitations under the License. """Unittests for the subcmds/sync.py module.""" +import contextlib import json import optparse import os @@ -20,7 +21,7 @@ from pathlib import Path import shutil import tempfile import time -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import unittest from unittest import mock @@ -516,6 +517,7 @@ class FakeProject: self.use_git_worktrees = False self.UseAlternates = False + self.UseReprojectCmd = False self.manifest = mock.MagicMock() self.manifest.GetProjectsWithName.return_value = [self] self.config = mock.MagicMock() @@ -642,6 +644,27 @@ class SafeCheckoutOrder(unittest.TestCase): ) +class NestedProjects(unittest.TestCase): + def test_flat_manifest(self) -> None: + p_foo = FakeProject("foo") + p_foo_bar = FakeProject("foo-bar") + self.assertEqual(sync._NestedProjects([p_foo, p_foo_bar]), []) + + def test_nested_paths(self) -> None: + p_foo = FakeProject("foo") + p_foo_bar = FakeProject("foo/bar") + p_foo_bar_baz = FakeProject("foo/bar/baz") + self.assertEqual( + sync._NestedProjects([p_foo_bar_baz, p_foo, p_foo_bar]), + [p_foo_bar, p_foo_bar_baz], + ) + + def test_submodule_of_a_parent(self) -> None: + parent = FakeProject("foo") + sub = FakeProject("foo/sub", parent=parent, is_derived=True) + self.assertEqual(sync._NestedProjects([parent, sub]), [sub]) + + class ParentFirstBatches(unittest.TestCase): def test_no_submodules(self) -> None: p_a = FakeProject("a") @@ -1083,7 +1106,10 @@ class SyncCommand(unittest.TestCase): self.project = p = mock.MagicMock( use_git_worktrees=False, UseAlternates=False, + UseReprojectCmd=False, name="project", + relpath="rel_path", + parent=None, Sync_NetworkHalf=Sync_NetworkHalf, Sync_LocalHalf=Sync_LocalHalf, RelPath=mock.Mock(return_value="rel_path"), @@ -1142,6 +1168,78 @@ class SyncCommand(unittest.TestCase): _, kwargs = self.cmd.GetProjects.call_args self.assertEqual(kwargs.get("groups"), "my_group") + def _ExecuteUntilSync( + self, args: List[str] + ) -> Tuple[mock.MagicMock, mock.MagicMock]: + """Run Execute up to the sync itself, returning the sync mocks.""" + self.opt.mp_update = False + with contextlib.ExitStack() as stack: + for name in ( + "_UpdateRepoProject", + "_UpdateProjectsRevisionId", + "_ValidateOptionsWithManifest", + "_RunPostSyncHook", + ): + stack.enter_context(mock.patch.object(self.cmd, name)) + phased = stack.enter_context( + mock.patch.object(self.cmd, "_SyncPhased") + ) + interleaved = stack.enter_context( + mock.patch.object(self.cmd, "_SyncInterleaved") + ) + self.cmd.Execute(self.opt, args) + return phased, interleaved + + def test_reproject_cmd_allows_a_flat_manifest(self) -> None: + """Ensure repo.reprojectcmd syncs a manifest without nesting.""" + self.project.UseReprojectCmd = True + phased, interleaved = self._ExecuteUntilSync([]) + self.assertTrue(phased.called or interleaved.called) + + def test_reproject_cmd_rejects_nested_projects(self) -> None: + """Ensure repo.reprojectcmd fails a manifest with nested projects.""" + p_foo = FakeProject("foo") + p_foo_bar = FakeProject("foo/bar") + p_foo.UseReprojectCmd = p_foo_bar.UseReprojectCmd = True + self.cmd.GetProjects.return_value = [p_foo, p_foo_bar] + with self.assertRaises(sync.SyncError) as e: + self._ExecuteUntilSync([]) + self.assertIn("foo/bar", str(e.exception)) + self.assertNotIn(" - foo\n", str(e.exception)) + + def test_reproject_cmd_rejects_a_submodule(self) -> None: + """Ensure repo.reprojectcmd fails a manifest with a submodule.""" + p_foo = FakeProject("foo") + p_sub = FakeProject("foo/sub", parent=p_foo, is_derived=True) + p_foo.UseReprojectCmd = p_sub.UseReprojectCmd = True + self.cmd.GetProjects.return_value = [p_foo, p_sub] + with self.assertRaises(sync.SyncError) as e: + self._ExecuteUntilSync([]) + self.assertIn("foo/sub", str(e.exception)) + + def test_reproject_cmd_checks_the_whole_manifest(self) -> None: + """Ensure nesting is checked beyond the projects given as args.""" + p_foo = FakeProject("foo") + p_foo_bar = FakeProject("foo/bar") + p_foo.UseReprojectCmd = p_foo_bar.UseReprojectCmd = True + self.cmd.GetProjects.side_effect = lambda args, **kwargs: ( + [p_foo_bar] if args else [p_foo, p_foo_bar] + ) + with self.assertRaises(sync.SyncError): + self._ExecuteUntilSync(["foo/bar"]) + self.assertEqual(self.cmd.GetProjects.call_count, 2) + _, kwargs = self.cmd.GetProjects.call_args + self.assertEqual(kwargs.get("missing_ok"), True) + + def test_reproject_cmd_off_ignores_nested_projects(self) -> None: + """Ensure nesting is only checked with repo.reprojectcmd in use.""" + projects = [FakeProject("foo"), FakeProject("foo/bar")] + for p in projects: + p.Exists = False + self.cmd.GetProjects.return_value = projects + phased, interleaved = self._ExecuteUntilSync([]) + self.assertTrue(phased.called or interleaved.called) + class SyncUpdateRepoProject(unittest.TestCase): """Tests for Sync._UpdateRepoProject."""