mirror of
https://gerrit.googlesource.com/git-repo
synced 2026-07-16 13:47:03 +00:00
info: Parallelize project data gathering for JSON output
https://gerrit-review.googlesource.com/c/git-repo/+/581921 parallelized `repo info` for text output. This commit does the same for JSON output format. Benchmarked `repo info --format=json` on an Android workspace with ~3k projects (N=3): - Before (sequential): 1m 30s average - After (parallelized): 46s average (~2x speedup) Verified that the JSON output is identical before and after. Bug: 526685287 Change-Id: If573223aba584f8b932f87d29e34ed565c5c930a Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/601861 Commit-Queue: Gavin Mak <gavinmak@google.com> Reviewed-by: Brian Gan <brgan@google.com> Tested-by: Gavin Mak <gavinmak@google.com>
This commit is contained in:
committed by
gerrit-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
a27dbcdb7b
commit
91986011b0
+24
-2
@@ -191,7 +191,8 @@ class Info(PagedCommand):
|
||||
"superproject_revision": srev,
|
||||
}
|
||||
|
||||
def _getProjectData(self, project) -> Dict[str, Any]:
|
||||
@classmethod
|
||||
def _getProjectData(cls, project) -> Dict[str, Any]:
|
||||
"""Gather project data as a dict."""
|
||||
data = {
|
||||
"name": project.name,
|
||||
@@ -206,6 +207,12 @@ class Info(PagedCommand):
|
||||
data["current_branch"] = currentBranch
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def _ProjectDataHelper(cls, project_idx: int) -> Dict[str, Any]:
|
||||
"""Helper to get project data in parallel."""
|
||||
project = cls.get_parallel_context()["projects"][project_idx]
|
||||
return cls._getProjectData(project)
|
||||
|
||||
def _ExecuteJson(self, opt, args) -> None:
|
||||
"""Output info as JSON."""
|
||||
result = {}
|
||||
@@ -215,7 +222,22 @@ class Info(PagedCommand):
|
||||
projs = self.GetProjects(
|
||||
args, all_manifests=not opt.this_manifest_only
|
||||
)
|
||||
result["projects"] = [self._getProjectData(p) for p in projs]
|
||||
project_data = []
|
||||
|
||||
def _ProcessResults(_pool, _output, results):
|
||||
project_data.extend(results)
|
||||
|
||||
with self.ParallelContext():
|
||||
self.get_parallel_context()["projects"] = projs
|
||||
self.ExecuteInParallel(
|
||||
opt.jobs,
|
||||
self._ProjectDataHelper,
|
||||
range(len(projs)),
|
||||
callback=_ProcessResults,
|
||||
ordered=True,
|
||||
chunksize=1,
|
||||
)
|
||||
result["projects"] = project_data
|
||||
|
||||
json_settings = {
|
||||
# JSON style guide says Unicode characters are fully allowed.
|
||||
|
||||
@@ -223,3 +223,33 @@ def test_get_project_data_uses_head_revision() -> None:
|
||||
project.GetHeadRevisionId.return_value = None
|
||||
data = cmd._getProjectData(project)
|
||||
assert data["current_revision"] == "manifest_sha_54321"
|
||||
|
||||
|
||||
def test_json_with_projects(capsys) -> None:
|
||||
"""--format=json should emit project data."""
|
||||
cmd = _get_cmd()
|
||||
opts, args = cmd.OptionParser.parse_args(["--format=json"])
|
||||
opts.jobs = 1 # To avoid multiprocessing pickle issues with mocks
|
||||
|
||||
project = mock.MagicMock()
|
||||
project.name = "foo"
|
||||
project.worktree = "/path/to/foo"
|
||||
project.revisionExpr = "refs/heads/main"
|
||||
project.GetBranches.return_value = {"branch1": mock.MagicMock()}
|
||||
project.GetHeadRevisionId.return_value = "head_sha_12345"
|
||||
project.CurrentBranch = "branch1"
|
||||
|
||||
cmd.GetProjects = mock.MagicMock(return_value=[project])
|
||||
|
||||
cmd.Execute(opts, args)
|
||||
|
||||
data = json.loads(capsys.readouterr().out)
|
||||
assert "projects" in data
|
||||
assert len(data["projects"]) == 1
|
||||
project_data = data["projects"][0]
|
||||
assert project_data["name"] == "foo"
|
||||
assert project_data["mount_path"] == "/path/to/foo"
|
||||
assert project_data["current_revision"] == "head_sha_12345"
|
||||
assert project_data["manifest_revision"] == "refs/heads/main"
|
||||
assert project_data["local_branches"] == ["branch1"]
|
||||
assert project_data["current_branch"] == "branch1"
|
||||
|
||||
Reference in New Issue
Block a user