Files
git-repo/subcmds/info.py
T
Gavin Mak 91986011b0 info: Parallelize project data gathering for JSON output
https://gerrit-review.googlesource.com/c/git-repo/+/581921 parallelized
`repo info` for text output. This commit does the same for JSON output
format.

Benchmarked `repo info --format=json` on an Android workspace with ~3k
projects (N=3):
- Before (sequential): 1m 30s average
- After (parallelized): 46s average (~2x speedup)

Verified that the JSON output is identical before and after.

Bug: 526685287
Change-Id: If573223aba584f8b932f87d29e34ed565c5c930a
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/601861
Commit-Queue: Gavin Mak <gavinmak@google.com>
Reviewed-by: Brian Gan <brgan@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
2026-06-30 09:19:35 -07:00

481 lines
15 KiB
Python

# Copyright (C) 2012 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import enum
import functools
import io
import json
import optparse
import sys
from typing import Any, Dict, List, NamedTuple
from color import Coloring
from command import DEFAULT_LOCAL_JOBS
from command import PagedCommand
from git_refs import R_HEADS
from git_refs import R_M
class BranchInfo(NamedTuple):
"""Holds information about a branch in a project."""
relpath: str
name: str
commits: Any
date: str
is_current: bool
class OutputFormat(enum.Enum):
"""Type for the requested output format."""
# Human-readable text output.
TEXT = enum.auto()
# Machine-readable JSON output.
JSON = enum.auto()
class _Coloring(Coloring):
def __init__(self, config):
Coloring.__init__(self, config, "status")
class Info(PagedCommand):
COMMON = True
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
helpSummary = (
"Get info on the manifest branch, current branch or unmerged branches"
)
helpUsage = "%prog [-dl] [-o [-c]] [--format=<format>] [<project>...]"
def _Options(self, p):
p.add_option(
"-d",
"--diff",
dest="all",
action="store_true",
help="show full info and commit diff including remote branches",
)
p.add_option(
"-o",
"--overview",
action="store_true",
help="show overview of all local commits",
)
p.add_option(
"--include-summary",
action="store_true",
default=True,
help="include manifest summary (default: true)",
)
p.add_option(
"--no-include-summary",
dest="include_summary",
action="store_false",
help="exclude manifest summary",
)
p.add_option(
"--include-projects",
action="store_true",
default=True,
help="include project details (default: true)",
)
p.add_option(
"--no-include-projects",
dest="include_projects",
action="store_false",
help="exclude project details",
)
p.add_option(
"-c",
"--current-branch",
action="store_true",
help="consider only checked out branches",
)
p.add_option(
"--no-current-branch",
dest="current_branch",
action="store_false",
help="consider all local branches",
)
# Turn this into a warning & remove this someday.
p.add_option(
"-b",
dest="current_branch",
action="store_true",
help=optparse.SUPPRESS_HELP,
)
p.add_option(
"-l",
"--local-only",
dest="local",
action="store_true",
help="disable all remote operations",
)
formats = tuple(x.lower() for x in OutputFormat.__members__.keys())
p.add_option(
"--format",
default=OutputFormat.TEXT.name.lower(),
choices=formats,
help=f"output format: {', '.join(formats)} (default: %default)",
)
def WantPager(self, opt):
return OutputFormat[opt.format.upper()] == OutputFormat.TEXT
def ValidateOptions(self, opt, args):
output_format = OutputFormat[opt.format.upper()]
if output_format == OutputFormat.JSON:
if opt.all:
self.OptionParser.error("--diff is not supported with JSON")
if opt.overview:
self.OptionParser.error("--overview is not supported with JSON")
def Execute(self, opt, args):
if not opt.this_manifest_only:
self.manifest = self.manifest.outer_client
self.TryOverrideManifestWithSmartSync()
output_format = OutputFormat[opt.format.upper()]
if output_format == OutputFormat.JSON:
self._ExecuteJson(opt, args)
else:
self._ExecuteText(opt, args)
def _ExecuteText(self, opt, args) -> None:
"""Output info as human-readable text."""
self.out = _Coloring(self.client.globalConfig)
self.heading = self.out.printer("heading", attr="bold")
self.headtext = self.out.nofmt_printer("headtext", fg="yellow")
self.redtext = self.out.printer("redtext", fg="red")
self.sha = self.out.printer("sha", fg="yellow")
self.text = self.out.nofmt_printer("text")
self.dimtext = self.out.printer("dimtext", attr="dim")
self.opt = opt
if opt.include_summary:
self._printSummary()
if not opt.include_projects:
return
elif not opt.overview:
self._printDiffInfo(opt, args)
else:
self._printCommitOverview(opt, args)
def _getSummaryData(self) -> Dict[str, Any]:
"""Gather manifest summary data as a dict."""
manifestConfig = self.manifest.manifestProject.config
mergeBranch = manifestConfig.GetBranch("default").merge
manifestGroups = self.manifest.GetManifestGroupsStr()
sp = self.manifest.superproject
srev = sp.commit_id if sp and sp.commit_id else None
return {
"manifest_branch": self.manifest.default.revisionExpr or "",
"manifest_merge_branch": mergeBranch or "",
"manifest_groups": manifestGroups,
"superproject_revision": srev,
}
@classmethod
def _getProjectData(cls, project) -> Dict[str, Any]:
"""Gather project data as a dict."""
data = {
"name": project.name,
"mount_path": project.worktree,
"current_revision": project.GetHeadRevisionId()
or project.GetRevisionId(),
"manifest_revision": project.revisionExpr,
"local_branches": list(project.GetBranches()),
}
currentBranch = project.CurrentBranch
if currentBranch:
data["current_branch"] = currentBranch
return data
@classmethod
def _ProjectDataHelper(cls, project_idx: int) -> Dict[str, Any]:
"""Helper to get project data in parallel."""
project = cls.get_parallel_context()["projects"][project_idx]
return cls._getProjectData(project)
def _ExecuteJson(self, opt, args) -> None:
"""Output info as JSON."""
result = {}
if opt.include_summary:
result["summary"] = self._getSummaryData()
if opt.include_projects:
projs = self.GetProjects(
args, all_manifests=not opt.this_manifest_only
)
project_data = []
def _ProcessResults(_pool, _output, results):
project_data.extend(results)
with self.ParallelContext():
self.get_parallel_context()["projects"] = projs
self.ExecuteInParallel(
opt.jobs,
self._ProjectDataHelper,
range(len(projs)),
callback=_ProcessResults,
ordered=True,
chunksize=1,
)
result["projects"] = project_data
json_settings = {
# JSON style guide says Unicode characters are fully allowed.
"ensure_ascii": False,
# We use 2 space indent to match JSON style guide.
"indent": 2,
"separators": (",", ": "),
"sort_keys": True,
}
sys.stdout.write(json.dumps(result, **json_settings) + "\n")
def _printSummary(self) -> None:
"""Print manifest summary in text format."""
data = self._getSummaryData()
self.heading("Manifest branch: ")
self.headtext(data["manifest_branch"])
self.out.nl()
self.heading("Manifest merge branch: ")
self.headtext(data["manifest_merge_branch"])
self.out.nl()
self.heading("Manifest groups: ")
self.headtext(data["manifest_groups"])
self.out.nl()
self.heading("Superproject revision: ")
self.headtext(data["superproject_revision"] or "None")
self.out.nl()
self.printSeparator()
def printSeparator(self):
self.text("----------------------------")
self.out.nl()
@classmethod
def _DiffHelper(cls, project_idx: int, opt: Any) -> str:
"""Helper for ParallelContext to get diff info for a project."""
buf = io.StringIO()
project = cls.get_parallel_context()["projects"][project_idx]
config = cls.get_parallel_context()["config"]
out = _Coloring(config)
out.redirect(buf)
heading = out.printer("heading", attr="bold")
headtext = out.nofmt_printer("headtext", fg="yellow")
redtext = out.printer("redtext", fg="red")
sha = out.printer("sha", fg="yellow")
text = out.nofmt_printer("text")
dimtext = out.printer("dimtext", attr="dim")
heading("Project: ")
headtext(project.name)
out.nl()
heading("Mount path: ")
headtext(project.worktree)
out.nl()
heading("Current revision: ")
headtext(project.GetHeadRevisionId() or project.GetRevisionId())
out.nl()
currentBranch = project.CurrentBranch
if currentBranch:
heading("Current branch: ")
headtext(currentBranch)
out.nl()
heading("Manifest revision: ")
headtext(project.revisionExpr)
out.nl()
localBranches = list(project.GetBranches().keys())
heading("Local Branches: ")
redtext(str(len(localBranches)))
if localBranches:
text(" [")
text(", ".join(localBranches))
text("]")
out.nl()
if opt.all:
if not opt.local:
project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
branch = project.manifest.manifestProject.config.GetBranch(
"default"
).merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS) :]
logTarget = R_M + branch
bareTmp = project.bare_git._bare
project.bare_git._bare = False
localCommits = project.bare_git.rev_list(
"--abbrev=8",
"--abbrev-commit",
"--pretty=oneline",
logTarget + "..",
"--",
)
originCommits = project.bare_git.rev_list(
"--abbrev=8",
"--abbrev-commit",
"--pretty=oneline",
".." + logTarget,
"--",
)
project.bare_git._bare = bareTmp
heading("Local Commits: ")
redtext(str(len(localCommits)))
dimtext(" (on current branch)")
out.nl()
for c in localCommits:
split = c.split()
sha(split[0] + " ")
text(" ".join(split[1:]))
out.nl()
text("----------------------------")
out.nl()
heading("Remote Commits: ")
redtext(str(len(originCommits)))
out.nl()
for c in originCommits:
split = c.split()
sha(split[0] + " ")
text(" ".join(split[1:]))
out.nl()
text("----------------------------")
out.nl()
return buf.getvalue()
def _printDiffInfo(self, opt, args):
projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
def _ProcessResults(_pool, _output, results):
for output in results:
if output:
print(output, end="")
with self.ParallelContext():
self.get_parallel_context()["projects"] = projs
self.get_parallel_context()[
"config"
] = self.manifest.manifestProject.config
self.ExecuteInParallel(
opt.jobs,
functools.partial(self._DiffHelper, opt=opt),
range(len(projs)),
callback=_ProcessResults,
ordered=True,
chunksize=1,
)
@classmethod
def _OverviewHelper(cls, project_idx: int, opt: Any) -> List[BranchInfo]:
"""Helper to get overview of uploadable branches."""
project = cls.get_parallel_context()["projects"][project_idx]
branches = []
br = [project.GetUploadableBranch(x) for x in project.GetBranches()]
br = [x for x in br if x]
if opt.current_branch:
br = [x for x in br if x.name == project.CurrentBranch]
for b in br:
branches.append(
BranchInfo(
relpath=project.RelPath(local=opt.this_manifest_only),
name=b.name,
commits=b.commits,
date=b.date,
is_current=b.name == project.CurrentBranch,
)
)
return branches
def _printCommitOverview(self, opt, args):
projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
all_branches = []
def _ProcessResults(_pool, _output, results):
for branches in results:
all_branches.extend(branches)
with self.ParallelContext():
self.get_parallel_context()["projects"] = projs
self.ExecuteInParallel(
opt.jobs,
functools.partial(self._OverviewHelper, opt=opt),
range(len(projs)),
callback=_ProcessResults,
ordered=True,
chunksize=1,
)
if not all_branches:
return
self.out.nl()
self.heading("Projects Overview")
current_relpath = None
for branch in all_branches:
if current_relpath != branch.relpath:
current_relpath = branch.relpath
self.out.nl()
self.headtext(current_relpath)
self.out.nl()
commits = branch.commits
date = branch.date
self.text(
"%s %-33s (%2d commit%s, %s)"
% (
branch.is_current and "*" or " ",
branch.name,
len(commits),
len(commits) != 1 and "s" or "",
date,
)
)
self.out.nl()
for commit in commits:
split = commit.split()
self.text(f"{'':38}{'-'} ")
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()