mirrors/git-repo
1
0
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2026-06-06 09:59:44 +00:00
Code Issues Projects Releases Wiki Activity

manifest: Introduce sync-j-max attribute to cap sync jobs

Browse Source 
Add a way for manifest owners to limit how many sync jobs run in
parallel.

Bug: 481100878
Change-Id: Ia6cbe02cbc83c9e414b53b8d14fe5e7e1b802505
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/548963
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
This commit is contained in:
Gavin Mak
2026-02-03 20:59:23 +00:00
committed by LUCI
parent 62cd0de6cf
commit a214fd31bd
6 changed files with 101 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/manifest-format.md
+4 -1
View File
@@ -51,6 +51,7 @@ following DTD:
<!ATTLIST default dest-branch CDATA #IMPLIED>
<!ATTLIST default upstream CDATA #IMPLIED>
<!ATTLIST default sync-j CDATA #IMPLIED>
<!ATTLIST default sync-j-max CDATA #IMPLIED>
<!ATTLIST default sync-c CDATA #IMPLIED>
<!ATTLIST default sync-s CDATA #IMPLIED>
<!ATTLIST default sync-tags CDATA #IMPLIED>
@@ -213,7 +214,9 @@ can be found. Used when syncing a revision locked manifest in
-c mode to avoid having to sync the entire ref space. Project elements
not setting their own `upstream` will inherit this value.
Attribute `sync-j`: Number of parallel jobs to use when synching.
Attribute `sync-j`: Number of parallel jobs to use when syncing.
Attribute `sync-j-max`: Maximum number of parallel jobs to use when syncing.
Attribute `sync-c`: Set to true to only sync the given Git
branch (specified in the `revision` attribute) rather than the
man/repo-manifest.1
+5 -2
View File
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
.TH REPO "1" "December 2025" "repo manifest" "Repo Manual"
.TH REPO "1" "February 2026" "repo manifest" "Repo Manual"
.SH NAME
repo \- repo manifest - manual page for repo manifest
.SH SYNOPSIS
@@ -131,6 +131,7 @@ include*)>
<!ATTLIST default dest\-branch CDATA #IMPLIED>
<!ATTLIST default upstream CDATA #IMPLIED>
<!ATTLIST default sync\-j CDATA #IMPLIED>
<!ATTLIST default sync\-j\-max CDATA #IMPLIED>
<!ATTLIST default sync\-c CDATA #IMPLIED>
<!ATTLIST default sync\-s CDATA #IMPLIED>
<!ATTLIST default sync\-tags CDATA #IMPLIED>
@@ -309,7 +310,9 @@ when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to syn
entire ref space. Project elements not setting their own `upstream` will inherit
this value.
.PP
Attribute `sync\-j`: Number of parallel jobs to use when synching.
Attribute `sync\-j`: Number of parallel jobs to use when syncing.
.PP
Attribute `sync\-j\-max`: Maximum number of parallel jobs to use when syncing.
.PP
Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
the `revision` attribute) rather than the whole ref space. Project elements
manifest_xml.py
+11
View File
@@ -155,6 +155,7 @@ class _Default:
upstreamExpr = None
remote = None
sync_j = None
sync_j_max = None
sync_c = False
sync_s = False
sync_tags = True
@@ -631,6 +632,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
if d.sync_j is not None:
have_default = True
e.setAttribute("sync-j", "%d" % d.sync_j)
if d.sync_j_max is not None:
have_default = True
e.setAttribute("sync-j-max", "%d" % d.sync_j_max)
if d.sync_c:
have_default = True
e.setAttribute("sync-c", "true")
@@ -1763,6 +1767,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
% (self.manifestFile, d.sync_j)
)
d.sync_j_max = XmlInt(node, "sync-j-max", None)
if d.sync_j_max is not None and d.sync_j_max <= 0:
raise ManifestParseError(
'%s: sync-j-max must be greater than 0, not "%s"'
% (self.manifestFile, d.sync_j_max)
)
d.sync_c = XmlBool(node, "sync-c", False)
d.sync_s = XmlBool(node, "sync-s", False)
d.sync_tags = XmlBool(node, "sync-tags", True)
subcmds/sync.py
+26 -8
View File
@@ -1940,15 +1940,33 @@ later is required to fix a server side protocol bug.
opt.jobs_network = min(opt.jobs_network, jobs_soft_limit)
opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit)
# Warn once if effective job counts seem excessively high.
sync_j_max = mp.manifest.default.sync_j_max or None
# Check for shared options.
# Prioritize --jobs, then --jobs-network, then --jobs-checkout.
job_options_to_check = (
("--jobs", opt.jobs),
("--jobs-network", opt.jobs_network),
("--jobs-checkout", opt.jobs_checkout),
job_attributes = (
("--jobs", "jobs"),
("--jobs-network", "jobs_network"),
("--jobs-checkout", "jobs_checkout"),
)
for name, value in job_options_to_check:
if value > self._JOBS_WARN_THRESHOLD:
warned = False
limit_warned = False
for name, attr in job_attributes:
value = getattr(opt, attr)
if sync_j_max and value > sync_j_max:
if not limit_warned:
logger.warning(
"warning: manifest limits %s to %d",
name,
sync_j_max,
)
limit_warned = True
setattr(opt, attr, sync_j_max)
value = sync_j_max
if not warned and value > self._JOBS_WARN_THRESHOLD:
logger.warning(
"High job count (%d > %d) specified for %s; this may "
"lead to excessive resource usage or diminishing returns.",
@@ -1956,7 +1974,7 @@ later is required to fix a server side protocol bug.
self._JOBS_WARN_THRESHOLD,
name,
)
break
warned = True
def Execute(self, opt, args):
errors = []
tests/test_manifest_xml.py
+26
View File
@@ -401,6 +401,32 @@ class XmlManifestTests(ManifestParseTestCase):
self.assertEqual(len(manifest.projects), 1)
self.assertEqual(manifest.projects[0].name, "test-project")
def test_sync_j_max(self):
"""Check sync-j-max handling."""
# Check valid value.
manifest = self.getXmlManifest(
'<manifest><default sync-j-max="5" /></manifest>'
)
self.assertEqual(manifest.default.sync_j_max, 5)
self.assertEqual(
manifest.ToXml().toxml(),
'<?xml version="1.0" ?>'
'<manifest><default sync-j-max="5"/></manifest>',
)
# Check invalid values.
with self.assertRaises(error.ManifestParseError):
manifest = self.getXmlManifest(
'<manifest><default sync-j-max="0" /></manifest>'
)
manifest.ToXml()
with self.assertRaises(error.ManifestParseError):
manifest = self.getXmlManifest(
'<manifest><default sync-j-max="-1" /></manifest>'
)
manifest.ToXml()
class IncludeElementTests(ManifestParseTestCase):
"""Tests for <include>."""
tests/test_subcmds_sync.py
+29
View File
@@ -97,6 +97,35 @@ def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check):
"""Tests --jobs option behavior."""
mp = mock.MagicMock()
mp.manifest.default.sync_j = jobs_manifest
mp.manifest.default.sync_j_max = None
cmd = sync.Sync()
opts, args = cmd.OptionParser.parse_args(argv)
cmd.ValidateOptions(opts, args)
with mock.patch.object(sync, "_rlimit_nofile", return_value=(256, 256)):
with mock.patch.object(os, "cpu_count", return_value=OS_CPU_COUNT):
cmd._ValidateOptionsWithManifest(opts, mp)
assert opts.jobs == jobs
assert opts.jobs_network == jobs_net
assert opts.jobs_checkout == jobs_check
@pytest.mark.parametrize(
"argv, jobs_manifest, jobs_manifest_max, jobs, jobs_net, jobs_check",
[
(["--jobs=10"], None, 5, 5, 5, 5),
(["--jobs=10", "--jobs-network=10"], None, 5, 5, 5, 5),
(["--jobs=10", "--jobs-checkout=10"], None, 5, 5, 5, 5),
],
)
def test_cli_jobs_sync_j_max(
argv, jobs_manifest, jobs_manifest_max, jobs, jobs_net, jobs_check
):
"""Tests --jobs option behavior with sync-j-max."""
mp = mock.MagicMock()
mp.manifest.default.sync_j = jobs_manifest
mp.manifest.default.sync_j_max = jobs_manifest_max
cmd = sync.Sync()
opts, args = cmd.OptionParser.parse_args(argv)