From a214fd31bd477dda2708ab9f3bc675081ceff696 Mon Sep 17 00:00:00 2001 From: Gavin Mak Date: Tue, 3 Feb 2026 20:59:23 +0000 Subject: [PATCH] manifest: Introduce `sync-j-max` attribute to cap sync jobs Add a way for manifest owners to limit how many sync jobs run in parallel. Bug: 481100878 Change-Id: Ia6cbe02cbc83c9e414b53b8d14fe5e7e1b802505 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/548963 Reviewed-by: Mike Frysinger Tested-by: Gavin Mak Commit-Queue: Gavin Mak --- docs/manifest-format.md | 5 ++++- man/repo-manifest.1 | 7 +++++-- manifest_xml.py | 11 +++++++++++ subcmds/sync.py | 34 ++++++++++++++++++++++++++-------- tests/test_manifest_xml.py | 26 ++++++++++++++++++++++++++ tests/test_subcmds_sync.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 101 insertions(+), 11 deletions(-) diff --git a/docs/manifest-format.md b/docs/manifest-format.md index 1eead91de..c3cbe0707 100644 --- a/docs/manifest-format.md +++ b/docs/manifest-format.md @@ -51,6 +51,7 @@ following DTD: + @@ -213,7 +214,9 @@ can be found. Used when syncing a revision locked manifest in -c mode to avoid having to sync the entire ref space. Project elements not setting their own `upstream` will inherit this value. -Attribute `sync-j`: Number of parallel jobs to use when synching. +Attribute `sync-j`: Number of parallel jobs to use when syncing. + +Attribute `sync-j-max`: Maximum number of parallel jobs to use when syncing. Attribute `sync-c`: Set to true to only sync the given Git branch (specified in the `revision` attribute) rather than the diff --git a/man/repo-manifest.1 b/man/repo-manifest.1 index dfb0160ec..4d650c155 100644 --- a/man/repo-manifest.1 +++ b/man/repo-manifest.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man. -.TH REPO "1" "December 2025" "repo manifest" "Repo Manual" +.TH REPO "1" "February 2026" "repo manifest" "Repo Manual" .SH NAME repo \- repo manifest - manual page for repo manifest .SH SYNOPSIS @@ -131,6 +131,7 @@ include*)> + @@ -309,7 +310,9 @@ when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to syn entire ref space. Project elements not setting their own `upstream` will inherit this value. .PP -Attribute `sync\-j`: Number of parallel jobs to use when synching. +Attribute `sync\-j`: Number of parallel jobs to use when syncing. +.PP +Attribute `sync\-j\-max`: Maximum number of parallel jobs to use when syncing. .PP Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in the `revision` attribute) rather than the whole ref space. Project elements diff --git a/manifest_xml.py b/manifest_xml.py index 6989aad53..084ca5ab2 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -155,6 +155,7 @@ class _Default: upstreamExpr = None remote = None sync_j = None + sync_j_max = None sync_c = False sync_s = False sync_tags = True @@ -631,6 +632,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if d.sync_j is not None: have_default = True e.setAttribute("sync-j", "%d" % d.sync_j) + if d.sync_j_max is not None: + have_default = True + e.setAttribute("sync-j-max", "%d" % d.sync_j_max) if d.sync_c: have_default = True e.setAttribute("sync-c", "true") @@ -1763,6 +1767,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md % (self.manifestFile, d.sync_j) ) + d.sync_j_max = XmlInt(node, "sync-j-max", None) + if d.sync_j_max is not None and d.sync_j_max <= 0: + raise ManifestParseError( + '%s: sync-j-max must be greater than 0, not "%s"' + % (self.manifestFile, d.sync_j_max) + ) + d.sync_c = XmlBool(node, "sync-c", False) d.sync_s = XmlBool(node, "sync-s", False) d.sync_tags = XmlBool(node, "sync-tags", True) diff --git a/subcmds/sync.py b/subcmds/sync.py index 726e6d079..89b58e6aa 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -1940,15 +1940,33 @@ later is required to fix a server side protocol bug. opt.jobs_network = min(opt.jobs_network, jobs_soft_limit) opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit) - # Warn once if effective job counts seem excessively high. + sync_j_max = mp.manifest.default.sync_j_max or None + + # Check for shared options. # Prioritize --jobs, then --jobs-network, then --jobs-checkout. - job_options_to_check = ( - ("--jobs", opt.jobs), - ("--jobs-network", opt.jobs_network), - ("--jobs-checkout", opt.jobs_checkout), + job_attributes = ( + ("--jobs", "jobs"), + ("--jobs-network", "jobs_network"), + ("--jobs-checkout", "jobs_checkout"), ) - for name, value in job_options_to_check: - if value > self._JOBS_WARN_THRESHOLD: + + warned = False + limit_warned = False + for name, attr in job_attributes: + value = getattr(opt, attr) + + if sync_j_max and value > sync_j_max: + if not limit_warned: + logger.warning( + "warning: manifest limits %s to %d", + name, + sync_j_max, + ) + limit_warned = True + setattr(opt, attr, sync_j_max) + value = sync_j_max + + if not warned and value > self._JOBS_WARN_THRESHOLD: logger.warning( "High job count (%d > %d) specified for %s; this may " "lead to excessive resource usage or diminishing returns.", @@ -1956,7 +1974,7 @@ later is required to fix a server side protocol bug. self._JOBS_WARN_THRESHOLD, name, ) - break + warned = True def Execute(self, opt, args): errors = [] diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index 97fea3da3..75efa95fc 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py @@ -401,6 +401,32 @@ class XmlManifestTests(ManifestParseTestCase): self.assertEqual(len(manifest.projects), 1) self.assertEqual(manifest.projects[0].name, "test-project") + def test_sync_j_max(self): + """Check sync-j-max handling.""" + # Check valid value. + manifest = self.getXmlManifest( + '' + ) + self.assertEqual(manifest.default.sync_j_max, 5) + self.assertEqual( + manifest.ToXml().toxml(), + '' + '', + ) + + # Check invalid values. + with self.assertRaises(error.ManifestParseError): + manifest = self.getXmlManifest( + '' + ) + manifest.ToXml() + + with self.assertRaises(error.ManifestParseError): + manifest = self.getXmlManifest( + '' + ) + manifest.ToXml() + class IncludeElementTests(ManifestParseTestCase): """Tests for .""" diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index 6eb8a5a71..9fef68425 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py @@ -97,6 +97,35 @@ def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check): """Tests --jobs option behavior.""" mp = mock.MagicMock() mp.manifest.default.sync_j = jobs_manifest + mp.manifest.default.sync_j_max = None + + cmd = sync.Sync() + opts, args = cmd.OptionParser.parse_args(argv) + cmd.ValidateOptions(opts, args) + + with mock.patch.object(sync, "_rlimit_nofile", return_value=(256, 256)): + with mock.patch.object(os, "cpu_count", return_value=OS_CPU_COUNT): + cmd._ValidateOptionsWithManifest(opts, mp) + assert opts.jobs == jobs + assert opts.jobs_network == jobs_net + assert opts.jobs_checkout == jobs_check + + +@pytest.mark.parametrize( + "argv, jobs_manifest, jobs_manifest_max, jobs, jobs_net, jobs_check", + [ + (["--jobs=10"], None, 5, 5, 5, 5), + (["--jobs=10", "--jobs-network=10"], None, 5, 5, 5, 5), + (["--jobs=10", "--jobs-checkout=10"], None, 5, 5, 5, 5), + ], +) +def test_cli_jobs_sync_j_max( + argv, jobs_manifest, jobs_manifest_max, jobs, jobs_net, jobs_check +): + """Tests --jobs option behavior with sync-j-max.""" + mp = mock.MagicMock() + mp.manifest.default.sync_j = jobs_manifest + mp.manifest.default.sync_j_max = jobs_manifest_max cmd = sync.Sync() opts, args = cmd.OptionParser.parse_args(argv)