From df01b29f3f54a82886a2852dcdd9c0359ecfee81 Mon Sep 17 00:00:00 2001 From: Peter Hoyes Date: Wed, 13 Sep 2023 15:03:42 +0100 Subject: [PATCH] CI: Make update-repos more resilient to network issues The update-repos script currently exits immediately if one of the underlying Git commands fails (e.g. because of a network issue). If the repo already exists, then catch this error inside the loop and carrying on attempting to update other repos, as the network error may be upstream. KAS_REPO_REF_DIR is ultimately an optimization and subsequent build stages should be able to continue if one of the updates fail. Therefore, ensure the script returns a special error code if at least of the Git commands fail, and use this to set the allow_failure property of the job. If a repo does not exist, fail immediately as before. Signed-off-by: Peter Hoyes Signed-off-by: Jon Mason --- .gitlab-ci.yml | 2 ++ ci/update-repos | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06fba597..c750698d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,6 +77,8 @@ stages: update-repos: extends: .setup stage: prep + allow_failure: + exit_codes: 128 script: - | flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos diff --git a/ci/update-repos b/ci/update-repos index 9487102d..069a94ea 100755 --- a/ci/update-repos +++ b/ci/update-repos @@ -32,6 +32,7 @@ if __name__ == "__main__": sys.exit(1) base_repodir = pathlib.Path(os.environ["KAS_REPO_REF_DIR"]) + failed = False for repo in repositories: repodir = base_repodir / repo_shortname(repo) @@ -41,8 +42,15 @@ if __name__ == "__main__": shutil.rmtree(repodir, ignore_errors=True) if repodir.exists(): - print("Updating %s..." % repo) - subprocess.run(["git", "-C", repodir, "-c", "gc.autoDetach=false", "fetch"], check=True) + try: + print("Updating %s..." % repo) + subprocess.run(["git", "-C", repodir, "-c", "gc.autoDetach=false", "fetch"], check=True) + except subprocess.CalledProcessError as e: + print(e) + failed = True else: print("Cloning %s..." % repo) subprocess.run(["git", "clone", "--bare", repo, repodir], check=True) + + if failed: + sys.exit(128)