CI: generate and use a Kas lock file

In the update-repos job, generate a Kas lock file to pin the SHAs of the
repositories being build during the build. This ensures that commits to
the repositories _during_ the build doesn't cause differing builds. All
of the kas calls use this lock file to ensure that their builds are
identical.

This lockfile is also added to the artifacts, so that it can be reused
afterwards to replicate the build: either as a known good base or to
replicate failures.

This lock file is only generated if it doesn't exist, so that
development branches can temporarily add a lockfile.yml if for example
master is too unstable to develop on.

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Jon Mason <jon.mason@arm.com>

.gitlab-ci.yml

@@ -58,8 +58,8 @@ stages:
     # Catch all for everything else
     - if: '$KERNEL != "linux-yocto-dev"'
   script:
-    - KASFILES=$(./ci/jobs-to-kas "$CI_JOB_NAME")
-    - kas dump --update --force-checkout --resolve-refs --resolve-env $KASFILES 
+    - KASFILES=$(./ci/jobs-to-kas "$CI_JOB_NAME"):lockfile.yml
+    - kas dump --update --force-checkout --resolve-refs --resolve-env $KASFILES
     - kas build $KASFILES
     - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log
   artifacts:
@@ -77,7 +77,19 @@ update-repos:
   extends: .setup
   stage: prep
   script:
-    - flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos
+    - |
+      flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos
+      # Only generate if doesn't already exist, to allow feature branches to drop one in.
+      if test -f lockfile.yml; then
+        echo Using existing lockfile.yml
+      else
+        # Be sure that this is the complete list of layers being fetched
+        kas dump --lock --update ci/qemuarm64.yml:ci/meta-openembedded.yml:ci/clang.yml:ci/meta-virtualization.yml | tee lockfile.yml
+      fi
+  artifacts:
+    name: "lockfile"
+    paths:
+      - lockfile.yml
 
 #
 # Build stage, the actual build jobs
@@ -233,14 +245,14 @@ toolchains:
 selftest:
   extends: .setup
   script:
-    - KASFILES=./ci/qemuarm64.yml:./ci/selftest.yml
+    - KASFILES=./ci/qemuarm64.yml:./ci/selftest.yml:lockfile.yml
     - kas shell --update --force-checkout $KASFILES -c 'oe-selftest --num-processes 1 --run-tests runfvp'
 
 # Validate layers are Yocto Project Compatible
 check-layers:
   extends: .setup
   script:
-    - kas shell --update --force-checkout ci/base.yml:ci/meta-openembedded.yml --command \
+    - kas shell --update --force-checkout ci/base.yml:ci/meta-openembedded.yml:lockfile.yml --command \
       "yocto-check-layer-wrapper $CI_PROJECT_DIR/$LAYER --dependency $CI_PROJECT_DIR/meta-* $KAS_WORK_DIR/meta-openembedded/meta-oe --no-auto-dependency"
   parallel:
     matrix:
@@ -254,7 +266,7 @@ pending-updates:
   script:
     - rm -fr update-report
     # This configuration has all of the layers we need enabled
-    - kas shell --update --force-checkout ci/qemuarm64.yml:ci/meta-openembedded.yml --command \
+    - kas shell --update --force-checkout ci/qemuarm64.yml:ci/meta-openembedded.yml:lockfile.yml --command \
       "$CI_PROJECT_DIR/scripts/machine-summary.py -t report -o $CI_PROJECT_DIR/update-report $($CI_PROJECT_DIR/ci/listmachines.py meta-arm meta-arm-bsp)"
   # Do this on x86 whilst the compilers are x86-only
   tags: