Add experimental CI using Kas+GitLab

Add kas scripts that generic to test builds, and a GitLab CI runner.

This is mainly a cut-down copy of what is in the master/gatesgarth
branches with the following changes:

- Just the BSPs that are in the Dunfell release, obviously
- No clang testing. There are patches in master that can be backported
  if required.
- Added testing of the armgcc-9.2 compiler. This is currently broken for
  some configurations in master and the testing will be forward-ported
  when it passes.

Change-Id: I9c2a4f66318b3ccc066d423f3533202ee33f0c9d
Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Jon Mason <jon.mason@arm.com>

.gitlab-ci.yml

@@ -0,0 +1,105 @@
+# Use our custom Crops-derived image
+image: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/yocto-builder:master
+
+  # First do a common bootstrap, and then build all the targets
+stages:
+  - bootstrap
+  - build
+
+# Common job fragment to get a worker ready
+.setup:
+  # Retry because the runner is flakey (see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2890)
+  retry: 2
+  stage: build
+  variables:
+    KAS_WORK_DIR: $CI_PROJECT_DIR/work
+    SSTATE_DIR: $CI_BUILDS_DIR/persist/sstate
+    DL_DIR: $CI_BUILDS_DIR/persist/downloads
+  before_script:
+    - echo KAS_WORK_DIR = $KAS_WORK_DIR
+    - echo SSTATE_DIR = $SSTATE_DIR
+    - echo DL_DIR = $DL_DIR
+    - mkdir --verbose --parents $KAS_WORK_DIR $SSTATE_DIR $DL_DIR
+
+# Generalised fragment to do a Kas build
+.build:
+  extends: .setup
+  script:
+  - KASFILES=$(kas/jobs-to-kas $CI_JOB_NAME)
+  - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
+  - kas build $KASFILES
+
+
+#
+# First phase, bootstrap and machine coverage
+#
+
+# Build a number of native tools first to ensure the other builders don't race
+# over them
+bootstrap:
+  extends: .build
+  stage: bootstrap
+  variables:
+    KAS_TARGET: binutils-cross-aarch64 gcc-cross-aarch64 python3-native opkg-native rpm-native
+  script:
+  - kas build kas/n1sdp.yml
+
+# What percentage of machines in the layer do we build
+machine-coverage:
+  stage: bootstrap
+  script:
+  - ./kas/check-machine-coverage
+  coverage: '/Coverage: \d+/'
+
+
+#
+# Second phase, the actual build jobs
+#
+
+a5ds:
+  extends: .build
+
+foundation-armv8:
+  extends: .build
+
+fvp-base:
+  extends: .build
+
+gem5-arm64:
+  extends: .build
+
+juno:
+  extends: .build
+
+juno/musl:
+  extends: .build
+
+n1sdp:
+  extends: .build
+
+n1sdp/armgcc:
+  extends: .build
+
+tc0:
+  extends: .build
+
+
+#
+# Utility tasks, not executed automatically
+#
+
+# Report on disk usage
+usage:
+  extends: .setup
+  stage: bootstrap
+  when: manual
+  script:
+  - du -h -s $DL_DIR $SSTATE_DIR
+
+# Wipe out old sstate
+prune-sstate:
+  extends: .setup
+  stage: bootstrap
+  when: manual
+  script:
+  - find $SSTATE_DIR -type f -atime +30 -delete

kas/a5ds.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: a5ds

kas/armgcc.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+
+local_conf_header:
+  libc: |
+    GCCVERSION = "arm-9.2"

kas/base.yml

@@ -0,0 +1,48 @@
+header:
+  version: 9
+
+distro: poky
+
+defaults:
+  repos:
+    refspec: dunfell
+
+repos:
+  meta-arm:
+    layers:
+      meta-arm:
+      meta-arm-bsp:
+      meta-arm-toolchain:
+
+  poky:
+    url: https://git.yoctoproject.org/git/poky
+    layers:
+      meta:
+      meta-poky:
+
+  meta-openembedded:
+    url: https://git.openembedded.org/meta-openembedded
+    layers:
+      meta-oe:
+
+  meta-kernel:
+    url: https://gitlab.com/openembedded/community/meta-kernel.git
+
+local_conf_header:
+  base: |
+    CONF_VERSION = "1"
+    PACKAGE_CLASSES = "package_ipk"
+    LICENSE_FLAGS_WHITELIST += "armcompiler"
+    PACKAGECONFIG_remove_pn-qemu-system-native = "gtk+ sdl"
+    BB_NUMBER_THREADS = "16"
+    PARALLEL_MAKE = "-j16"
+    INHERIT += "rm_work"
+    ERROR_QA = "${WARN_QA}"
+  ptest: |
+    DISTRO_FEATURES_remove = "ptest"
+
+machine: unset
+
+target:
+  - core-image-base
+#  - perf

kas/check-machine-coverage

@@ -0,0 +1,26 @@
+#! /usr/bin/env python3
+
+from pathlib import Path
+import sys
+
+metaarm = Path.cwd()
+
+if metaarm.name != "meta-arm":
+    print("Not running inside meta-arm")
+    sys.exit(1)
+
+# All machine configurations
+machines = metaarm.glob("meta-*/conf/machine/*.conf")
+machines = set(p.stem for p in machines)
+
+# All kas files
+kas = metaarm.glob("kas/*.yml")
+kas = set(p.stem for p in kas)
+
+missing = machines - kas
+print(f"The following machines are missing: {', '.join(sorted(missing))}.")
+
+covered = len(machines) - len(missing)
+total = len(machines)
+percent = int(covered / total * 100)
+print(f"Coverage: {percent}%")

kas/clang.yml

@@ -0,0 +1,10 @@
+header:
+  version: 9
+
+repos:
+  meta-clang:
+    url: https://github.com/kraj/meta-clang
+
+local_conf_header:
+  clang: |
+    TOOLCHAIN = "clang"

kas/foundation-armv8.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: foundation-armv8

kas/fvp-base.yml

@@ -0,0 +1,7 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: fvp-base
+

kas/gem5-arm64.yml

@@ -0,0 +1,10 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: gem5-arm64
+
+target:
+  - core-image-minimal
+  - gem5-aarch64-native

kas/jobs-to-kas

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+# Read a GitLab CI job name on $1 and transform it to a
+# list of Kas yaml files
+
+set -e -u
+
+# Read Job namne from $1 and split on /
+IFS=/ read -r -a PARTS<<<$1
+
+# Prefix each part with kas/
+PARTS=("${PARTS[@]/#/kas/}")
+
+# Suffix each part with .yml
+PARTS=("${PARTS[@]/%/.yml}")
+
+# Print colon-separated
+IFS=":"
+echo "${PARTS[*]}"

kas/juno.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: juno

kas/musl.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+
+local_conf_header:
+  libc: |
+    TCLIBC = "musl"

kas/n1sdp.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: n1sdp

kas/tc0.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: tc0