Add experimental CI using Kas+GitLab

Add kas scripts that generic to test builds, and a GitLab CI runner.

Change-Id: I9026fd1af4155288c4adb523d00b1562ea8515e9
Signed-off-by: Ross Burton <ross.burton@arm.com>

.gitlab-ci.yml

@@ -0,0 +1,129 @@
+# Use our custom Crops-derived image
+image: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/yocto-builder:master
+
+  # First do a common bootstrap, and then build all the targets
+stages:
+  - bootstrap
+  - build
+
+# Common job fragment to get a worker ready
+.setup:
+  # Retry because the runner is flakey (see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2890)
+  retry: 2
+  stage: build
+  variables:
+    KAS_WORK_DIR: $CI_PROJECT_DIR/work
+    SSTATE_DIR: $CI_BUILDS_DIR/persist/sstate
+    DL_DIR: $CI_BUILDS_DIR/persist/downloads
+  before_script:
+    - echo KAS_WORK_DIR = $KAS_WORK_DIR
+    - echo SSTATE_DIR = $SSTATE_DIR
+    - echo DL_DIR = $DL_DIR
+    - mkdir --verbose --parents $KAS_WORK_DIR $SSTATE_DIR $DL_DIR
+
+# Generalised fragment to do a Kas build
+.build:
+  extends: .setup
+  script:
+  - KASFILES=$(kas/jobs-to-kas $CI_JOB_NAME)
+  - kas shell --update $KASFILES -c 'cat conf/*.conf'
+  - kas build $KASFILES
+
+
+#
+# First phase, bootstrap and machine coverage
+#
+
+# Build a number of native tools first to ensure the other builders don't race
+# over them
+bootstrap:
+  extends: .build
+  stage: bootstrap
+  variables:
+    KAS_TARGET: binutils-cross-aarch64 gcc-cross-aarch64 python3-native opkg-native rpm-native
+  script:
+  - kas build kas/n1sdp.yml
+
+# What percentage of machines in the layer do we build
+machine-coverage:
+  stage: bootstrap
+  script:
+  - ./kas/check-machine-coverage
+  coverage: '/Coverage: \d+/'
+
+
+#
+# Second phase, the actual build jobs
+#
+
+corstone500:
+  extends: .build
+
+corstone700-fvp:
+  extends: .build
+
+corstone700-mps3:
+  extends: .build
+
+fvp-base:
+  extends: .build
+
+fvp-base-arm32:
+  extends: .build
+
+gem5-arm64:
+  extends: .build
+
+juno:
+  extends: .build
+
+juno/clang:
+  extends: .build
+
+musca-b1:
+  extends: .build
+
+musca-s1:
+  extends: .build
+
+n1sdp:
+  extends: .build
+
+qemuarm64-secureboot:
+  extends: .build
+
+qemuarm64-secureboot/clang:
+  extends: .build
+
+qemuarm64-secureboot/clang/musl:
+  extends: .build
+
+qemuarm64-secureboot/musl:
+  extends: .build
+
+sgi575:
+  extends: .build
+
+tc0:
+  extends: .build
+
+
+#
+# Utility tasks, not executed automatically
+#
+
+# Report on disk usage
+usage:
+  extends: .setup
+  stage: bootstrap
+  when: manual
+  script:
+  - du -h -s $DL_DIR $SSTATE_DIR
+
+# Wipe out old sstate
+prune-sstate:
+  extends: .setup
+  stage: bootstrap
+  when: manual
+  script:
+  - find $SSTATE_DIR -type f -atime +30 -delete

kas/base.yml

@@ -0,0 +1,45 @@
+header:
+  version: 9
+
+distro: poky
+
+defaults:
+  repos:
+    refspec: master
+
+repos:
+  meta-arm:
+    layers:
+      meta-arm:
+      meta-arm-bsp:
+      meta-arm-toolchain:
+
+  poky:
+    url: https://git.yoctoproject.org/git/poky
+    layers:
+      meta:
+      meta-poky:
+
+env:
+  BB_LOGCONFIG: ""
+  PYTHONPATH: ""
+
+local_conf_header:
+  base: |
+    CONF_VERSION = "1"
+    PACKAGE_CLASSES = "package_ipk"
+    LICENSE_FLAGS_WHITELIST += "armcompiler"
+    PACKAGECONFIG_remove_pn-qemu-system-native = "gtk+ sdl"
+    BB_NUMBER_THREADS = "16"
+    PARALLEL_MAKE = "-j16"
+    INHERIT += "rm_work"
+    PACKAGECONFIG_append_pn-perf = " coresight"
+    ERROR_QA = "${WARN_QA}"
+  ptest: |
+    DISTRO_FEATURES_remove = "ptest"
+
+machine: unset
+
+target:
+  - core-image-base
+  - perf

kas/check-machine-coverage

@@ -0,0 +1,26 @@
+#! /usr/bin/env python3
+
+from pathlib import Path
+import sys
+
+metaarm = Path.cwd()
+
+if metaarm.name != "meta-arm":
+    print("Not running inside meta-arm")
+    sys.exit(1)
+
+# All machine configurations
+machines = metaarm.glob("meta-*/conf/machine/*.conf")
+machines = set(p.stem for p in machines)
+
+# All kas files
+kas = metaarm.glob("kas/*.yml")
+kas = set(p.stem for p in kas)
+
+missing = machines - kas
+print(f"The following machines are missing: {', '.join(sorted(missing))}.")
+
+covered = len(machines) - len(missing)
+total = len(machines)
+percent = int(covered / total * 100)
+print(f"Coverage: {percent}%")

kas/clang.yml

@@ -0,0 +1,10 @@
+header:
+  version: 9
+
+repos:
+  meta-clang:
+    url: https://github.com/kraj/meta-clang
+
+local_conf_header:
+  clang: |
+    TOOLCHAIN = "clang"

kas/corstone500.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: corstone500

kas/corstone700-fvp.yml

@@ -0,0 +1,10 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: corstone700-fvp
+
+local_conf_header:
+  image: |
+    CORE_IMAGE_EXTRA_INSTALL = "corstone700-test-app"

kas/corstone700-mps3.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - corstone700-fvp.yml
+
+machine: corstone700-mps3

kas/fvp-base-arm32.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: fvp-base-arm32

kas/fvp-base.yml

@@ -0,0 +1,7 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: fvp-base
+

kas/gem5-arm64.yml

@@ -0,0 +1,21 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+repos:
+  meta-arm:
+    layers:
+      meta-gem5:
+
+  meta-openembedded:
+    url: https://git.openembedded.org/meta-openembedded
+    layers:
+      meta-oe:
+
+machine: gem5-arm64
+
+target:
+  - core-image-minimal
+  - perf
+  - gem5-aarch64-native

kas/jobs-to-kas

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+# Read a GitLab CI job name on $1 and transform it to a
+# list of Kas yaml files
+
+set -e -u
+
+# Read Job namne from $1 and split on /
+IFS=/ read -r -a PARTS<<<$1
+
+# Prefix each part with kas/
+PARTS=("${PARTS[@]/#/kas/}")
+
+# Suffix each part with .yml
+PARTS=("${PARTS[@]/%/.yml}")
+
+# Print colon-separated
+IFS=":"
+echo "${PARTS[*]}"

kas/juno.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: juno

kas/meta-python.yml

@@ -0,0 +1,9 @@
+header:
+  version: 9
+
+repos:
+  meta-openembedded:
+    url: https://git.openembedded.org/meta-openembedded
+    layers:
+      meta-oe:
+      meta-python:

kas/meta-zephyr.yml

@@ -0,0 +1,8 @@
+header:
+  version: 9
+  includes:
+    - meta-python.yml
+
+repos:
+  meta-zephyr:
+    url: https://git.yoctoproject.org/git/meta-zephyr

kas/musca-b1.yml

@@ -0,0 +1,11 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+    - meta-zephyr.yml
+
+machine: musca-b1
+
+target:
+  - trusted-firmware-m
+  - zephyr-philosophers

kas/musca-s1.yml

@@ -0,0 +1,10 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+    - meta-python.yml
+
+machine: musca-s1
+
+target:
+  - trusted-firmware-m

kas/musl.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+
+local_conf_header:
+  libc: |
+    TCLIBC = "musl"

kas/n1sdp.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: n1sdp

kas/qemuarm64-secureboot.yml

@@ -0,0 +1,11 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: qemuarm64-secureboot
+
+target:
+  - core-image-base
+  - perf
+  - optee-examples

kas/sgi575.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: sgi575

kas/tc0.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: tc0