From e64c553fa03d0078bcd826e9cdc2bd1efce8f4e7 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Thu, 1 Oct 2020 17:52:40 +0100 Subject: [PATCH] Add experimental CI using Kas+GitLab Add kas scripts that generic to test builds, and a GitLab CI runner. Change-Id: I9026fd1af4155288c4adb523d00b1562ea8515e9 Signed-off-by: Ross Burton --- .gitlab-ci.yml | 129 +++++++++++++++++++++++++++++++++++ kas/base.yml | 45 ++++++++++++ kas/check-machine-coverage | 26 +++++++ kas/clang.yml | 10 +++ kas/corstone500.yml | 6 ++ kas/corstone700-fvp.yml | 10 +++ kas/corstone700-mps3.yml | 6 ++ kas/fvp-base-arm32.yml | 6 ++ kas/fvp-base.yml | 7 ++ kas/gem5-arm64.yml | 21 ++++++ kas/jobs-to-kas | 19 ++++++ kas/juno.yml | 6 ++ kas/meta-python.yml | 9 +++ kas/meta-zephyr.yml | 8 +++ kas/musca-b1.yml | 11 +++ kas/musca-s1.yml | 10 +++ kas/musl.yml | 6 ++ kas/n1sdp.yml | 6 ++ kas/qemuarm64-secureboot.yml | 11 +++ kas/sgi575.yml | 6 ++ kas/tc0.yml | 6 ++ 21 files changed, 364 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 kas/base.yml create mode 100755 kas/check-machine-coverage create mode 100644 kas/clang.yml create mode 100644 kas/corstone500.yml create mode 100644 kas/corstone700-fvp.yml create mode 100644 kas/corstone700-mps3.yml create mode 100644 kas/fvp-base-arm32.yml create mode 100644 kas/fvp-base.yml create mode 100644 kas/gem5-arm64.yml create mode 100755 kas/jobs-to-kas create mode 100644 kas/juno.yml create mode 100644 kas/meta-python.yml create mode 100644 kas/meta-zephyr.yml create mode 100644 kas/musca-b1.yml create mode 100644 kas/musca-s1.yml create mode 100644 kas/musl.yml create mode 100644 kas/n1sdp.yml create mode 100644 kas/qemuarm64-secureboot.yml create mode 100644 kas/sgi575.yml create mode 100644 kas/tc0.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..fc5b7b4c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,129 @@ +# Use our custom Crops-derived image +image: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/yocto-builder:master + + # First do a common bootstrap, and then build all the targets +stages: + - bootstrap + - build + +# Common job fragment to get a worker ready +.setup: + # Retry because the runner is flakey (see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2890) + retry: 2 + stage: build + variables: + KAS_WORK_DIR: $CI_PROJECT_DIR/work + SSTATE_DIR: $CI_BUILDS_DIR/persist/sstate + DL_DIR: $CI_BUILDS_DIR/persist/downloads + before_script: + - echo KAS_WORK_DIR = $KAS_WORK_DIR + - echo SSTATE_DIR = $SSTATE_DIR + - echo DL_DIR = $DL_DIR + - mkdir --verbose --parents $KAS_WORK_DIR $SSTATE_DIR $DL_DIR + +# Generalised fragment to do a Kas build +.build: + extends: .setup + script: + - KASFILES=$(kas/jobs-to-kas $CI_JOB_NAME) + - kas shell --update $KASFILES -c 'cat conf/*.conf' + - kas build $KASFILES + + +# +# First phase, bootstrap and machine coverage +# + +# Build a number of native tools first to ensure the other builders don't race +# over them +bootstrap: + extends: .build + stage: bootstrap + variables: + KAS_TARGET: binutils-cross-aarch64 gcc-cross-aarch64 python3-native opkg-native rpm-native + script: + - kas build kas/n1sdp.yml + +# What percentage of machines in the layer do we build +machine-coverage: + stage: bootstrap + script: + - ./kas/check-machine-coverage + coverage: '/Coverage: \d+/' + + +# +# Second phase, the actual build jobs +# + +corstone500: + extends: .build + +corstone700-fvp: + extends: .build + +corstone700-mps3: + extends: .build + +fvp-base: + extends: .build + +fvp-base-arm32: + extends: .build + +gem5-arm64: + extends: .build + +juno: + extends: .build + +juno/clang: + extends: .build + +musca-b1: + extends: .build + +musca-s1: + extends: .build + +n1sdp: + extends: .build + +qemuarm64-secureboot: + extends: .build + +qemuarm64-secureboot/clang: + extends: .build + +qemuarm64-secureboot/clang/musl: + extends: .build + +qemuarm64-secureboot/musl: + extends: .build + +sgi575: + extends: .build + +tc0: + extends: .build + + +# +# Utility tasks, not executed automatically +# + +# Report on disk usage +usage: + extends: .setup + stage: bootstrap + when: manual + script: + - du -h -s $DL_DIR $SSTATE_DIR + +# Wipe out old sstate +prune-sstate: + extends: .setup + stage: bootstrap + when: manual + script: + - find $SSTATE_DIR -type f -atime +30 -delete diff --git a/kas/base.yml b/kas/base.yml new file mode 100644 index 00000000..06ea9621 --- /dev/null +++ b/kas/base.yml @@ -0,0 +1,45 @@ +header: + version: 9 + +distro: poky + +defaults: + repos: + refspec: master + +repos: + meta-arm: + layers: + meta-arm: + meta-arm-bsp: + meta-arm-toolchain: + + poky: + url: https://git.yoctoproject.org/git/poky + layers: + meta: + meta-poky: + +env: + BB_LOGCONFIG: "" + PYTHONPATH: "" + +local_conf_header: + base: | + CONF_VERSION = "1" + PACKAGE_CLASSES = "package_ipk" + LICENSE_FLAGS_WHITELIST += "armcompiler" + PACKAGECONFIG_remove_pn-qemu-system-native = "gtk+ sdl" + BB_NUMBER_THREADS = "16" + PARALLEL_MAKE = "-j16" + INHERIT += "rm_work" + PACKAGECONFIG_append_pn-perf = " coresight" + ERROR_QA = "${WARN_QA}" + ptest: | + DISTRO_FEATURES_remove = "ptest" + +machine: unset + +target: + - core-image-base + - perf diff --git a/kas/check-machine-coverage b/kas/check-machine-coverage new file mode 100755 index 00000000..53d39832 --- /dev/null +++ b/kas/check-machine-coverage @@ -0,0 +1,26 @@ +#! /usr/bin/env python3 + +from pathlib import Path +import sys + +metaarm = Path.cwd() + +if metaarm.name != "meta-arm": + print("Not running inside meta-arm") + sys.exit(1) + +# All machine configurations +machines = metaarm.glob("meta-*/conf/machine/*.conf") +machines = set(p.stem for p in machines) + +# All kas files +kas = metaarm.glob("kas/*.yml") +kas = set(p.stem for p in kas) + +missing = machines - kas +print(f"The following machines are missing: {', '.join(sorted(missing))}.") + +covered = len(machines) - len(missing) +total = len(machines) +percent = int(covered / total * 100) +print(f"Coverage: {percent}%") diff --git a/kas/clang.yml b/kas/clang.yml new file mode 100644 index 00000000..f3524473 --- /dev/null +++ b/kas/clang.yml @@ -0,0 +1,10 @@ +header: + version: 9 + +repos: + meta-clang: + url: https://github.com/kraj/meta-clang + +local_conf_header: + clang: | + TOOLCHAIN = "clang" diff --git a/kas/corstone500.yml b/kas/corstone500.yml new file mode 100644 index 00000000..c348101d --- /dev/null +++ b/kas/corstone500.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: corstone500 diff --git a/kas/corstone700-fvp.yml b/kas/corstone700-fvp.yml new file mode 100644 index 00000000..e69fc4c0 --- /dev/null +++ b/kas/corstone700-fvp.yml @@ -0,0 +1,10 @@ +header: + version: 9 + includes: + - base.yml + +machine: corstone700-fvp + +local_conf_header: + image: | + CORE_IMAGE_EXTRA_INSTALL = "corstone700-test-app" diff --git a/kas/corstone700-mps3.yml b/kas/corstone700-mps3.yml new file mode 100644 index 00000000..3fa1a503 --- /dev/null +++ b/kas/corstone700-mps3.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - corstone700-fvp.yml + +machine: corstone700-mps3 diff --git a/kas/fvp-base-arm32.yml b/kas/fvp-base-arm32.yml new file mode 100644 index 00000000..bb83a725 --- /dev/null +++ b/kas/fvp-base-arm32.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: fvp-base-arm32 diff --git a/kas/fvp-base.yml b/kas/fvp-base.yml new file mode 100644 index 00000000..1c3675fa --- /dev/null +++ b/kas/fvp-base.yml @@ -0,0 +1,7 @@ +header: + version: 9 + includes: + - base.yml + +machine: fvp-base + diff --git a/kas/gem5-arm64.yml b/kas/gem5-arm64.yml new file mode 100644 index 00000000..fc1b5c91 --- /dev/null +++ b/kas/gem5-arm64.yml @@ -0,0 +1,21 @@ +header: + version: 9 + includes: + - base.yml + +repos: + meta-arm: + layers: + meta-gem5: + + meta-openembedded: + url: https://git.openembedded.org/meta-openembedded + layers: + meta-oe: + +machine: gem5-arm64 + +target: + - core-image-minimal + - perf + - gem5-aarch64-native diff --git a/kas/jobs-to-kas b/kas/jobs-to-kas new file mode 100755 index 00000000..d1f2f108 --- /dev/null +++ b/kas/jobs-to-kas @@ -0,0 +1,19 @@ +#! /bin/bash + +# Read a GitLab CI job name on $1 and transform it to a +# list of Kas yaml files + +set -e -u + +# Read Job namne from $1 and split on / +IFS=/ read -r -a PARTS<<<$1 + +# Prefix each part with kas/ +PARTS=("${PARTS[@]/#/kas/}") + +# Suffix each part with .yml +PARTS=("${PARTS[@]/%/.yml}") + +# Print colon-separated +IFS=":" +echo "${PARTS[*]}" diff --git a/kas/juno.yml b/kas/juno.yml new file mode 100644 index 00000000..0badddd4 --- /dev/null +++ b/kas/juno.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: juno diff --git a/kas/meta-python.yml b/kas/meta-python.yml new file mode 100644 index 00000000..0b4b958e --- /dev/null +++ b/kas/meta-python.yml @@ -0,0 +1,9 @@ +header: + version: 9 + +repos: + meta-openembedded: + url: https://git.openembedded.org/meta-openembedded + layers: + meta-oe: + meta-python: diff --git a/kas/meta-zephyr.yml b/kas/meta-zephyr.yml new file mode 100644 index 00000000..aa3f97c1 --- /dev/null +++ b/kas/meta-zephyr.yml @@ -0,0 +1,8 @@ +header: + version: 9 + includes: + - meta-python.yml + +repos: + meta-zephyr: + url: https://git.yoctoproject.org/git/meta-zephyr diff --git a/kas/musca-b1.yml b/kas/musca-b1.yml new file mode 100644 index 00000000..4f5f02d8 --- /dev/null +++ b/kas/musca-b1.yml @@ -0,0 +1,11 @@ +header: + version: 9 + includes: + - base.yml + - meta-zephyr.yml + +machine: musca-b1 + +target: + - trusted-firmware-m + - zephyr-philosophers diff --git a/kas/musca-s1.yml b/kas/musca-s1.yml new file mode 100644 index 00000000..30a12d53 --- /dev/null +++ b/kas/musca-s1.yml @@ -0,0 +1,10 @@ +header: + version: 9 + includes: + - base.yml + - meta-python.yml + +machine: musca-s1 + +target: + - trusted-firmware-m diff --git a/kas/musl.yml b/kas/musl.yml new file mode 100644 index 00000000..cd58df79 --- /dev/null +++ b/kas/musl.yml @@ -0,0 +1,6 @@ +header: + version: 9 + +local_conf_header: + libc: | + TCLIBC = "musl" diff --git a/kas/n1sdp.yml b/kas/n1sdp.yml new file mode 100644 index 00000000..9e0d26a2 --- /dev/null +++ b/kas/n1sdp.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: n1sdp diff --git a/kas/qemuarm64-secureboot.yml b/kas/qemuarm64-secureboot.yml new file mode 100644 index 00000000..8236f87c --- /dev/null +++ b/kas/qemuarm64-secureboot.yml @@ -0,0 +1,11 @@ +header: + version: 9 + includes: + - base.yml + +machine: qemuarm64-secureboot + +target: + - core-image-base + - perf + - optee-examples diff --git a/kas/sgi575.yml b/kas/sgi575.yml new file mode 100644 index 00000000..43ee4b59 --- /dev/null +++ b/kas/sgi575.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: sgi575 diff --git a/kas/tc0.yml b/kas/tc0.yml new file mode 100644 index 00000000..47f95af3 --- /dev/null +++ b/kas/tc0.yml @@ -0,0 +1,6 @@ +header: + version: 9 + includes: + - base.yml + +machine: tc0