From 26f92c011e2347a64bfee7aa8ae01e557f9b4d34 Mon Sep 17 00:00:00 2001 From: Jon Mason Date: Wed, 29 Sep 2021 10:44:54 -0400 Subject: [PATCH] CI: use matrix for more defined way of doing things and cleanup Signed-off-by: Jon Mason --- .gitlab-ci.yml | 111 +++++++++++++++++++++-------------------- ci/bootstrap.yml | 9 ---- ci/clang-bootstrap.yml | 10 ---- ci/jobs-to-kas | 32 +++++++----- 4 files changed, 77 insertions(+), 85 deletions(-) delete mode 100644 ci/bootstrap.yml delete mode 100644 ci/clang-bootstrap.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50f5ec13..e2d000af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,7 @@ image: ghcr.io/siemens/kas/kas -# First do a common bootstrap, and then build all the targets stages: - prep - - bootstrap - build # Common job fragment to get a worker ready @@ -33,7 +31,7 @@ stages: .build: extends: .setup script: - - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME) + - KASFILES=$(./ci/jobs-to-kas "$CI_JOB_NAME") - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf' - kas build $KASFILES - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log @@ -42,7 +40,7 @@ stages: .build_and_test: extends: .setup script: - - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME) + - KASFILES=$(./ci/jobs-to-kas "$CI_JOB_NAME") - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf' - kas build $KASFILES - kas build $KASFILES -c testimage @@ -58,23 +56,10 @@ update-repos: script: - flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos -# -# Bootstrap stage, bootstrap and machine coverage -# - -# Build a number of native tools first to ensure the other builders don't race -# over them -n1sdp/bootstrap: - extends: .build - stage: bootstrap - -n1sdp/clang/clang-bootstrap: - extends: .build - stage: bootstrap # What percentage of machines in the layer do we build machine-coverage: - stage: bootstrap + stage: build interruptible: true script: - ./ci/check-machine-coverage @@ -83,6 +68,11 @@ machine-coverage: # # Build stage, the actual build jobs # +# Available options for building are +# TOOLCHAINS: [gcc, clang, armgcc, external-gccarm] +# TCLIBC: [glibc, musl] +# VIRT: [none, xen] +# TESTING: testimage # Validate layers are Yocto Project Compatible check-layers: @@ -112,16 +102,19 @@ corstone700-mps3: corstone1000-fvp: extends: .build -fvp-base/testimage: +fvp-base: extends: .build + parallel: + matrix: + - TESTING: testimage tags: - - x86_64 + - x86_64 fvp-base-arm32: extends: .build - -fvp-base-arm32/external-gccarm: - extends: .build + parallel: + matrix: + - TOOLCHAINS: [gcc, external-gccarm] fvp-baser-aemv8r64: extends: .build @@ -131,9 +124,9 @@ fvps: gem5-arm64: extends: .build - -gem5-arm64/xen: - extends: .build + parallel: + matrix: + - VIRT: [none, xen] gem5-atp-arm64: extends: .build @@ -143,12 +136,15 @@ generic-arm64: juno: extends: .build + parallel: + matrix: + - TOOLCHAINS: [gcc, clang] -juno/clang: - extends: .build - -microbit-v1/testimage-zephyr: +microbit-v1: extends: .build_and_test + parallel: + matrix: + - TESTING: testimage-zephyr musca-b1: extends: .build @@ -158,48 +154,55 @@ musca-s1: n1sdp: extends: .build - -n1sdp/armgcc: - extends: .build + parallel: + matrix: + - TOOLCHAINS: [gcc, armgcc] qemu-cortex-a53: extends: .build -qemu-cortex-m3/testimage-zephyr: +qemu-cortex-m3: extends: .build_and_test + parallel: + matrix: + - TESTING: testimage-zephyr qemu-cortex-r5: extends: .build -qemu-generic-arm64/testimage: +qemu-generic-arm64: extends: .build + parallel: + matrix: + - TESTING: testimage -qemuarm64-secureboot/testimage: +qemuarm64-secureboot: extends: .build + parallel: + matrix: + - TOOLCHAINS: [gcc, clang] + TCLIBC: [glibc, musl] + TESTING: testimage -qemuarm64-secureboot/clang/testimage: +qemuarm64: extends: .build + parallel: + matrix: + - VIRT: xen -qemuarm64-secureboot/clang/musl/testimage: +qemuarm: extends: .build + parallel: + matrix: + - TOOLCHAINS: [gcc, clang] + TESTING: testimage + - VIRT: xen -qemuarm64-secureboot/musl/testimage: - extends: .build - -qemuarm64/xen: - extends: .build - -qemuarm/clang/testimage: - extends: .build - -qemuarm/testimage: - extends: .build - -qemuarm/xen: - extends: .build - -qemuarmv5/testimage: +qemuarmv5: extends: .build + parallel: + matrix: + - TESTING: testimage sgi575: extends: .build diff --git a/ci/bootstrap.yml b/ci/bootstrap.yml deleted file mode 100644 index 0e05e46b..00000000 --- a/ci/bootstrap.yml +++ /dev/null @@ -1,9 +0,0 @@ -header: - version: 9 - -target: - - binutils-cross-aarch64 - - gcc-cross-aarch64 - - python3-native - - opkg-native - - rpm-native diff --git a/ci/clang-bootstrap.yml b/ci/clang-bootstrap.yml deleted file mode 100644 index f5497ffc..00000000 --- a/ci/clang-bootstrap.yml +++ /dev/null @@ -1,10 +0,0 @@ -header: - version: 9 - -target: - - binutils-cross-aarch64 - - clang-cross-aarch64 - - clang-native - - python3-native - - opkg-native - - rpm-native diff --git a/ci/jobs-to-kas b/ci/jobs-to-kas index 70579703..f442a22d 100755 --- a/ci/jobs-to-kas +++ b/ci/jobs-to-kas @@ -1,19 +1,27 @@ #! /bin/bash -# Read a GitLab CI job name on $1 and transform it to a -# list of Kas yaml files +# This script is expecting an input of machine name, optionally followed by a +# colon and a list of one or more parameters separated by commas between +# brackets. For example, the following are acceptable: +# corstone500 +# fvp-base: [testimage] +# qemuarm64-secureboot: [clang, glibc, testimage] +# +# Turn this list into a series of yml files separated by colons to pass to kas set -e -u -# Read Job namne from $1 and split on / -IFS=/ read -r -a PARTS<<<$1 +FILES="ci/$(echo $1 | cut -d ':' -f 1).yml" -# Prefix each part with ci/ -PARTS=("${PARTS[@]/#/ci/}") +for i in $(echo $1 | cut -s -d ':' -f 2 | sed 's/[][,]//g'); do + # Given that there are no yml files for gcc or glibc, as those are the + # defaults, we can simply ignore those parameters. They are necessary + # to pass in so that matrix can correctly setup all of the permutations + # of each individual run. + if [[ $i == 'none' || $i == 'gcc' || $i == 'glibc' ]]; then + continue + fi + FILES+=":ci/$i.yml" +done -# Suffix each part with .yml -PARTS=("${PARTS[@]/%/.yml}") - -# Print colon-separated -IFS=":" -echo "${PARTS[*]}" +echo $FILES