CI: add ssh tests

Signed-off-by: Jon Mason <jon.mason@arm.com>

.gitlab-ci.yml

@@ -1,39 +1,50 @@
-# Use our custom Crops-derived image
+image: ghcr.io/siemens/kas/kas
-image: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/yocto-builder:master
 
   # First do a common bootstrap, and then build all the targets
 stages:
+  - prep
   - bootstrap
   - build
 
 # Common job fragment to get a worker ready
 .setup:
-  # Retry because the runner is flakey (see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2890)
-  retry: 2
   stage: build
   variables:
     KAS_WORK_DIR: $CI_PROJECT_DIR/work
+    KAS_REPO_REF_DIR: $CI_BUILDS_DIR/persist/repos
     SSTATE_DIR: $CI_BUILDS_DIR/persist/sstate
     DL_DIR: $CI_BUILDS_DIR/persist/downloads
-    BB_LOGCONFIG: $CI_PROJECT_DIR/kas/logging.yml
+    BB_LOGCONFIG: $CI_PROJECT_DIR/ci/logging.yml
+    IMAGE_DIR: $CI_PROJECT_DIR/work/build/tmp/deploy/images
   before_script:
     - echo KAS_WORK_DIR = $KAS_WORK_DIR
     - echo SSTATE_DIR = $SSTATE_DIR
     - echo DL_DIR = $DL_DIR
-    - mkdir --verbose --parents $KAS_WORK_DIR $SSTATE_DIR $DL_DIR
+    - rm -rf $KAS_WORK_DIR
+    - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR $DL_DIR
 
 # Generalised fragment to do a Kas build
 .build:
   extends: .setup
+  interruptible: true
   script:
-  - KASFILES=$(kas/jobs-to-kas $CI_JOB_NAME)
+    - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME)
-  - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
+    - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
-  - kas build $KASFILES
+    - kas build $KASFILES
-  - ./kas/check-warnings $KAS_WORK_DIR/build/warnings.log
+    - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log
+
+#
+# Prep stage, update repositories once
+#
+update-repos:
+  extends: .setup
+  stage: prep
+  script:
+  - flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos
 
 
 #
-# First phase, bootstrap and machine coverage
+# Bootstrap stage, bootstrap and machine coverage
 #
 
 # Build a number of native tools first to ensure the other builders don't race
@@ -46,12 +57,12 @@ n1sdp/bootstrap:
 machine-coverage:
   stage: bootstrap
   script:
-  - ./kas/check-machine-coverage
+  - ./ci/check-machine-coverage
   coverage: '/Coverage: \d+/'
 
 
 #
-# Second phase, the actual build jobs
+# Build stage, the actual build jobs
 #
 
 a5ds:
@@ -78,6 +89,12 @@ n1sdp:
 n1sdp/armgcc:
   extends: .build
 
+qemuarm/testimage:
+  extends: .build
+
+qemuarmv5/testimage:
+  extends: .build
+
 tc0:
   extends: .build
 
@@ -86,18 +103,42 @@ tc0:
 # Utility tasks, not executed automatically
 #
 
-# Report on disk usage
+delete-dl-dir:
-usage:
   extends: .setup
-  stage: bootstrap
+  stage: prep
   when: manual
   script:
-  - du -h -s $DL_DIR $SSTATE_DIR
+  - rm -rf $DL_DIR/*
+
+delete-repo-dir:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - rm -rf $KAS_REPO_REF_DIR/*
+
+# Delete all sstate
+delete-sstate:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - rm -rf $SSTATE_DIR/*
 
 # Wipe out old sstate
 prune-sstate:
   extends: .setup
-  stage: bootstrap
+  stage: prep
   when: manual
   script:
+  - du -h -s $SSTATE_DIR
   - find $SSTATE_DIR -type f -atime +30 -delete
+  - du -h -s $SSTATE_DIR
+
+# Report on disk usage
+usage:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - du -h -s $DL_DIR $SSTATE_DIR $KAS_REPO_REF_DIR

ci/base.yml

@@ -34,10 +34,10 @@ local_conf_header:
     PACKAGE_CLASSES = "package_ipk"
     LICENSE_FLAGS_WHITELIST += "armcompiler"
     PACKAGECONFIG_remove_pn-qemu-system-native = "gtk+ sdl"
+    EXTRA_IMAGE_FEATURES_append = " debug-tweaks"
     BB_NUMBER_THREADS = "16"
     PARALLEL_MAKE = "-j16"
     INHERIT += "rm_work"
-    ERROR_QA = "${WARN_QA}"
   ptest: |
     DISTRO_FEATURES_remove = "ptest"
 

ci/check-machine-coverage

@@ -13,11 +13,11 @@ if metaarm.name != "meta-arm":
 machines = metaarm.glob("meta-*/conf/machine/*.conf")
 machines = set(p.stem for p in machines)
 
-# All kas files
+# All ci files
-kas = metaarm.glob("kas/*.yml")
+ci = metaarm.glob("ci/*.yml")
-kas = set(p.stem for p in kas)
+ci = set(p.stem for p in ci)
 
-missing = machines - kas
+missing = machines - ci
 print(f"The following machines are missing: {', '.join(sorted(missing))}.")
 
 covered = len(machines) - len(missing)

ci/jobs-to-kas

@@ -8,8 +8,8 @@ set -e -u
 # Read Job namne from $1 and split on /
 IFS=/ read -r -a PARTS<<<$1
 
-# Prefix each part with kas/
+# Prefix each part with ci/
-PARTS=("${PARTS[@]/#/kas/}")
+PARTS=("${PARTS[@]/#/ci/}")
 
 # Suffix each part with .yml
 PARTS=("${PARTS[@]/%/.yml}")

ci/qemuarm.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: qemuarm

ci/qemuarmv5.yml

@@ -0,0 +1,11 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: qemuarmv5
+
+local_conf_header:
+  bugs: |
+    # Remove parselogs until errors can be resolved
+    TEST_SUITES_remove = "parselogs"

ci/testimage.yml

@@ -0,0 +1,17 @@
+header:
+  version: 9
+
+local_conf_header:
+  testimage: |
+    IMAGE_CLASSES += "testimage"
+    TESTIMAGE_AUTO = "1"
+  slirp: |
+    TEST_RUNQEMUPARAMS = "slirp"
+    TEST_SERVER_IP = "127.0.0.1"
+    QEMU_USE_SLIRP = "1"
+  packages: |
+    IMAGE_FEATURES_append = " ssh-server-dropbear"
+
+# Multiple targets are available, put it down to just one
+target:
+  - core-image-base

ci/update-repos

@@ -0,0 +1,41 @@
+#! /usr/bin/env python3
+
+# Update clones of the repositories we need in KAS_REPO_REF_DIR to speed up fetches
+
+import sys
+import os
+import subprocess
+import pathlib
+
+def repo_shortname(url):
+    # Taken from Kas (Repo.__getattr__) to ensure the logic is right
+    from urllib.parse import urlparse
+    url = urlparse(url)
+    return ('{url.netloc}{url.path}'
+            .format(url=url)
+            .replace('@', '.')
+            .replace(':', '.')
+            .replace('/', '.')
+            .replace('*', '.'))
+
+repositories = (
+    "https://git.yoctoproject.org/git/poky",
+    "https://git.openembedded.org/meta-openembedded",
+    "https://git.yoctoproject.org/git/meta-virtualization",
+    "https://git.yoctoproject.org/git/meta-zephyr",
+    "https://github.com/kraj/meta-clang",
+)
+
+if __name__ == "__main__":
+    if "KAS_REPO_REF_DIR" not in os.environ:
+        print("KAS_REPO_REF_DIR needs to be set")
+        sys.exit(1)
+
+    base_repodir = pathlib.Path(os.environ["KAS_REPO_REF_DIR"])
+
+    for repo in repositories:
+        repodir = base_repodir / repo_shortname(repo)
+        if repodir.exists():
+            subprocess.run(["git", "-C", repodir, "fetch"], check=True)
+        else:
+            subprocess.run(["git", "clone", "--bare", repo, repodir], check=True)

meta-arm-toolchain/conf/distro/include/external-arm-toolchain-versions.inc

@@ -139,19 +139,5 @@ python external_arm_toolchain_version_handler () {
     d.setVar('EAT_VER_KERNEL', eat_get_kernel_version(ld))
     d.setVar('EAT_VER_GDB', eat_get_gdb_version(ld))
     d.setVar('EAT_VER_BFD', eat_get_bfd_version(ld))
-
-    lc_libc = "LGPL-2.1"
-    lc_gcc = "GPL-3.0"
-    lc_rle = "GPL-3.0-with-GCC-exception"
-    lc_gdb = "GPL-3.0"
-    lc_bfd = "GPL-3.0"
-
-    d.setVar('EAT_LIBC_LICENSE', lc_libc)
-    d.setVar('EAT_GCC_LICENSE', lc_gcc)
-    d.setVar('EAT_RLE_LICENSE', lc_rle)
-    d.setVar('EAT_GDB_LICENSE', lc_gdb)
-    d.setVar('EAT_BFD_LICENSE', lc_bfd)
-
-    d.setVar('EAT_LICENSE', "%s & %s & %s & %s & %s" % (lc_libc, lc_gcc, lc_rle, lc_gdb, lc_bfd))
 }
 addhandler external_arm_toolchain_version_handler

meta-arm-toolchain/recipes-devtools/external-arm-toolchain/external-arm-sdk-toolchain.bb

@@ -1,5 +1,7 @@
 inherit cross-canadian
 
+require license.inc
+
 PN = "external-arm-sdk-toolchain-${TARGET_ARCH}"
 BPN = "external-arm-sdk-toolchain"
 PV = "${EAT_VER_MAIN}"
@@ -10,9 +12,6 @@ INHIBIT_SYSROOT_STRIP = "1"
 INHIBIT_DEFAULT_DEPS = "1"
 EXCLUDE_FROM_SHLIBS = "1"
 
-EAT_LICENSE ??= ""
-
-LICENSE = "${EAT_LICENSE}"
 LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"
 
 # Skip packaging QA checks for prebuilt binaries

meta-arm-toolchain/recipes-devtools/external-arm-toolchain/license.inc

@@ -1,7 +1,10 @@
 LICENSE = "GPL-3.0-with-GCC-exception & GPLv3 & GPLv2 & LGPLv3 & LGPLv2.1 & LGPLv2"
 
+EAT_BFD_LICENSE = "GPL-3.0"
 EAT_GCC_LICENSE = "GPL-3.0-with-GCC-exception & GPLv3"
+EAT_GDB_LICENSE = "GPL-3.0"
 EAT_LIBC_LICENSE = "GPLv2 & LGPLv2.1"
+EAT_RLE_LICENSE = "GPL-3.0-with-GCC-exception"
 
 LICENSE_${PN} = "${EAT_LIBC_LICENSE}"
 LICENSE_${PN}-dev = "${EAT_LIBC_LICENSE}"