external-arm-toolchain: Misc. fixup for GCC 10.2 support

Arm GCC 10.2 prebuilt toolchain comes with ldd and tzselect scripts
which uses "/usr/bin/bash", so replace this with "/bin/sh" as the
default shell.

Signed-off-by: Sumit Garg <sumit.garg@linaro.org>
Signed-off-by: Jon Mason <jon.mason@arm.com>

.gitlab-ci.yml

@@ -1,37 +1,57 @@
-# Use our custom Crops-derived image
-image: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/yocto-builder:master
+image: ghcr.io/siemens/kas/kas
 
   # First do a common bootstrap, and then build all the targets
 stages:
+  - prep
   - bootstrap
   - build
 
 # Common job fragment to get a worker ready
 .setup:
   stage: build
+  interruptible: true
   variables:
     KAS_WORK_DIR: $CI_PROJECT_DIR/work
+    KAS_REPO_REF_DIR: $CI_BUILDS_DIR/persist/repos
     SSTATE_DIR: $CI_BUILDS_DIR/persist/sstate
     DL_DIR: $CI_BUILDS_DIR/persist/downloads
-    BB_LOGCONFIG: $CI_PROJECT_DIR/kas/logging.yml
+    BB_LOGCONFIG: $CI_PROJECT_DIR/ci/logging.yml
+    TOOLCHAIN_DIR: $CI_BUILDS_DIR/persist/toolchains
+    IMAGE_DIR: $CI_PROJECT_DIR/work/build/tmp/deploy/images
   before_script:
     - echo KAS_WORK_DIR = $KAS_WORK_DIR
     - echo SSTATE_DIR = $SSTATE_DIR
     - echo DL_DIR = $DL_DIR
-    - mkdir --verbose --parents $KAS_WORK_DIR $SSTATE_DIR $DL_DIR
+    - rm -rf $KAS_WORK_DIR
+    - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR $DL_DIR $TOOLCHAIN_DIR
 
 # Generalised fragment to do a Kas build
 .build:
   extends: .setup
   script:
-  - KASFILES=$(kas/jobs-to-kas $CI_JOB_NAME)
-  - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
-  - kas build $KASFILES
-  - ./kas/check-warnings $KAS_WORK_DIR/build/warnings.log
+    - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME)
+    - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
+    - kas build $KASFILES
+    - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log
+
+#
+# Prep stage, update repositories once
+#
+update-repos:
+  extends: .setup
+  stage: prep
+  script:
+  - flock --verbose --timeout 60 $KAS_REPO_REF_DIR ./ci/update-repos
+
+get-binary-toolchains:
+  extends: .setup
+  stage: prep
+  script:
+    - ./ci/get-binary-toolchains $DL_DIR $TOOLCHAIN_DIR
 
 
 #
-# First phase, bootstrap and machine coverage
+# Bootstrap stage, bootstrap and machine coverage
 #
 
 # Build a number of native tools first to ensure the other builders don't race
@@ -43,13 +63,14 @@ n1sdp/bootstrap:
 # What percentage of machines in the layer do we build
 machine-coverage:
   stage: bootstrap
+  interruptible: true
   script:
-  - ./kas/check-machine-coverage
+  - ./ci/check-machine-coverage
   coverage: '/Coverage: \d+/'
 
 
 #
-# Second phase, the actual build jobs
+# Build stage, the actual build jobs
 #
 
 a5ds:
@@ -70,6 +91,9 @@ fvp-base:
 fvp-base-arm32:
   extends: .build
 
+fvp-base-arm32/external-gccarm:
+  extends: .build
+
 gem5-arm64:
   extends: .build
 
@@ -88,16 +112,29 @@ musca-s1:
 n1sdp:
   extends: .build
 
-qemuarm64-secureboot:
+# This job currently fails when building gcc-runtime:
+# cc1: error: switch '-mcpu=armv8.2-a' conflicts with '-march=armv8-a' switch [-Werror]
+# Mark as manual so it doesn't get executed automatically
+n1sdp/armgcc:
+  extends: .build
+  when: manual
+
+qemuarm/testimage:
   extends: .build
 
-qemuarm64-secureboot/clang:
+qemuarm64-secureboot/testimage:
   extends: .build
 
-qemuarm64-secureboot/clang/musl:
+qemuarm64-secureboot/clang/testimage:
   extends: .build
 
-qemuarm64-secureboot/musl:
+qemuarm64-secureboot/clang/musl/testimage:
+  extends: .build
+
+qemuarm64-secureboot/musl/testimage:
+  extends: .build
+
+qemuarmv5/testimage:
   extends: .build
 
 sgi575:
@@ -111,18 +148,49 @@ tc0:
 # Utility tasks, not executed automatically
 #
 
-# Report on disk usage
-usage:
+delete-dl-dir:
   extends: .setup
-  stage: bootstrap
+  stage: prep
   when: manual
   script:
-  - du -h -s $DL_DIR $SSTATE_DIR
+  - rm -rf $DL_DIR/*
+
+delete-repo-dir:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - rm -rf $KAS_REPO_REF_DIR/*
+
+# Delete all sstate
+delete-sstate:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - rm -rf $SSTATE_DIR/*
+
+delete-toolchains:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - rm -rf $TOOLCHAIN_DIR/*
 
 # Wipe out old sstate
 prune-sstate:
   extends: .setup
-  stage: bootstrap
+  stage: prep
   when: manual
   script:
+  - du -h -s $SSTATE_DIR
   - find $SSTATE_DIR -type f -atime +30 -delete
+  - du -h -s $SSTATE_DIR
+
+# Report on disk usage
+usage:
+  extends: .setup
+  stage: prep
+  when: manual
+  script:
+  - du -h -s $DL_DIR $SSTATE_DIR $KAS_REPO_REF_DIR $TOOLCHAIN_DIR

ci/armgcc.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+
+local_conf_header:
+  libc: |
+    GCCVERSION = "arm-9.2"

ci/base.yml

@@ -22,6 +22,7 @@ repos:
 
 env:
   BB_LOGCONFIG: ""
+  TOOLCHAIN_DIR: ""
 
 local_conf_header:
   base: |
@@ -29,11 +30,11 @@ local_conf_header:
     PACKAGE_CLASSES = "package_ipk"
     LICENSE_FLAGS_WHITELIST += "armcompiler"
     PACKAGECONFIG_remove_pn-qemu-system-native = "gtk+ sdl"
+    EXTRA_IMAGE_FEATURES_append = " debug-tweaks"
     BB_NUMBER_THREADS = "16"
     PARALLEL_MAKE = "-j16"
     INHERIT += "rm_work"
     PACKAGECONFIG_append_pn-perf = " coresight"
-    ERROR_QA = "${WARN_QA}"
   ptest: |
     DISTRO_FEATURES_remove = "ptest"
 

ci/check-machine-coverage

@@ -13,11 +13,11 @@ if metaarm.name != "meta-arm":
 machines = metaarm.glob("meta-*/conf/machine/*.conf")
 machines = set(p.stem for p in machines)
 
-# All kas files
-kas = metaarm.glob("kas/*.yml")
-kas = set(p.stem for p in kas)
+# All ci files
+ci = metaarm.glob("ci/*.yml")
+ci = set(p.stem for p in ci)
 
-missing = machines - kas
+missing = machines - ci
 print(f"The following machines are missing: {', '.join(sorted(missing))}.")
 
 covered = len(machines) - len(missing)

ci/check-warnings

@@ -7,7 +7,8 @@ set -e -u
 
 LOGFILE=$1
 
-if test -s $LOGFILE; then
+LINES=$(grep --invert-match "attempting MIRRORS if available" $LOGFILE | wc -l)
+if test "$LINES" -ne 0; then
     echo ==============================
     echo The build had warnings/errors:
     echo ==============================

ci/external-gccarm.yml

@@ -0,0 +1,8 @@
+header:
+  version: 9
+
+local_conf_header:
+  cc: |
+    PNBLACKLIST[gcc-cross-arm] = "Using external toolchain"
+    TCMODE = "external-arm"
+    EXTERNAL_TOOLCHAIN = "${TOOLCHAIN_DIR}/${TARGET_ARCH}"

ci/get-binary-toolchains

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -u
+
+HOST_ARCH=$(uname -m)
+VER="10.2-2020.11"
+
+DOWNLOAD_DIR=$1
+TOOLCHAIN_DIR=$2
+
+# These should be already created by .bitlab-ci.yml, but do here if run outside of that env
+mkdir -p $DOWNLOAD_DIR $TOOLCHAIN_DIR
+
+if [ $HOST_ARCH = "aarch64" ]; then
+	#AArch64 Linux hosted cross compilers
+
+	#AArch32 target with hard float (arm-none-linux-gnueabihf)
+	wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu-a/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-arm-none-linux-gnueabihf.tar.xz
+elif [ $HOST_ARCH = "x86_64" ]; then
+	#x86_64 Linux hosted cross compilers
+
+	#AArch32 target with hard float (arm-linux-none-gnueabihf)
+	wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu-a/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-arm-none-linux-gnueabihf.tar.xz
+
+	#AArch64 GNU/Linux target (aarch64-none-linux-gnu)
+	wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu-a/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-aarch64-none-linux-gnu.tar.xz
+
+	#AArch64 GNU/Linux target (aarch64_be-none-linux-gnu)
+	wget -P $DOWNLOAD_DIR -nc https://developer.arm.com/-/media/Files/downloads/gnu-a/$VER/binrel/gcc-arm-$VER-$HOST_ARCH-aarch64_be-none-linux-gnu.tar.xz
+else
+	echo "ERROR - Unknown build arch of $HOST_ARCH"
+	exit 1
+fi
+
+for i in arm aarch64 aarch64_be; do
+	if [ ! -f $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz ]; then
+		continue
+	fi
+
+	if [ -d $TOOLCHAIN_DIR/$i ]; then
+		echo "$TOOLCHAIN_DIR/$i EXISTS!"
+		MANIFEST=$(ls $TOOLCHAIN_DIR/$i | grep txt)
+		if [[ $MANIFEST != $VER-$HOST_ARCH-$i-none-linux-gnu*.txt ]]; then
+			echo "Removing old $MANIFEST for $VER-$HOST_ARCH-$i-*.txt toolchain"
+			rm -rf $TOOLCHAIN_DIR/$i
+		fi
+	fi
+
+	if [ ! -d $TOOLCHAIN_DIR/$i ]; then
+		tar -C $TOOLCHAIN_DIR -axvf $DOWNLOAD_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*.tar.xz
+		mv $TOOLCHAIN_DIR/gcc-arm-$VER-$HOST_ARCH-$i-none-linux-gnu*/ $TOOLCHAIN_DIR/$i
+	fi
+done

ci/jobs-to-kas

@@ -8,8 +8,8 @@ set -e -u
 # Read Job namne from $1 and split on /
 IFS=/ read -r -a PARTS<<<$1
 
-# Prefix each part with kas/
-PARTS=("${PARTS[@]/#/kas/}")
+# Prefix each part with ci/
+PARTS=("${PARTS[@]/#/ci/}")
 
 # Suffix each part with .yml
 PARTS=("${PARTS[@]/%/.yml}")

ci/qemuarm.yml

@@ -0,0 +1,6 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: qemuarm

ci/qemuarm64-secureboot.yml

@@ -5,6 +5,11 @@ header:
 
 machine: qemuarm64-secureboot
 
+local_conf_header:
+  bugs: |
+    # Only ping until errors can be resolved
+    TEST_SUITES = "ping"
+
 target:
   - core-image-base
   - perf

ci/qemuarmv5.yml

@@ -0,0 +1,11 @@
+header:
+  version: 9
+  includes:
+    - base.yml
+
+machine: qemuarmv5
+
+local_conf_header:
+  bugs: |
+    # Remove parselogs until errors can be resolved
+    TEST_SUITES_remove = "parselogs"

ci/testimage.yml

@@ -0,0 +1,17 @@
+header:
+  version: 9
+
+local_conf_header:
+  testimage: |
+    IMAGE_CLASSES += "testimage"
+    TESTIMAGE_AUTO = "1"
+  slirp: |
+    TEST_RUNQEMUPARAMS = "slirp"
+    TEST_SERVER_IP = "127.0.0.1"
+    QEMU_USE_SLIRP = "1"
+  packages: |
+    IMAGE_FEATURES_append = " ssh-server-dropbear"
+
+# Multiple targets are available, put it down to just one
+target:
+  - core-image-base

ci/update-repos

@@ -0,0 +1,41 @@
+#! /usr/bin/env python3
+
+# Update clones of the repositories we need in KAS_REPO_REF_DIR to speed up fetches
+
+import sys
+import os
+import subprocess
+import pathlib
+
+def repo_shortname(url):
+    # Taken from Kas (Repo.__getattr__) to ensure the logic is right
+    from urllib.parse import urlparse
+    url = urlparse(url)
+    return ('{url.netloc}{url.path}'
+            .format(url=url)
+            .replace('@', '.')
+            .replace(':', '.')
+            .replace('/', '.')
+            .replace('*', '.'))
+
+repositories = (
+    "https://git.yoctoproject.org/git/poky",
+    "https://git.openembedded.org/meta-openembedded",
+    "https://git.yoctoproject.org/git/meta-virtualization",
+    "https://git.yoctoproject.org/git/meta-zephyr",
+    "https://github.com/kraj/meta-clang",
+)
+
+if __name__ == "__main__":
+    if "KAS_REPO_REF_DIR" not in os.environ:
+        print("KAS_REPO_REF_DIR needs to be set")
+        sys.exit(1)
+
+    base_repodir = pathlib.Path(os.environ["KAS_REPO_REF_DIR"])
+
+    for repo in repositories:
+        repodir = base_repodir / repo_shortname(repo)
+        if repodir.exists():
+            subprocess.run(["git", "-C", repodir, "fetch"], check=True)
+        else:
+            subprocess.run(["git", "clone", "--bare", repo, repodir], check=True)

meta-arm-bsp/recipes-kernel/linux/linux-arm-platforms.inc

@@ -51,6 +51,10 @@ COMPATIBLE_MACHINE_juno = "juno"
 KMACHINE_juno = "juno"
 KBUILD_DEFCONFIG_juno = "defconfig"
 KCONFIG_MODE_juno = "--alldefconfig"
+#FIXME - temporarily pin kernel version to allow for juno patches to apply
+KBRANCH_juno = "v5.4/standard/base"
+KERNEL_VERSION_SANITY_SKIP_juno = "1"
+SRCREV_machine_juno = "dd8a64a523fb714a98328441e0de72cde115a6fc"
 SRC_URI_append_juno = " \
     file://0001-mailbox-add-support-for-doorbell-signal-mode-control.patch \
     file://0002-dt-bindings-mailbox-add-bindings-to-support-ARM-MHU-.patch \

meta-arm-toolchain/conf/distro/include/external-arm-toolchain-versions.inc

@@ -139,19 +139,5 @@ python external_arm_toolchain_version_handler () {
     d.setVar('EAT_VER_KERNEL', eat_get_kernel_version(ld))
     d.setVar('EAT_VER_GDB', eat_get_gdb_version(ld))
     d.setVar('EAT_VER_BFD', eat_get_bfd_version(ld))
-
-    lc_libc = "LGPL-2.1"
-    lc_gcc = "GPL-3.0"
-    lc_rle = "GPL-3.0-with-GCC-exception"
-    lc_gdb = "GPL-3.0"
-    lc_bfd = "GPL-3.0"
-
-    d.setVar('EAT_LIBC_LICENSE', lc_libc)
-    d.setVar('EAT_GCC_LICENSE', lc_gcc)
-    d.setVar('EAT_RLE_LICENSE', lc_rle)
-    d.setVar('EAT_GDB_LICENSE', lc_gdb)
-    d.setVar('EAT_BFD_LICENSE', lc_bfd)
-
-    d.setVar('EAT_LICENSE', "%s & %s & %s & %s & %s" % (lc_libc, lc_gcc, lc_rle, lc_gdb, lc_bfd))
 }
 addhandler external_arm_toolchain_version_handler

meta-arm-toolchain/recipes-devtools/external-arm-toolchain/external-arm-sdk-toolchain.bb

@@ -1,5 +1,7 @@
 inherit cross-canadian
 
+require license.inc
+
 PN = "external-arm-sdk-toolchain-${TARGET_ARCH}"
 BPN = "external-arm-sdk-toolchain"
 PV = "${EAT_VER_MAIN}"
@@ -10,9 +12,6 @@ INHIBIT_SYSROOT_STRIP = "1"
 INHIBIT_DEFAULT_DEPS = "1"
 EXCLUDE_FROM_SHLIBS = "1"
 
-EAT_LICENSE ??= ""
-
-LICENSE = "${EAT_LICENSE}"
 LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"
 
 # Skip packaging QA checks for prebuilt binaries

meta-arm-toolchain/recipes-devtools/external-arm-toolchain/external-arm-toolchain.bb

@@ -115,6 +115,8 @@ do_install() {
 	rm -rf ${D}${bindir}/gdbserver
 	sed -i -e 's#/arm/tools/gnu/bash/4.2/rhe6-x86_64##' ${D}${bindir}/tzselect
 	sed -i -e 's#/arm/tools/gnu/bash/4.2/rhe6-x86_64##' ${D}${bindir}/ldd
+	sed -i -e 's#/usr/bin/bash#/bin/sh#' ${D}${bindir}/tzselect
+	sed -i -e 's#/usr/bin/bash#/bin/sh#' ${D}${bindir}/ldd
 	sed -i -e 's#/bin/bash#/bin/sh#' ${D}${bindir}/tzselect
 	sed -i -e 's#/bin/bash#/bin/sh#' ${D}${bindir}/ldd
 

meta-arm-toolchain/recipes-devtools/external-arm-toolchain/license.inc

@@ -1,7 +1,10 @@
 LICENSE = "GPL-3.0-with-GCC-exception & GPLv3 & GPLv2 & LGPLv3 & LGPLv2.1 & LGPLv2"
 
+EAT_BFD_LICENSE = "GPL-3.0"
 EAT_GCC_LICENSE = "GPL-3.0-with-GCC-exception & GPLv3"
+EAT_GDB_LICENSE = "GPL-3.0"
 EAT_LIBC_LICENSE = "GPLv2 & LGPLv2.1"
+EAT_RLE_LICENSE = "GPL-3.0-with-GCC-exception"
 
 LICENSE_${PN} = "${EAT_LIBC_LICENSE}"
 LICENSE_${PN}-dev = "${EAT_LIBC_LICENSE}"