diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..2fc6fdc --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,36 @@ +def targets = [ 'qemux86', 'qemux86-64', 'qemuarm', 'qemuarm64' ] + +def machine_builds = [:] + +for (int i = 0; i < targets.size(); i++) { + def machine = targets.get(i) + + machine_builds["$machine"] = { + node { + try { + stage('Checkout') { + checkout scm + } + stage('Setup Environment') { + sh "./scripts/setup-env.sh" + } + stage('Yocto Fetch') { + sh "GIT_LOCAL_REF_DIR=/srv/git-cache/ ./scripts/fetch.sh master" + } + stage('Build') { + sh "MACHINE=${machine} ./scripts/build.sh" + } + } catch (e) { + echo "Caught: ${e}" + throw e + } finally { + stage('Cleanup Environment') { + sh "./scripts/cleanup-env.sh" + deleteDir() + } + } + } + } +} + +parallel machine_builds diff --git a/conf/bblayers.conf.sample b/conf/bblayers.conf.sample new file mode 100644 index 0000000..c2d73a2 --- /dev/null +++ b/conf/bblayers.conf.sample @@ -0,0 +1,21 @@ +# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf +# changes incompatibly +LCONF_VERSION = "6" + +BBPATH = "${TOPDIR}" +BBFILES ?= "" + +BBLAYERS ?= " \ + ##OEROOT##/meta-rust \ + ##OEROOT##/meta \ + ##OEROOT##/meta-yocto \ + ##OEROOT##/meta-yocto-bsp \ + ##OEROOT##/meta-openembedded/meta-oe \ + ##OEROOT##/meta-openembedded/meta-networking \ + ##OEROOT##/meta-openembedded/meta-python \ + ##OEROOT##/meta-openembedded/meta-ruby \ + " +BBLAYERS_NON_REMOVABLE ?= " \ + ##OEROOT##/meta \ + ##OEROOT##/meta-yocto \ + " diff --git a/conf/local.conf.sample b/conf/local.conf.sample new file mode 100644 index 0000000..08e6e7e --- /dev/null +++ b/conf/local.conf.sample @@ -0,0 +1,242 @@ +# +# This file is your local configuration file and is where all local user settings +# are placed. The comments in this file give some guide to the options a new user +# to the system might want to change but pretty much any configuration option can +# be set in this file. More adventurous users can look at local.conf.extended +# which contains other examples of configuration which can be placed in this file +# but new users likely won't need any of them initially. +# +# Lines starting with the '#' character are commented out and in some cases the +# default values are provided as comments to show people example syntax. Enabling +# the option is a question of removing the # character and making any change to the +# variable as required. + +# +# Machine Selection +# +# You need to select a specific machine to target the build with. There are a selection +# of emulated machines available which can boot and run in the QEMU emulator: +# +#MACHINE ?= "qemuarm" +#MACHINE ?= "qemuarm64" +#MACHINE ?= "qemumips" +#MACHINE ?= "qemumips64" +#MACHINE ?= "qemuppc" +#MACHINE ?= "qemux86" +#MACHINE ?= "qemux86-64" +# +# There are also the following hardware board target machines included for +# demonstration purposes: +# +#MACHINE ?= "beaglebone" +#MACHINE ?= "genericx86" +#MACHINE ?= "genericx86-64" +#MACHINE ?= "mpc8315e-rdb" +#MACHINE ?= "edgerouter" +# +# This sets the default machine to be qemux86 if no other machine is selected: +MACHINE ??= "qemux86" + +# +# Where to place downloads +# +# During a first build the system will download many different source code tarballs +# from various upstream projects. This can take a while, particularly if your network +# connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you +# can preserve this directory to speed up this part of subsequent builds. This directory +# is safe to share between multiple builds on the same machine too. +# +# The default is a downloads directory under TOPDIR which is the build directory. +# +#DL_DIR ?= "${TOPDIR}/downloads" + +# +# Where to place shared-state files +# +# BitBake has the capability to accelerate builds based on previously built output. +# This is done using "shared state" files which can be thought of as cache objects +# and this option determines where those files are placed. +# +# You can wipe out TMPDIR leaving this directory intact and the build would regenerate +# from these files if no changes were made to the configuration. If changes were made +# to the configuration, only shared state files where the state was still valid would +# be used (done using checksums). +# +# The default is a sstate-cache directory under TOPDIR. +# +#SSTATE_DIR ?= "${TOPDIR}/sstate-cache" + +# +# Where to place the build output +# +# This option specifies where the bulk of the building work should be done and +# where BitBake should place its temporary files and output. Keep in mind that +# this includes the extraction and compilation of many applications and the toolchain +# which can use Gigabytes of hard disk space. +# +# The default is a tmp directory under TOPDIR. +# +#TMPDIR = "${TOPDIR}/tmp" + +# +# Default policy config +# +# The distribution setting controls which policy settings are used as defaults. +# The default value is fine for general Yocto project use, at least initially. +# Ultimately when creating custom policy, people will likely end up subclassing +# these defaults. +# +DISTRO ?= "poky" +# As an example of a subclass there is a "bleeding" edge policy configuration +# where many versions are set to the absolute latest code from the upstream +# source control systems. This is just mentioned here as an example, its not +# useful to most new users. +# DISTRO ?= "poky-bleeding" + +# +# Package Management configuration +# +# This variable lists which packaging formats to enable. Multiple package backends +# can be enabled at once and the first item listed in the variable will be used +# to generate the root filesystems. +# Options are: +# - 'package_deb' for debian style deb files +# - 'package_ipk' for ipk files are used by opkg (a debian style embedded package manager) +# - 'package_rpm' for rpm style packages +# E.g.: PACKAGE_CLASSES ?= "package_rpm package_deb package_ipk" +# We default to rpm: +PACKAGE_CLASSES ?= "package_rpm" + +# +# SDK target architecture +# +# This variable specifies the architecture to build SDK items for and means +# you can build the SDK packages for architectures other than the machine you are +# running the build on (i.e. building i686 packages on an x86_64 host). +# Supported values are i686 and x86_64 +#SDKMACHINE ?= "i686" + +# +# Extra image configuration defaults +# +# The EXTRA_IMAGE_FEATURES variable allows extra packages to be added to the generated +# images. Some of these options are added to certain image types automatically. The +# variable can contain the following options: +# "dbg-pkgs" - add -dbg packages for all installed packages +# (adds symbol information for debugging/profiling) +# "dev-pkgs" - add -dev packages for all installed packages +# (useful if you want to develop against libs in the image) +# "ptest-pkgs" - add -ptest packages for all ptest-enabled packages +# (useful if you want to run the package test suites) +# "tools-sdk" - add development tools (gcc, make, pkgconfig etc.) +# "tools-debug" - add debugging tools (gdb, strace) +# "eclipse-debug" - add Eclipse remote debugging support +# "tools-profile" - add profiling tools (oprofile, lttng, valgrind) +# "tools-testapps" - add useful testing tools (ts_print, aplay, arecord etc.) +# "debug-tweaks" - make an image suitable for development +# e.g. ssh root access has a blank password +# There are other application targets that can be used here too, see +# meta/classes/image.bbclass and meta/classes/core-image.bbclass for more details. +# We default to enabling the debugging tweaks. +EXTRA_IMAGE_FEATURES ?= "debug-tweaks" + +# +# Additional image features +# +# The following is a list of additional classes to use when building images which +# enable extra features. Some available options which can be included in this variable +# are: +# - 'buildstats' collect build statistics +# - 'image-mklibs' to reduce shared library files size for an image +# - 'image-prelink' in order to prelink the filesystem image +# - 'image-swab' to perform host system intrusion detection +# NOTE: if listing mklibs & prelink both, then make sure mklibs is before prelink +# NOTE: mklibs also needs to be explicitly enabled for a given image, see local.conf.extended +# image-prelink disabled for now due to issues with IFUNC symbol relocation +USER_CLASSES ?= "buildstats image-mklibs" + +# +# Runtime testing of images +# +# The build system can test booting virtual machine images under qemu (an emulator) +# after any root filesystems are created and run tests against those images. To +# enable this uncomment this line. See classes/testimage(-auto).bbclass for +# further details. +#TEST_IMAGE = "1" +# +# Interactive shell configuration +# +# Under certain circumstances the system may need input from you and to do this it +# can launch an interactive shell. It needs to do this since the build is +# multithreaded and needs to be able to handle the case where more than one parallel +# process may require the user's attention. The default is iterate over the available +# terminal types to find one that works. +# +# Examples of the occasions this may happen are when resolving patches which cannot +# be applied, to use the devshell or the kernel menuconfig +# +# Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none +# Note: currently, Konsole support only works for KDE 3.x due to the way +# newer Konsole versions behave +#OE_TERMINAL = "auto" +# By default disable interactive patch resolution (tasks will just fail instead): +PATCHRESOLVE = "noop" + +# +# Disk Space Monitoring during the build +# +# Monitor the disk space during the build. If there is less that 1GB of space or less +# than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully +# shutdown the build. If there is less that 100MB or 1K inodes, perform a hard abort +# of the build. The reason for this is that running completely out of space can corrupt +# files and damages the build in ways which may not be easily recoverable. +# It's necesary to monitor /tmp, if there is no space left the build will fail +# with very exotic errors. +BB_DISKMON_DIRS = "\ + STOPTASKS,${TMPDIR},1G,100K \ + STOPTASKS,${DL_DIR},1G,100K \ + STOPTASKS,${SSTATE_DIR},1G,100K \ + STOPTASKS,/tmp,100M,100K \ + ABORT,${TMPDIR},100M,1K \ + ABORT,${DL_DIR},100M,1K \ + ABORT,${SSTATE_DIR},100M,1K \ + ABORT,/tmp,10M,1K" + +# +# Shared-state files from other locations +# +# As mentioned above, shared state files are prebuilt cache data objects which can +# used to accelerate build time. This variable can be used to configure the system +# to search other mirror locations for these objects before it builds the data itself. +# +# This can be a filesystem directory, or a remote url such as http or ftp. These +# would contain the sstate-cache results from previous builds (possibly from other +# machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the +# cache locations to check for the shared objects. +# NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH +# at the end as shown in the examples below. This will be substituted with the +# correct path within the directory structure. +#SSTATE_MIRRORS ?= "\ +#file://.* http://someserver.tld/share/sstate/PATH;downloadfilename=PATH \n \ +#file://.* file:///some/local/dir/sstate/PATH" + +SSTATE_MIRRORS ?= "file://.* http://build-cache.asterius.io/sstate/PATH;downloadfilename=PATH \n" + +SOURCE_MIRROR_URL ?= "http://build-cache.asterius.io/downloads/" +INHERIT += "own-mirrors rm_work" + +# +# Qemu configuration +# +# By default qemu will build with a builtin VNC server where graphical output can be +# seen. The two lines below enable the SDL backend too. By default libsdl-native will +# be built, if you want to use your host's libSDL instead of the minimal libsdl built +# by libsdl-native then uncomment the ASSUME_PROVIDED line below. +PACKAGECONFIG_append_pn-qemu-native = " sdl" +PACKAGECONFIG_append_pn-nativesdk-qemu = " sdl" +#ASSUME_PROVIDED += "libsdl-native" + +# CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to +# track the version of this file when it was generated. This can safely be ignored if +# this doesn't mean anything to you. +CONF_VERSION = "1" diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..7c40381 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Grab the MACHINE from the environment; otherwise, set it to a sane default +export MACHINE="${MACHINE-qemux86}" + +# What to build +BUILD_TARGETS="\ + rustfmt \ + " + +die() { + echo "$*" >&2 + exit 1 +} + +rm -f build/conf/bblayers.conf || die "failed to nuke bblayers.conf" +rm -f build/conf/local.conf || die "failed to nuke local.conf" + +./scripts/containerize.sh bitbake ${BUILD_TARGETS} || die "failed to build" diff --git a/scripts/cleanup-env.sh b/scripts/cleanup-env.sh new file mode 100755 index 0000000..1367724 --- /dev/null +++ b/scripts/cleanup-env.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +sudo umount build + +exit 0 diff --git a/scripts/containerize.sh b/scripts/containerize.sh new file mode 100755 index 0000000..c9ac0db --- /dev/null +++ b/scripts/containerize.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# what container are we using to build this +CONTAINER="starlabio/yocto:1.5" + +einfo() { + echo "$*" >&2 +} + +die() { + echo "$*" >&2 + exit 1 +} + +# Save the commands for future use +cmd=$@ + +# If no command was specified, just drop us into a shell if we're interactive +[ $# -eq 0 ] && tty -s && cmd="/bin/bash" + +# user and group we are running as to ensure files created inside +# the container retain the same permissions +my_uid=$(id -u) +my_gid=$(id -g) + +# Are we in an interactive terminal? +tty -s && termint=t + +# Fetch the latest version of the container +einfo "*** Ensuring local container is up to date" +docker pull ${CONTAINER} > /dev/null || die "Failed to update docker container" + +# Ensure we've got what we need for SSH_AUTH_SOCK +if [[ -n ${SSH_AUTH_SOCK} ]]; then + SSH_AUTH_DIR=$(dirname $(readlink -f ${SSH_AUTH_SOCK})) + SSH_AUTH_NAME=$(basename ${SSH_AUTH_SOCK}) +fi + +# Kick off Docker +einfo "*** Launching container ..." +exec docker run \ + --privileged \ + -e BUILD_UID=${my_uid} \ + -e BUILD_GID=${my_gid} \ + -e TEMPLATECONF=meta-rust/conf \ + -e MACHINE=${MACHINE:-qemux86} \ + ${SSH_AUTH_SOCK:+-e SSH_AUTH_SOCK="/tmp/ssh-agent/${SSH_AUTH_NAME}"} \ + -v ${HOME}/.ssh:/var/build/.ssh \ + -v "${PWD}":/var/build:rw \ + ${SSH_AUTH_SOCK:+-v "${SSH_AUTH_DIR}":/tmp/ssh-agent} \ + ${EXTRA_CONTAINER_ARGS} \ + -${termint}i --rm -- \ + ${CONTAINER} \ + ${cmd} diff --git a/scripts/fetch.sh b/scripts/fetch.sh new file mode 100755 index 0000000..4a54df3 --- /dev/null +++ b/scripts/fetch.sh @@ -0,0 +1,95 @@ +#!/bin/bash -x + +# the repos we want to check out, must setup variables below +# NOTE: poky must remain first +REPOS="poky metaoe" + +POKY_URI="git://git.yoctoproject.org/poky.git" +POKY_PATH="poky" +POKY_REV="${POKY_REV-refs/remotes/origin/$1}" + +METAOE_URI="git://git.openembedded.org/meta-openembedded.git" +METAOE_PATH="poky/meta-openembedded" +METAOE_REV="${METAOE_REV-refs/remotes/origin/$1}" + +METARUST_URI="." +METARUST_PATH="poky/meta-rust" + +die() { + echo "$*" >&2 + exit 1 +} + +update_repo() { + uri=$1 + path=$2 + rev=$3 + + # check if we already have it checked out, if so we just want to update + if [[ -d ${path} ]]; then + pushd ${path} > /dev/null + echo "Updating '${path}'" + git remote set-url origin "${uri}" + git fetch origin || die "unable to fetch ${uri}" + else + echo "Cloning '${path}'" + if [ -z "${GIT_LOCAL_REF_DIR}" ]; then + git clone ${uri} ${path} || die "unable to clone ${uri}" + else + git clone --reference ${GIT_LOCAL_REF_DIR}/`basename ${path}` ${uri} ${path} + fi + pushd ${path} > /dev/null + fi + + # The reset steps are taken from Jenkins + + # Reset + # * drop -d from clean to not nuke build/tmp + # * add -e to not clear out bitbake bits + git reset --hard || die "failed reset" + git clean -fx -e bitbake -e meta/lib/oe || die "failed clean" + + # Call the branch what we're basing it on, otherwise use default + # if the revision was not a branch. + branch=$(basename ${rev}) + [[ "${branch}" == "${rev}" ]] && branch="default" + + # Create 'default' branch + git update-ref refs/heads/${branch} ${rev} || \ + die "unable to get ${rev} of ${uri}" + git config branch.${branch}.remote origin || die "failed config remote" + git config branch.${branch}.merge ${rev} || die "failed config merge" + git symbolic-ref HEAD refs/heads/${branch} || die "failed symbolic-ref" + git reset --hard || die "failed reset" + popd > /dev/null + echo "Updated '${path}' to '${rev}'" +} + +# For each repo, do the work +for repo in ${REPOS}; do + # upper case the name + repo=$(echo ${repo} | tr '[:lower:]' '[:upper:]') + + # expand variables + expand_uri="${repo}_URI" + expand_path="${repo}_PATH" + expand_rev="${repo}_REV" + repo_uri=${!expand_uri} + repo_path=${!expand_path} + repo_rev=${!expand_rev} + + # check that we've got data + [[ -z ${repo_uri} ]] && die "No revision defined in ${expand_uri}" + [[ -z ${repo_path} ]] && die "No revision defined in ${expand_path}" + [[ -z ${repo_rev} ]] && die "No revision defined in ${expand_rev}" + + # now fetch/clone/update repo + update_repo "${repo_uri}" "${repo_path}" "${repo_rev}" + +done + +rm -rf "${METARUST_PATH}" || die "unable to clear old ${METARUST_PATH}" +ln -sf "../${METARUST_URI}" "${METARUST_PATH}" || \ + die "unable to symlink ${METARUST_PATH}" + +exit 0 diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh new file mode 100755 index 0000000..e5fc3b1 --- /dev/null +++ b/scripts/setup-env.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +mkdir -p build + +sudo mount -t tmpfs -o size=64G,mode=755,uid=${UID} tmpfs build + +exit 0