Merge pull request #99 from cardoe/krogoth-ci

Jenkins CI

Jenkinsfile

@@ -0,0 +1,36 @@
+def targets = [ 'qemux86', 'qemux86-64', 'qemuarm', 'qemuarm64' ]
+
+def machine_builds = [:]
+
+for (int i = 0; i < targets.size(); i++) {
+    def machine = targets.get(i)
+
+    machine_builds["$machine"] = {
+        node {
+            try {
+                stage('Checkout') {
+                    checkout scm
+                }
+                stage('Setup Environment') {
+                    sh "./scripts/setup-env.sh"
+                }
+                stage('Yocto Fetch') {
+                    sh "GIT_LOCAL_REF_DIR=/srv/git-cache/ ./scripts/fetch.sh krogoth"
+                }
+                stage('Build') {
+                    sh "MACHINE=${machine} ./scripts/build.sh"
+                }
+            } catch (e) {
+                echo "Caught: ${e}"
+                throw e
+            } finally {
+                stage('Cleanup Environment') {
+                    sh "./scripts/cleanup-env.sh"
+                    deleteDir()
+                }
+            }
+        }
+    }
+}
+
+parallel machine_builds

conf/bblayers.conf.sample

@@ -0,0 +1,21 @@
+# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
+# changes incompatibly
+LCONF_VERSION = "6"
+
+BBPATH = "${TOPDIR}"
+BBFILES ?= ""
+
+BBLAYERS ?= " \
+  ##OEROOT##/meta-rust \
+  ##OEROOT##/meta \
+  ##OEROOT##/meta-yocto \
+  ##OEROOT##/meta-yocto-bsp \
+  ##OEROOT##/meta-openembedded/meta-oe \
+  ##OEROOT##/meta-openembedded/meta-networking \
+  ##OEROOT##/meta-openembedded/meta-python \
+  ##OEROOT##/meta-openembedded/meta-ruby \
+  "
+BBLAYERS_NON_REMOVABLE ?= " \
+  ##OEROOT##/meta \
+  ##OEROOT##/meta-yocto \
+  "

conf/local.conf.sample

@@ -0,0 +1,242 @@
+#
+# This file is your local configuration file and is where all local user settings
+# are placed. The comments in this file give some guide to the options a new user
+# to the system might want to change but pretty much any configuration option can
+# be set in this file. More adventurous users can look at local.conf.extended
+# which contains other examples of configuration which can be placed in this file
+# but new users likely won't need any of them initially.
+#
+# Lines starting with the '#' character are commented out and in some cases the
+# default values are provided as comments to show people example syntax. Enabling
+# the option is a question of removing the # character and making any change to the
+# variable as required.
+
+#
+# Machine Selection
+#
+# You need to select a specific machine to target the build with. There are a selection
+# of emulated machines available which can boot and run in the QEMU emulator:
+#
+#MACHINE ?= "qemuarm"
+#MACHINE ?= "qemuarm64"
+#MACHINE ?= "qemumips"
+#MACHINE ?= "qemumips64"
+#MACHINE ?= "qemuppc"
+#MACHINE ?= "qemux86"
+#MACHINE ?= "qemux86-64"
+#
+# There are also the following hardware board target machines included for 
+# demonstration purposes:
+#
+#MACHINE ?= "beaglebone"
+#MACHINE ?= "genericx86"
+#MACHINE ?= "genericx86-64"
+#MACHINE ?= "mpc8315e-rdb"
+#MACHINE ?= "edgerouter"
+#
+# This sets the default machine to be qemux86 if no other machine is selected:
+MACHINE ??= "qemux86"
+
+#
+# Where to place downloads
+#
+# During a first build the system will download many different source code tarballs
+# from various upstream projects. This can take a while, particularly if your network
+# connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you
+# can preserve this directory to speed up this part of subsequent builds. This directory
+# is safe to share between multiple builds on the same machine too.
+#
+# The default is a downloads directory under TOPDIR which is the build directory.
+#
+#DL_DIR ?= "${TOPDIR}/downloads"
+
+#
+# Where to place shared-state files
+#
+# BitBake has the capability to accelerate builds based on previously built output.
+# This is done using "shared state" files which can be thought of as cache objects
+# and this option determines where those files are placed.
+#
+# You can wipe out TMPDIR leaving this directory intact and the build would regenerate
+# from these files if no changes were made to the configuration. If changes were made
+# to the configuration, only shared state files where the state was still valid would
+# be used (done using checksums).
+#
+# The default is a sstate-cache directory under TOPDIR.
+#
+#SSTATE_DIR ?= "${TOPDIR}/sstate-cache"
+
+#
+# Where to place the build output
+#
+# This option specifies where the bulk of the building work should be done and
+# where BitBake should place its temporary files and output. Keep in mind that
+# this includes the extraction and compilation of many applications and the toolchain
+# which can use Gigabytes of hard disk space.
+#
+# The default is a tmp directory under TOPDIR.
+#
+#TMPDIR = "${TOPDIR}/tmp"
+
+#
+# Default policy config
+#
+# The distribution setting controls which policy settings are used as defaults.
+# The default value is fine for general Yocto project use, at least initially.
+# Ultimately when creating custom policy, people will likely end up subclassing 
+# these defaults.
+#
+DISTRO ?= "poky"
+# As an example of a subclass there is a "bleeding" edge policy configuration
+# where many versions are set to the absolute latest code from the upstream 
+# source control systems. This is just mentioned here as an example, its not
+# useful to most new users.
+# DISTRO ?= "poky-bleeding"
+
+#
+# Package Management configuration
+#
+# This variable lists which packaging formats to enable. Multiple package backends
+# can be enabled at once and the first item listed in the variable will be used
+# to generate the root filesystems.
+# Options are:
+#  - 'package_deb' for debian style deb files
+#  - 'package_ipk' for ipk files are used by opkg (a debian style embedded package manager)
+#  - 'package_rpm' for rpm style packages
+# E.g.: PACKAGE_CLASSES ?= "package_rpm package_deb package_ipk"
+# We default to rpm:
+PACKAGE_CLASSES ?= "package_rpm"
+
+#
+# SDK target architecture
+#
+# This variable specifies the architecture to build SDK items for and means
+# you can build the SDK packages for architectures other than the machine you are
+# running the build on (i.e. building i686 packages on an x86_64 host).
+# Supported values are i686 and x86_64
+#SDKMACHINE ?= "i686"
+
+#
+# Extra image configuration defaults
+#
+# The EXTRA_IMAGE_FEATURES variable allows extra packages to be added to the generated
+# images. Some of these options are added to certain image types automatically. The
+# variable can contain the following options:
+#  "dbg-pkgs"       - add -dbg packages for all installed packages
+#                     (adds symbol information for debugging/profiling)
+#  "dev-pkgs"       - add -dev packages for all installed packages
+#                     (useful if you want to develop against libs in the image)
+#  "ptest-pkgs"     - add -ptest packages for all ptest-enabled packages
+#                     (useful if you want to run the package test suites)
+#  "tools-sdk"      - add development tools (gcc, make, pkgconfig etc.)
+#  "tools-debug"    - add debugging tools (gdb, strace)
+#  "eclipse-debug"  - add Eclipse remote debugging support
+#  "tools-profile"  - add profiling tools (oprofile, lttng, valgrind)
+#  "tools-testapps" - add useful testing tools (ts_print, aplay, arecord etc.)
+#  "debug-tweaks"   - make an image suitable for development
+#                     e.g. ssh root access has a blank password
+# There are other application targets that can be used here too, see
+# meta/classes/image.bbclass and meta/classes/core-image.bbclass for more details.
+# We default to enabling the debugging tweaks.
+EXTRA_IMAGE_FEATURES ?= "debug-tweaks"
+
+#
+# Additional image features
+#
+# The following is a list of additional classes to use when building images which
+# enable extra features. Some available options which can be included in this variable
+# are:
+#   - 'buildstats' collect build statistics
+#   - 'image-mklibs' to reduce shared library files size for an image
+#   - 'image-prelink' in order to prelink the filesystem image
+#   - 'image-swab' to perform host system intrusion detection
+# NOTE: if listing mklibs & prelink both, then make sure mklibs is before prelink
+# NOTE: mklibs also needs to be explicitly enabled for a given image, see local.conf.extended
+# image-prelink disabled for now due to issues with IFUNC symbol relocation
+USER_CLASSES ?= "buildstats image-mklibs"
+
+#
+# Runtime testing of images
+#
+# The build system can test booting virtual machine images under qemu (an emulator)
+# after any root filesystems are created and run tests against those images. To
+# enable this uncomment this line. See classes/testimage(-auto).bbclass for
+# further details.
+#TEST_IMAGE = "1"
+#
+# Interactive shell configuration
+#
+# Under certain circumstances the system may need input from you and to do this it
+# can launch an interactive shell. It needs to do this since the build is
+# multithreaded and needs to be able to handle the case where more than one parallel
+# process may require the user's attention. The default is iterate over the available
+# terminal types to find one that works.
+#
+# Examples of the occasions this may happen are when resolving patches which cannot
+# be applied, to use the devshell or the kernel menuconfig
+#
+# Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none
+# Note: currently, Konsole support only works for KDE 3.x due to the way
+# newer Konsole versions behave
+#OE_TERMINAL = "auto"
+# By default disable interactive patch resolution (tasks will just fail instead):
+PATCHRESOLVE = "noop"
+
+#
+# Disk Space Monitoring during the build
+#
+# Monitor the disk space during the build. If there is less that 1GB of space or less
+# than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully
+# shutdown the build. If there is less that 100MB or 1K inodes, perform a hard abort
+# of the build. The reason for this is that running completely out of space can corrupt
+# files and damages the build in ways which may not be easily recoverable.
+# It's necesary to monitor /tmp, if there is no space left the build will fail
+# with very exotic errors.
+BB_DISKMON_DIRS = "\
+    STOPTASKS,${TMPDIR},1G,100K \
+    STOPTASKS,${DL_DIR},1G,100K \
+    STOPTASKS,${SSTATE_DIR},1G,100K \
+    STOPTASKS,/tmp,100M,100K \
+    ABORT,${TMPDIR},100M,1K \
+    ABORT,${DL_DIR},100M,1K \
+    ABORT,${SSTATE_DIR},100M,1K \
+    ABORT,/tmp,10M,1K"
+
+#
+# Shared-state files from other locations
+#
+# As mentioned above, shared state files are prebuilt cache data objects which can
+# used to accelerate build time. This variable can be used to configure the system
+# to search other mirror locations for these objects before it builds the data itself.
+#
+# This can be a filesystem directory, or a remote url such as http or ftp. These
+# would contain the sstate-cache results from previous builds (possibly from other
+# machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the
+# cache locations to check for the shared objects.
+# NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH
+# at the end as shown in the examples below. This will be substituted with the
+# correct path within the directory structure.
+#SSTATE_MIRRORS ?= "\
+#file://.* http://someserver.tld/share/sstate/PATH;downloadfilename=PATH \n \
+#file://.* file:///some/local/dir/sstate/PATH"
+
+SSTATE_MIRRORS ?= "file://.* http://build-cache.asterius.io/sstate/PATH;downloadfilename=PATH \n"
+
+SOURCE_MIRROR_URL ?= "http://build-cache.asterius.io/downloads/"
+INHERIT += "own-mirrors rm_work"
+
+#
+# Qemu configuration
+#
+# By default qemu will build with a builtin VNC server where graphical output can be
+# seen. The two lines below enable the SDL backend too. By default libsdl-native will
+# be built, if you want to use your host's libSDL instead of the minimal libsdl built
+# by libsdl-native then uncomment the ASSUME_PROVIDED line below.
+PACKAGECONFIG_append_pn-qemu-native = " sdl"
+PACKAGECONFIG_append_pn-nativesdk-qemu = " sdl"
+#ASSUME_PROVIDED += "libsdl-native"
+
+# CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to
+# track the version of this file when it was generated. This can safely be ignored if
+# this doesn't mean anything to you.
+CONF_VERSION = "1"

scripts/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Grab the MACHINE from the environment; otherwise, set it to a sane default
+export MACHINE="${MACHINE-qemux86}"
+
+# What to build
+BUILD_TARGETS="\
+    rustfmt \
+    "
+
+die() {
+    echo "$*" >&2
+    exit 1
+}
+
+rm -f build/conf/bblayers.conf || die "failed to nuke bblayers.conf"
+rm -f build/conf/local.conf || die "failed to nuke local.conf"
+
+./scripts/containerize.sh bitbake ${BUILD_TARGETS} || die "failed to build"

scripts/cleanup-env.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+sudo umount build
+
+exit 0

scripts/containerize.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# what container are we using to build this
+CONTAINER="starlabio/yocto:1.5"
+
+einfo() {
+	echo "$*" >&2
+}
+
+die() {
+    echo "$*" >&2
+    exit 1
+}
+
+# Save the commands for future use
+cmd=$@
+
+# If no command was specified, just drop us into a shell if we're interactive
+[ $# -eq 0 ] && tty -s && cmd="/bin/bash"
+
+# user and group we are running as to ensure files created inside
+# the container retain the same permissions
+my_uid=$(id -u)
+my_gid=$(id -g)
+
+# Are we in an interactive terminal?
+tty -s && termint=t
+
+# Fetch the latest version of the container
+einfo "*** Ensuring local container is up to date"
+docker pull ${CONTAINER} > /dev/null || die "Failed to update docker container"
+
+# Ensure we've got what we need for SSH_AUTH_SOCK
+if [[ -n ${SSH_AUTH_SOCK} ]]; then
+	SSH_AUTH_DIR=$(dirname $(readlink -f ${SSH_AUTH_SOCK}))
+	SSH_AUTH_NAME=$(basename ${SSH_AUTH_SOCK})
+fi
+
+# Kick off Docker
+einfo "*** Launching container ..."
+exec docker run \
+    --privileged \
+    -e BUILD_UID=${my_uid} \
+    -e BUILD_GID=${my_gid} \
+    -e TEMPLATECONF=meta-rust/conf \
+    -e MACHINE=${MACHINE:-qemux86} \
+    ${SSH_AUTH_SOCK:+-e SSH_AUTH_SOCK="/tmp/ssh-agent/${SSH_AUTH_NAME}"} \
+    -v ${HOME}/.ssh:/var/build/.ssh \
+    -v "${PWD}":/var/build:rw \
+    ${SSH_AUTH_SOCK:+-v "${SSH_AUTH_DIR}":/tmp/ssh-agent} \
+    ${EXTRA_CONTAINER_ARGS} \
+    -${termint}i --rm -- \
+    ${CONTAINER} \
+    ${cmd}

scripts/fetch.sh

@@ -0,0 +1,95 @@
+#!/bin/bash -x
+
+# the repos we want to check out, must setup variables below
+# NOTE: poky must remain first
+REPOS="poky metaoe"
+
+POKY_URI="git://git.yoctoproject.org/poky.git"
+POKY_PATH="poky"
+POKY_REV="${POKY_REV-refs/remotes/origin/$1}"
+
+METAOE_URI="git://git.openembedded.org/meta-openembedded.git"
+METAOE_PATH="poky/meta-openembedded"
+METAOE_REV="${METAOE_REV-refs/remotes/origin/$1}"
+
+METARUST_URI="."
+METARUST_PATH="poky/meta-rust"
+
+die() {
+	echo "$*" >&2
+	exit 1
+}
+
+update_repo() {
+	uri=$1
+	path=$2
+	rev=$3
+
+	# check if we already have it checked out, if so we just want to update
+	if [[ -d ${path} ]]; then
+		pushd ${path} > /dev/null
+		echo "Updating '${path}'"
+		git remote set-url origin "${uri}"
+		git fetch origin || die "unable to fetch ${uri}"
+	else
+		echo "Cloning '${path}'"
+		if [ -z "${GIT_LOCAL_REF_DIR}" ]; then
+			git clone ${uri} ${path} || die "unable to clone ${uri}"
+		else
+			git clone --reference ${GIT_LOCAL_REF_DIR}/`basename ${path}` ${uri} ${path}
+		fi
+		pushd ${path} > /dev/null
+	fi
+
+	# The reset steps are taken from Jenkins
+
+	# Reset
+	# * drop -d from clean to not nuke build/tmp
+	# * add -e to not clear out bitbake bits
+	git reset --hard || die "failed reset"
+	git clean -fx -e bitbake -e meta/lib/oe || die "failed clean"
+
+	# Call the branch what we're basing it on, otherwise use default
+	# if the revision was not a branch.
+	branch=$(basename ${rev})
+	[[ "${branch}" == "${rev}" ]] && branch="default"
+
+	# Create 'default' branch
+	git update-ref refs/heads/${branch} ${rev} || \
+		die "unable to get ${rev} of ${uri}"
+	git config branch.${branch}.remote origin || die "failed config remote"
+	git config branch.${branch}.merge ${rev} || die "failed config merge"
+	git symbolic-ref HEAD refs/heads/${branch} || die "failed symbolic-ref"
+	git reset --hard || die "failed reset"
+	popd > /dev/null
+	echo "Updated '${path}' to '${rev}'"
+}
+
+# For each repo, do the work
+for repo in ${REPOS}; do
+	# upper case the name
+	repo=$(echo ${repo} | tr '[:lower:]' '[:upper:]')
+
+	# expand variables
+	expand_uri="${repo}_URI"
+	expand_path="${repo}_PATH"
+	expand_rev="${repo}_REV"
+	repo_uri=${!expand_uri}
+	repo_path=${!expand_path}
+	repo_rev=${!expand_rev}
+
+	# check that we've got data
+	[[ -z ${repo_uri} ]] && die "No revision defined in ${expand_uri}"
+	[[ -z ${repo_path} ]] && die "No revision defined in ${expand_path}"
+	[[ -z ${repo_rev} ]] && die "No revision defined in ${expand_rev}"
+
+	# now fetch/clone/update repo
+	update_repo "${repo_uri}" "${repo_path}" "${repo_rev}"
+
+done
+
+rm -rf "${METARUST_PATH}" || die "unable to clear old ${METARUST_PATH}"
+ln -sf "../${METARUST_URI}" "${METARUST_PATH}" || \
+	die "unable to symlink ${METARUST_PATH}"
+
+exit 0

scripts/setup-env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+mkdir -p build
+
+sudo mount -t tmpfs -o size=64G,mode=755,uid=${UID} tmpfs build
+
+exit 0