From 85471f88e5e51e482c540b78542d352dcf793592 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Wed, 29 Jun 2016 17:18:09 -0500 Subject: [PATCH 1/4] bitbake: add crate fetcher This adds a handler for crate:// URLs and rewrites them to crates.io https endpoint and handles properly unpacking the crate. This fixes #24. --- classes/crate-fetch.bbclass | 13 +++ lib/crate.py | 202 ++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 classes/crate-fetch.bbclass create mode 100644 lib/crate.py diff --git a/classes/crate-fetch.bbclass b/classes/crate-fetch.bbclass new file mode 100644 index 0000000..c0ed434 --- /dev/null +++ b/classes/crate-fetch.bbclass @@ -0,0 +1,13 @@ +# +# crate-fetch class +# +# Registers 'crate' method for Bitbake fetch2. +# +# Adds support for following format in recipe SRC_URI: +# crate:/// +# + +python () { + import crate + bb.fetch2.methods.append( crate.Crate() ) +} diff --git a/lib/crate.py b/lib/crate.py new file mode 100644 index 0000000..49c5735 --- /dev/null +++ b/lib/crate.py @@ -0,0 +1,202 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementation for crates.io +""" + +# Copyright (C) 2016 Doug Goldstein +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import os +import shutil +import subprocess +import bb +from bb.fetch2 import logger, subprocess_setup, UnpackError +from bb.fetch2.wget import Wget + + +class Crate(Wget): + + """Class to fetch crates via wget""" + + def _cargo_path(self, rootdir, component): + # TODO: make this less brittle + repo = "github.com-88ac128001ac3a9a" + return os.path.join(rootdir, "cargo_home", "registry", component, repo) + + def _cargo_src_path(self, rootdir): + return self._cargo_path(rootdir, "src") + + def _cargo_index_path(self, rootdir): + return self._cargo_path(rootdir, "index") + + def _cargo_cache_path(self, rootdir): + return self._cargo_path(rootdir, "cache") + + def supports(self, ud, d): + """ + Check to see if a given url is for this fetcher + """ + return ud.type in ['crate', 'crate-index'] + + def recommends_checksum(self, urldata): + return False + + def urldata_init(self, ud, d): + """ + Sets up to download the respective crate from crates.io + """ + + if ud.type == 'crate': + self._crate_urldata_init(ud, d) + elif ud.type == 'crate-index': + self._index_urldata_init(ud, d) + + super(Crate, self).urldata_init(ud, d) + + def _crate_urldata_init(self, ud, d): + """ + Sets up the download for a crate + """ + + # URL syntax is: crate://NAME/VERSION + # break the URL apart by / + parts = ud.url.split('/') + if len(parts) < 5: + raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url) + + # last field is version + version = parts[len(parts) - 1] + # second to last field is name + name = parts[len(parts) - 2] + # host (this is to allow custom crate registries to be specified + host = '/'.join(parts[2:len(parts) - 2]) + + # if using upstream just fix it up nicely + if host == 'crates.io': + host = 'crates.io/api/v1/crates' + + ud.url = "https://%s/%s/%s/download" % (host, name, version) + ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) + ud.parm['name'] = name + + logger.debug(2, "Fetching %s to %s" % (ud.url, ud.parm['downloadfilename'])) + + def _index_urldata_init(self, ud, d): + """ + Sets up the download for the cargo index + """ + + # URL syntax is: crate-index://REV + # break the URL apart by / + parts = ud.url.split('/') + if len(parts) != 4: + raise bb.fetch2.ParameterError("Invalid URL: Must be crate-index://HOST/REV", ud.url) + + # last field is the rev + rev = parts[3] + host = parts[2] + + if host == 'crates.io': + host = 'github.com/rust-lang/crates.io-index' + + ud.url = "https://%s/archive/%s.tar.gz" % (host, rev) + ud.parm['downloadfilename'] = 'cargo-index-%s.tar.gz' % rev + ud.parm['name'] = "index" + + logger.debug(2, "Fetching crate index %s" % ud.url) + + def unpack(self, ud, rootdir, d): + """ + Uses the crate to build the necessary paths for cargo to utilize it + """ + if ud.type == 'crate-index': + return self._index_unpack(ud, rootdir, d) + elif ud.type == 'crate': + return self._crate_unpack(ud, rootdir, d) + else: + super(Crate, self).unpack(ud, rootdir, d) + + def _index_unpack(self, ud, rootdir, d): + """ + Unpacks the index + """ + thefile = ud.localpath + + cargo_index = self._cargo_index_path(rootdir) + + cmd = "tar -xz --no-same-owner --strip-components 1 -f %s -C %s" % (thefile, cargo_index) + + # change to the rootdir to unpack but save the old working dir + save_cwd = os.getcwd() + os.chdir(rootdir) + + # ensure we've got these paths made + bb.utils.mkdirhier(cargo_index) + + # path it + path = d.getVar('PATH', True) + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (thefile, cargo_index)) + + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) + + def _crate_unpack(self, ud, rootdir, d): + """ + Unpacks a crate + """ + thefile = ud.localpath + + # change to the rootdir to unpack but save the old working dir + save_cwd = os.getcwd() + os.chdir(rootdir) + + pn = d.getVar('PN', True) + if pn == ud.parm.get('name'): + cmd = "tar -xz --no-same-owner -f %s" % thefile + else: + cargo_src = self._cargo_src_path(rootdir) + cargo_cache = self._cargo_cache_path(rootdir) + + cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_src) + + # ensure we've got these paths made + bb.utils.mkdirhier(cargo_cache) + bb.utils.mkdirhier(cargo_src) + + bb.note("Copying %s to %s/" % (thefile, cargo_cache)) + shutil.copy(thefile, cargo_cache) + + # path it + path = d.getVar('PATH', True) + if path: + cmd = "PATH=\"%s\" %s" % (path, cmd) + bb.note("Unpacking %s to %s/" % (thefile, os.getcwd())) + + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) + + os.chdir(save_cwd) + + if ret != 0: + raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) + From d947c2e40eebc5e8a69a72eabae1572a4ee0f422 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Thu, 30 Jun 2016 14:06:59 -0500 Subject: [PATCH 2/4] cargo_util: simplified cargo bbclass for crates The existing cargo class attempts to jump through a few hoops for settings that are not necessarily with newer versions of cargo. --- classes/cargo_util.bbclass | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 classes/cargo_util.bbclass diff --git a/classes/cargo_util.bbclass b/classes/cargo_util.bbclass new file mode 100644 index 0000000..b5a5b61 --- /dev/null +++ b/classes/cargo_util.bbclass @@ -0,0 +1,87 @@ +# add crate fetch support +inherit crate-fetch + +# the binary we will use +CARGO = "cargo" + +# Where we download our registry and dependencies to +export CARGO_HOME = "${WORKDIR}/cargo_home" + +# We need cargo to compile for the target +BASEDEPENDS_append = " cargo-native" + +# Ensure we get the right rust variant +DEPENDS_append_class-target = "virtual/${TARGET_PREFIX}rust" + +# Cargo only supports in-tree builds at the moment +B = "${S}" + +# In case something fails in the build process, give a bit more feedback on +# where the issue occured +export RUST_BACKTRACE = "1" + +# The pkg-config-rs library used by cargo build scripts disables itself when +# cross compiling unless this is defined. We set up pkg-config appropriately +# for cross compilation, so tell it we know better than it. +export PKG_CONFIG_ALLOW_CROSS = "1" + +# All the rust & cargo ecosystem assume that CC, LD, etc are a path to a single +# command. Fixup the ones we give it so that is the case. +# XXX: this is hard coded based on meta/conf/bitbake.conf +# TODO: we do quite a bit very similar to this in rust.inc, see if it can be +# generalized. +export RUST_CC = "${CCACHE}${HOST_PREFIX}gcc" +export RUST_CFLAGS = "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${CFLAGS}" +export RUST_BUILD_CC = "${CCACHE}${BUILD_PREFIX}gcc" +export RUST_BUILD_CFLAGS = "${BUILD_CC_ARCH} ${BUILD_CFLAGS}" + +export CARGO_BUILD_FLAGS = "-v --target ${HOST_SYS} --release" + +# This is based on the content of CARGO_BUILD_FLAGS and generally will need to +# change if CARGO_BUILD_FLAGS changes. +export CARGO_TARGET_SUBDIR="${HOST_SYS}/release" +oe_cargo_build () { + bbnote "cargo = $(which cargo)" + bbnote "rustc = $(which rustc)" + bbnote "${CARGO} build ${CARGO_BUILD_FLAGS} $@" + "${CARGO}" build ${CARGO_BUILD_FLAGS} "$@" +} + +oe_cargo_fix_env () { + export CC="${RUST_CC}" + export CFLAGS="${RUST_CFLAGS}" + export AR="${AR}" + export TARGET_CC="${RUST_CC}" + export TARGET_CFLAGS="${RUST_CFLAGS}" + export TARGET_AR="${AR}" + export HOST_CC="${RUST_BUILD_CC}" + export HOST_CFLAGS="${RUST_BUILD_CFLAGS}" + export HOST_AR="${BUILD_AR}" +} + +cargo_util_do_compile () { + cd "${B}" + + # prevent cargo from trying to fetch down new data + touch "${WORKDIR}/cargo_home/registry/index/.cargo-index-lock" + + oe_cargo_fix_env + oe_cargo_build +} + +# All but the most simple projects will need to override this. +cargo_util_do_install () { + local have_installed=false + install -d "${D}${bindir}" + for tgt in "${B}/target/${CARGO_TARGET_SUBDIR}/"*; do + if [ -f "$tgt" ] && [ -x "$tgt" ]; then + install -m755 "$tgt" "${D}${bindir}" + have_installed=true + fi + done + if ! $have_installed; then + die "Did not find anything to install" + fi +} + +EXPORT_FUNCTIONS do_compile do_install From 49d94ef0a74bb3274ffcb3fe25011b64865b80bc Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Thu, 30 Jun 2016 08:51:40 -0500 Subject: [PATCH 3/4] rustfmt: an example cargo build package rustfmt is a Rust package to format Rust code. This package is being used as an example of building a crate in Yocto with Cargo. --- recipes-example/rustfmt/rustfmt_0.4.0.bb | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 recipes-example/rustfmt/rustfmt_0.4.0.bb diff --git a/recipes-example/rustfmt/rustfmt_0.4.0.bb b/recipes-example/rustfmt/rustfmt_0.4.0.bb new file mode 100644 index 0000000..27bb92d --- /dev/null +++ b/recipes-example/rustfmt/rustfmt_0.4.0.bb @@ -0,0 +1,44 @@ +inherit cargo_util + +CARGO_INDEX_COMMIT = "3b3994e099281c394a6a66604d1af6c0920e4c31" + +SRC_URI = " \ + crate://crates.io/aho-corasick/0.5.1 \ + crate://crates.io/bitflags/0.5.0 \ + crate://crates.io/diff/0.1.9 \ + crate://crates.io/env_logger/0.3.2 \ + crate://crates.io/getopts/0.2.14 \ + crate://crates.io/kernel32-sys/0.2.1 \ + crate://crates.io/libc/0.2.8 \ + crate://crates.io/log/0.3.5 \ + crate://crates.io/memchr/0.1.10 \ + crate://crates.io/regex/0.1.58 \ + crate://crates.io/regex-syntax/0.3.0 \ + crate://crates.io/rustc-serialize/0.3.18 \ + crate://crates.io/strings/0.0.1 \ + crate://crates.io/syntex_syntax/0.30.0 \ + crate://crates.io/term/0.2.14 \ + crate://crates.io/toml/0.1.28 \ + crate://crates.io/unicode-segmentation/0.1.2 \ + crate://crates.io/unicode-xid/0.0.3 \ + crate://crates.io/utf8-ranges/0.1.3 \ + crate://crates.io/winapi/0.2.6 \ + crate://crates.io/winapi-build/0.1.1 \ + crate://crates.io/rustfmt/0.4.0 \ + crate-index://crates.io/${CARGO_INDEX_COMMIT} \ +" +SRC_URI[md5sum] = "7fc46357c9c5e72a3a1ec3630c8c7a05" +SRC_URI[sha256sum] = "770c66dc845424a0c9a7f51b47d8de1e2605298da9b257ddde1d5be6fe01331f" +SRC_URI[index.md5sum] = "79f10f436dbf26737cc80445746f16b4" +SRC_URI[index.sha256sum] = "86114b93f1f51aaf0aec3af0751d214b351f4ff9839ba031315c1b19dcbb1913" + +# rustfmt 0.5.0 +#LIC_FILES_CHKSUM=" \ +# file://LICENSE-APACHE;md5=1836efb2eb779966696f473ee8540542 \ +# file://LICENSE-MIT;md5=0b29d505d9225d1f0815cbdcf602b901 \ +#" +LIC_FILES_CHKSUM="file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" + +SUMMARY = "Format Rust Code" +HOMEPAGE = "https://github.com/rust-lang-nursery/rustfmt" +LICENSE = "MIT | Apache-2.0" From a67fbf22231bb45af2ea1fea024049bb61201f47 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Tue, 12 Jul 2016 16:31:34 -0500 Subject: [PATCH 4/4] rust-hello-world: convert to using cargo_util Convert to using the newer cargo_util bbclass. --- recipes-example/rust-hello-world/rust-hello-world_git.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-example/rust-hello-world/rust-hello-world_git.bb b/recipes-example/rust-hello-world/rust-hello-world_git.bb index ba88548..7b5e53e 100644 --- a/recipes-example/rust-hello-world/rust-hello-world_git.bb +++ b/recipes-example/rust-hello-world/rust-hello-world_git.bb @@ -1,4 +1,4 @@ -inherit cargo +inherit cargo_util SRC_URI = "git://github.com/jmesmon/rust-hello-world.git;protocol=https" SRCREV="e0fa23f1a3cb1eb1407165bd2fc36d2f6e6ad728"