Merge pull request #71 from srwalter/cargo-no-network

Update cargo support so that there truly is no network access

README.md

@@ -17,6 +17,19 @@ This openembedded layer provides the rust compiler, tools for building packages
 
  - rust (built for target)
 
+## Building a rust package
+
+When building a rust package in bitbake, it's usually easiest to build with
+cargo using cargo.bbclass.  If the package already has a Cargo.toml file (most
+rust packages do), then it's especially easy.  Otherwise you should probably
+get the code building in cargo first.
+
+Once your package builds in cargo, you can use
+[cargo-bitbake](https://github.com/cardoe/cargo-bitbake) to generate a bitbake
+recipe for it.  This allows bitbake to fetch all the necessary dependent
+crates, as well as a pegged version of the crates.io index, to ensure maximum
+reproducibility.
+
 ## Common issues when packaging things using cargo
 
  You may run into errors similar to:

classes/cargo.bbclass

@@ -1,20 +1,23 @@
-inherit rust
+inherit rust-vars
+# add crate fetch support
+inherit crate-fetch
 
-CARGO ?= "cargo"
+# the binary we will use
+CARGO = "cargo"
+
+# Where we download our registry and dependencies to
 export CARGO_HOME = "${WORKDIR}/cargo_home"
 
-def cargo_base_dep(d):
+# We need cargo to compile for the target
-    deps = ""
+BASEDEPENDS_append = " cargo-native"
-    if not d.getVar('INHIBIT_DEFAULT_DEPS', True) and not d.getVar('INHIBIT_CARGO_DEP', True):
-        deps += " cargo-native"
-    return deps
 
-BASEDEPENDS_append = " ${@cargo_base_dep(d)}"
+# Ensure we get the right rust variant
+DEPENDS_append_class-target = " virtual/${TARGET_PREFIX}rust ${RUSTLIB_DEP}"
+DEPENDS_append_class-native = " rust-native"
 
 # Cargo only supports in-tree builds at the moment
 B = "${S}"
 
-
 # In case something fails in the build process, give a bit more feedback on
 # where the issue occured
 export RUST_BACKTRACE = "1"
@@ -24,22 +27,26 @@ export RUST_BACKTRACE = "1"
 # for cross compilation, so tell it we know better than it.
 export PKG_CONFIG_ALLOW_CROSS = "1"
 
-EXTRA_OECARGO_PATHS ??= ""
-
 cargo_do_configure () {
-	# FIXME: we currently make a mess in the directory above us
+	mkdir -p ${CARGO_HOME}
-	# (${WORKDIR}), which may not be ideal. Look into whether this is
-	# allowed
-	mkdir -p ../.cargo
 	# NOTE: we cannot pass more flags via this interface, the 'linker' is
 	# assumed to be a path to a binary. If flags are needed, a wrapper must
 	# be used.
-	echo "paths = [" >../.cargo/config
+	echo "paths = [" > ${CARGO_HOME}/config
 
 	for p in ${EXTRA_OECARGO_PATHS}; do
 		printf "\"%s\"\n" "$p"
-	done | sed -e 's/$/,/' >>../.cargo/config
+	done | sed -e 's/$/,/' >> ${CARGO_HOME}/config
-	echo "]" >>../.cargo/config
+	echo "]" >> ${CARGO_HOME}/config
+
+	# Point cargo at our local mirror of the registry
+	cat >> ${CARGO_HOME}/config <<EOF
+[source.local]
+local-registry = "${WORKDIR}/cargo_registry"
+[source.crates-io]
+replace-with = "local"
+registry = "https://github.com/rust-lang/crates.io-index"
+EOF
 }
 
 # All the rust & cargo ecosystem assume that CC, LD, etc are a path to a single
@@ -52,6 +59,7 @@ export RUST_CFLAGS = "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${CFLAGS}"
 export RUST_BUILD_CC = "${CCACHE}${BUILD_PREFIX}gcc"
 export RUST_BUILD_CFLAGS = "${BUILD_CC_ARCH} ${BUILD_CFLAGS}"
 
+RUSTFLAGS ??= ""
 export CARGO_BUILD_FLAGS = "-v --target ${HOST_SYS} --release"
 
 # This is based on the content of CARGO_BUILD_FLAGS and generally will need to
@@ -59,9 +67,9 @@ export CARGO_BUILD_FLAGS = "-v --target ${HOST_SYS} --release"
 export CARGO_TARGET_SUBDIR="${HOST_SYS}/release"
 oe_cargo_build () {
 	export RUSTFLAGS="${RUSTFLAGS}"
-	which cargo
+	bbnote "cargo = $(which cargo)"
-	which rustc
+	bbnote "rustc = $(which rustc)"
-	bbnote ${CARGO} build ${CARGO_BUILD_FLAGS} "$@"
+	bbnote "${CARGO} build ${CARGO_BUILD_FLAGS} $@"
 	"${CARGO}" build ${CARGO_BUILD_FLAGS} "$@"
 }
 
@@ -77,18 +85,26 @@ oe_cargo_fix_env () {
 	export HOST_AR="${BUILD_AR}"
 }
 
+EXTRA_OECARGO_PATHS ??= ""
+
 cargo_do_compile () {
-	cd "${B}"
+	# prevent cargo from trying to fetch down new data
+	mkdir -p "${WORKDIR}/cargo_home/registry/index/github.com-1ecc6299db9ec823"
+	touch "${WORKDIR}/cargo_home/registry/index/github.com-1ecc6299db9ec823/.cargo-index-lock"
+
 	oe_cargo_fix_env
 	oe_cargo_build
 }
 
-# All but the most simple projects will need to override this.
 cargo_do_install () {
 	local have_installed=false
-	install -d "${D}${bindir}"
 	for tgt in "${B}/target/${CARGO_TARGET_SUBDIR}/"*; do
-		if [ -f "$tgt" ] && [ -x "$tgt" ]; then
+		if [[ $tgt == *.so || $tgt == *.rlib ]]; then
+			install -d "${D}${rustlibdir}"
+			install -m755 "$tgt" "${D}${rustlibdir}"
+			have_installed=true
+		elif [ -f "$tgt" ] && [ -x "$tgt" ]; then
+			install -d "${D}${bindir}"
 			install -m755 "$tgt" "${D}${bindir}"
 			have_installed=true
 		fi
@@ -98,4 +114,4 @@ cargo_do_install () {
 	fi
 }
 
-EXPORT_FUNCTIONS do_compile do_install do_configure
+EXPORT_FUNCTIONS do_configure do_compile do_install

classes/cargo_util.bbclass

@@ -1,94 +0,0 @@
-inherit rust-vars
-# add crate fetch support
-inherit crate-fetch
-
-# the binary we will use
-CARGO = "cargo"
-
-# Where we download our registry and dependencies to
-export CARGO_HOME = "${WORKDIR}/cargo_home"
-
-# We need cargo to compile for the target
-BASEDEPENDS_append = " cargo-native"
-
-# Ensure we get the right rust variant
-DEPENDS_append_class-target = " virtual/${TARGET_PREFIX}rust ${RUSTLIB_DEP}"
-
-# Cargo only supports in-tree builds at the moment
-B = "${S}"
-
-# In case something fails in the build process, give a bit more feedback on
-# where the issue occured
-export RUST_BACKTRACE = "1"
-
-# The pkg-config-rs library used by cargo build scripts disables itself when
-# cross compiling unless this is defined. We set up pkg-config appropriately
-# for cross compilation, so tell it we know better than it.
-export PKG_CONFIG_ALLOW_CROSS = "1"
-
-# All the rust & cargo ecosystem assume that CC, LD, etc are a path to a single
-# command. Fixup the ones we give it so that is the case.
-# XXX: this is hard coded based on meta/conf/bitbake.conf
-# TODO: we do quite a bit very similar to this in rust.inc, see if it can be
-# generalized.
-export RUST_CC = "${CCACHE}${HOST_PREFIX}gcc"
-export RUST_CFLAGS = "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${CFLAGS}"
-export RUST_BUILD_CC = "${CCACHE}${BUILD_PREFIX}gcc"
-export RUST_BUILD_CFLAGS = "${BUILD_CC_ARCH} ${BUILD_CFLAGS}"
-
-RUSTFLAGS ??= ""
-export CARGO_BUILD_FLAGS = "-v --target ${HOST_SYS} --release"
-
-# This is based on the content of CARGO_BUILD_FLAGS and generally will need to
-# change if CARGO_BUILD_FLAGS changes.
-export CARGO_TARGET_SUBDIR="${HOST_SYS}/release"
-oe_cargo_build () {
-	export RUSTFLAGS="${RUSTFLAGS}"
-	bbnote "cargo = $(which cargo)"
-	bbnote "rustc = $(which rustc)"
-	bbnote "${CARGO} build ${CARGO_BUILD_FLAGS} $@"
-	"${CARGO}" build ${CARGO_BUILD_FLAGS} "$@"
-}
-
-oe_cargo_fix_env () {
-	export CC="${RUST_CC}"
-	export CFLAGS="${RUST_CFLAGS}"
-	export AR="${AR}"
-	export TARGET_CC="${RUST_CC}"
-	export TARGET_CFLAGS="${RUST_CFLAGS}"
-	export TARGET_AR="${AR}"
-	export HOST_CC="${RUST_BUILD_CC}"
-	export HOST_CFLAGS="${RUST_BUILD_CFLAGS}"
-	export HOST_AR="${BUILD_AR}"
-}
-
-cargo_util_do_compile () {
-	cd "${B}"
-
-	# prevent cargo from trying to fetch down new data
-	mkdir -p "${WORKDIR}/cargo_home/registry/index/"
-	touch "${WORKDIR}/cargo_home/registry/index/.cargo-index-lock"
-
-	oe_cargo_fix_env
-	oe_cargo_build
-}
-
-cargo_util_do_install () {
-	local have_installed=false
-	for tgt in "${B}/target/${CARGO_TARGET_SUBDIR}/"*; do
-		if [[ $tgt == *.so || $tgt == *.rlib ]]; then
-			install -d "${D}${rustlibdir}"
-			install -m755 "$tgt" "${D}${rustlibdir}"
-			have_installed=true
-		elif [ -f "$tgt" ] && [ -x "$tgt" ]; then
-			install -d "${D}${bindir}"
-			install -m755 "$tgt" "${D}${bindir}"
-			have_installed=true
-		fi
-	done
-	if ! $have_installed; then
-		die "Did not find anything to install"
-	fi
-}
-
-EXPORT_FUNCTIONS do_compile do_install

lib/crate.py

@@ -35,7 +35,8 @@ class Crate(Wget):
 
     def _cargo_path(self, rootdir, component):
         # TODO: make this less brittle
-        repo = "github.com-88ac128001ac3a9a"
+        # This can go away entirely once we can build a cargo that supports source-replacement
+        repo = "github.com-1ecc6299db9ec823"
         return os.path.join(rootdir, "cargo_home", "registry", component, repo)
 
     def _cargo_src_path(self, rootdir):
@@ -47,6 +48,9 @@ class Crate(Wget):
     def _cargo_cache_path(self, rootdir):
         return self._cargo_path(rootdir, "cache")
 
+    def _cargo_registry_path(self, rootdir, component=""):
+        return os.path.join(rootdir, "cargo_registry", component)
+
     def supports(self, ud, d):
         """
         Check to see if a given url is for this fetcher
@@ -132,13 +136,18 @@ class Crate(Wget):
             super(Crate, self).unpack(ud, rootdir, d)
 
     def _index_unpack(self, ud, rootdir, d):
+        cargo_index = self._cargo_index_path(rootdir)
+        self._index_unpack_to(ud, rootdir, d, cargo_index)
+
+        cargo_registry_index = self._cargo_registry_path(rootdir, "index")
+        self._index_unpack_to(ud, rootdir, d, cargo_registry_index)
+
+    def _index_unpack_to(self, ud, rootdir, d, cargo_index):
         """
         Unpacks the index
         """
         thefile = ud.localpath
 
-        cargo_index = self._cargo_index_path(rootdir)
-
         cmd = "tar -xz --no-same-owner --strip-components 1 -f %s -C %s" % (thefile, cargo_index)
 
         # change to the rootdir to unpack but save the old working dir
@@ -177,16 +186,21 @@ class Crate(Wget):
         else:
             cargo_src = self._cargo_src_path(rootdir)
             cargo_cache = self._cargo_cache_path(rootdir)
+            cargo_registry = self._cargo_registry_path(rootdir)
 
             cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_src)
 
             # ensure we've got these paths made
             bb.utils.mkdirhier(cargo_cache)
+            bb.utils.mkdirhier(cargo_registry)
             bb.utils.mkdirhier(cargo_src)
 
             bb.note("Copying %s to %s/" % (thefile, cargo_cache))
             shutil.copy(thefile, cargo_cache)
 
+            bb.note("Copying %s to %s/" % (thefile, cargo_registry))
+            shutil.copy(thefile, cargo_registry)
+
         # path it
         path = d.getVar('PATH', True)
         if path:

recipes-devtools/cargo/cargo-snapshot.inc

@@ -1,4 +1,4 @@
 
-CARGO_SNAPSHOT = "2016-01-31/cargo-nightly-x86_64-unknown-linux-gnu.tar.gz"
+CARGO_SNAPSHOT = "2016-09-01/cargo-nightly-x86_64-unknown-linux-gnu.tar.gz;downloadfilename=cargo-nightly-x86_64-unknown-linux-gnu-2016-09-01.tar.gz"
-SRC_URI[md5sum] = "52f48780b7cfadc88813766048d4d402"
+SRC_URI[md5sum] = "d41ebf79290a7c9c9e5df87cb27e5091"
-SRC_URI[sha256sum] = "1920e661bab536eba763ff6704a1d62fb20bb0f67d8c5a119e41c49510ea5fa6"
+SRC_URI[sha256sum] = "365e5cad79512d244b8ced32f8e5b86a710fc6c17f0d0f5f744b8058ef6dc756"

recipes-devtools/cargo/cargo.inc

@@ -1,5 +1,3 @@
-INHIBIT_CARGO_DEP = "1"
-
 inherit cargo
 inherit patch
 inherit rust-installer
@@ -11,8 +9,60 @@ LICENSE = "MIT | Apache-2.0"
 
 DEPENDS = "openssl zlib libgit2 curl ca-certificates libssh2"
 
+CARGO_INDEX_COMMIT = "6127fc24b0b6fe73fe4d339817fbf000b9a798a2"
+
 SRC_URI = "\
 	http://static-rust-lang-org.s3.amazonaws.com/cargo-dist/${CARGO_SNAPSHOT} \
+	crate://crates.io/advapi32-sys/0.1.2 \
+	crate://crates.io/bufstream/0.1.1 \
+	crate://crates.io/crossbeam/0.2.8 \
+	crate://crates.io/docopt/0.6.78 \
+	crate://crates.io/env_logger/0.3.2 \
+	crate://crates.io/filetime/0.1.10 \
+	crate://crates.io/flate2/0.2.13 \
+	crate://crates.io/fs2/0.2.3 \
+	crate://crates.io/glob/0.2.11 \
+	crate://crates.io/hamcrest/0.1.0 \
+	crate://crates.io/kernel32-sys/0.2.1 \
+	crate://crates.io/libc/0.2.8 \
+	crate://crates.io/log/0.3.5 \
+	crate://crates.io/num_cpus/0.2.11 \
+	crate://crates.io/regex/0.1.58 \
+	crate://crates.io/rustc-serialize/0.3.18 \
+	crate://crates.io/tar/0.4.5 \
+	crate://crates.io/tempdir/0.3.4 \
+	crate://crates.io/term/0.4.4 \
+	crate://crates.io/toml/0.1.28 \
+	crate://crates.io/url/1.1.0 \
+	crate://crates.io/winapi/0.2.6 \
+	crate://crates.io/semver/0.2.3 \
+	crate://crates.io/regex-syntax/0.3.0 \
+	crate://crates.io/utf8-ranges/0.1.3 \
+	crate://crates.io/gcc/0.3.26 \
+	crate://crates.io/unicode-normalization/0.1.2 \
+	crate://crates.io/libz-sys/1.0.2 \
+	crate://crates.io/rand/0.3.14 \
+	crate://crates.io/user32-sys/0.1.2 \
+	crate://crates.io/idna/0.1.0 \
+	crate://crates.io/strsim/0.3.0 \
+	crate://crates.io/matches/0.1.2 \
+	crate://crates.io/cmake/0.1.16 \
+	crate://crates.io/gdi32-sys/0.1.1 \
+	crate://crates.io/bitflags/0.1.1 \
+	crate://crates.io/unicode-bidi/0.2.3 \
+	crate://crates.io/pkg-config/0.3.8 \
+	crate://crates.io/winapi-build/0.1.1 \
+	crate://crates.io/memchr/0.1.10 \
+	crate://crates.io/pnacl-build-helper/1.4.10 \
+	crate://crates.io/nom/1.2.2 \
+	crate://crates.io/num/0.1.31 \
+	crate://crates.io/uuid/0.2.3 \
+	crate://crates.io/aho-corasick/0.5.1 \
+	crate://crates.io/libressl-pnacl-sys/2.1.6 \
+	crate://crates.io/miniz-sys/0.1.7 \
+	crate://crates.io/openssl-sys/0.7.8 \
+	crate://crates.io/url/0.5.10 \
+	crate-index://crates.io/${CARGO_INDEX_COMMIT} \
 "
 
 B = "${S}"
@@ -44,8 +94,6 @@ do_configure () {
 		${EXTRA_OECONF} \
 		|| die "Could not configure cargo"
 
-	# cargo downloads a cargo snapshot to build itself using cargo, we need
-	# to override it's arch info.
 	cargo_do_configure
 }
 
@@ -54,7 +102,7 @@ do_compile () {
 
 	rm -rf target/snapshot
 	mkdir -p target
-	cp -R ${WORKDIR}/$(basename ${CARGO_SNAPSHOT} .tar.gz)/cargo target/snapshot
+	cp -R ${WORKDIR}/cargo-nightly-x86_64-unknown-linux-gnu/cargo target/snapshot
 
 	oe_runmake ARGS="--verbose"
 }

recipes-devtools/cargo/cargo_0.11.0.bb

@@ -4,6 +4,7 @@ require cargo.inc
 SRC_URI += " \
 	git://github.com/rust-lang/cargo.git;protocol=https;name=cargo \
 	file://0001-disable-cargo-snapshot-fetch.patch \
+	file://0001-Never-update-the-registry-index.patch \
 	git://github.com/rust-lang/rust-installer.git;protocol=https;name=rust-installer;destsuffix=${S}/src/rust-installer \
 "
 # Compatible with Rust 1.10.0

recipes-devtools/cargo/files/0001-Never-update-the-registry-index.patch

@@ -0,0 +1,88 @@
+From 27df8ab04275dfd715d1756fc517bb0323f1b210 Mon Sep 17 00:00:00 2001
+From: Steven Walter <stevenrwalter@gmail.com>
+Date: Fri, 2 Sep 2016 14:02:03 -0400
+Subject: [PATCH] Never update the registry index
+
+Bitbake will fetch the index for us so that we needn't do network IO
+during a build
+---
+ src/cargo/ops/cargo_run.rs    |  2 +-
+ src/cargo/sources/registry.rs | 34 +---------------------------------
+ 2 files changed, 2 insertions(+), 34 deletions(-)
+
+diff --git a/src/cargo/ops/cargo_run.rs b/src/cargo/ops/cargo_run.rs
+index 6764118..379b98b 100644
+--- a/src/cargo/ops/cargo_run.rs
++++ b/src/cargo/ops/cargo_run.rs
+@@ -1,7 +1,7 @@
+ use std::path::Path;
+ 
+ use ops::{self, CompileFilter};
+-use util::{self, CargoResult, process, ProcessError};
++use util::{self, CargoResult, ProcessError};
+ use core::Package;
+ 
+ pub fn run(manifest_path: &Path,
+diff --git a/src/cargo/sources/registry.rs b/src/cargo/sources/registry.rs
+index 614d654..a0c7fe6 100644
+--- a/src/cargo/sources/registry.rs
++++ b/src/cargo/sources/registry.rs
+@@ -166,7 +166,6 @@ use std::path::{PathBuf, Path};
+ 
+ use curl::http;
+ use flate2::read::GzDecoder;
+-use git2;
+ use rustc_serialize::hex::ToHex;
+ use rustc_serialize::json;
+ use tar::Archive;
+@@ -174,7 +173,7 @@ use url::Url;
+ 
+ use core::{Source, SourceId, PackageId, Package, Summary, Registry};
+ use core::dependency::{Dependency, DependencyInner, Kind};
+-use sources::{PathSource, git};
++use sources::PathSource;
+ use util::{CargoResult, Config, internal, ChainError, ToUrl, human};
+ use util::{hex, Sha256, paths, Filesystem, FileLock};
+ use ops;
+@@ -464,38 +463,7 @@ impl<'cfg> RegistrySource<'cfg> {
+ 
+     /// Actually perform network operations to update the registry
+     fn do_update(&mut self) -> CargoResult<()> {
+-        if self.updated {
+-            return Ok(())
+-        }
+-        try!(self.checkout_path.create_dir());
+-        let lock = try!(self.checkout_path.open_rw(Path::new(INDEX_LOCK),
+-                                                   self.config,
+-                                                   "the registry index"));
+-        let path = lock.path().parent().unwrap();
+-
+-        try!(self.config.shell().status("Updating",
+-             format!("registry `{}`", self.source_id.url())));
+-        let repo = try!(git2::Repository::open(path).or_else(|_| {
+-            let _ = lock.remove_siblings();
+-            git2::Repository::init(path)
+-        }));
+-
+-        // git fetch origin
+-        let url = self.source_id.url().to_string();
+-        let refspec = "refs/heads/*:refs/remotes/origin/*";
+-
+-        try!(git::fetch(&repo, &url, refspec, &self.config).chain_error(|| {
+-            internal(format!("failed to fetch `{}`", url))
+-        }));
+-
+-        // git reset --hard origin/master
+-        let reference = "refs/remotes/origin/master";
+-        let oid = try!(repo.refname_to_id(reference));
+-        trace!("[{}] updating to rev {}", self.source_id, oid);
+-        let object = try!(repo.find_object(oid, None));
+-        try!(repo.reset(&object, git2::ResetType::Hard, None));
+         self.updated = true;
+-        self.cache.clear();
+         Ok(())
+     }
+ }
+-- 
+2.7.4
+

recipes-devtools/rust/libstd-rs.bb

@@ -6,22 +6,27 @@ LICENSE = "MIT | Apache-2.0"
 LIC_FILES_CHKSUM ="file://COPYRIGHT;md5=43e1f1fb9c0ee3af66693d8c4fecafa8"
 require rust-shared-source.inc
 
+CARGO_INDEX_COMMIT = "6127fc24b0b6fe73fe4d339817fbf000b9a798a2"
+
+SRC_URI += "\
+	crate://crates.io/gcc/0.3.26 \
+	crate-index://crates.io/${CARGO_INDEX_COMMIT} \
+"
+
 DEPENDS += "compiler-rt"
 
 RUSTLIB_DEP = ""
-inherit cargo_util
+inherit cargo
 
 # Needed so cargo can find libbacktrace
 RUSTFLAGS += "-L ${STAGING_LIBDIR}"
 
 B = "${WORKDIR}/build"
 
-do_compile () {
+do_compile_prepend () {
     cd ${S}/src/rustc/std_shim
     export CARGO_TARGET_DIR="${B}"
     export RUSTC_BOOTSTRAP_KEY="e8edd0fd"
-    oe_cargo_fix_env
-    oe_cargo_build
 }
 
 do_install () {

recipes-example/rust-hello-world/rust-hello-world_git.bb

@@ -1,4 +1,4 @@
-inherit cargo_util
+inherit cargo
 
 SRC_URI = "git://github.com/jmesmon/rust-hello-world.git;protocol=https"
 SRCREV="e0fa23f1a3cb1eb1407165bd2fc36d2f6e6ad728"

recipes-example/rustfmt/rustfmt_0.4.0.bb

@@ -1,4 +1,4 @@
-inherit cargo_util
+inherit cargo
 
 CARGO_INDEX_COMMIT = "3b3994e099281c394a6a66604d1af6c0920e4c31"