mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-07 16:59:22 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: fetch2: Revalidate checksums, YOCTO #5571

Browse Source 
[YOCTO #5571] -- https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571

The following workflow (whether accidentally or deliberately) would
previously not result in a checksum error, but would be helpful to do
so:
 - Write a recipe with correct checksums
 - Fetch the sources for this recipe using bitbake
 - Change the checksums
Since the bitbake fetcher writes a done stamp after the initial download
and does not verify the checksums again (even if they are changed in the
recipe afterwards), the change of checksums is silently ignored.

Fix this without the overhead of computing the checksums from scratch on
every do_fetch by storing them in pickled format in the done stamp and
verifying that they still match those in the recipe.

(Bitbake rev: fbd4a0d422cf7f43db2f9eab2e47c41246a9031e)

Signed-off-by: Clemens Lang <clemens.lang@bmw-carit.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Clemens Lang
2015-03-06 15:28:54 +01:00
committed by Richard Purdie
parent 3c641e7238
commit 36e47a99ae
1 changed files with 103 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/fetch2/__init__.py
+103 -11
View File
@@ -45,6 +45,13 @@ _checksum_cache = bb.checksum.FileChecksumCache()
logger = logging.getLogger("BitBake.Fetcher")
try:
import cPickle as pickle
except ImportError:
import pickle
logger.info("Importing cPickle failed. "
"Falling back to a very slow implementation.")
class BBFetchException(Exception):
"""Class all fetch exceptions inherit from"""
def __init__(self, message):
@@ -525,7 +532,7 @@ def fetcher_compare_revisions(d):
def mirror_from_string(data):
return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
def verify_checksum(ud, d):
def verify_checksum(ud, d, precomputed={}):
"""
verify the MD5 and SHA256 checksum for downloaded src
@@ -533,13 +540,28 @@ def verify_checksum(ud, d):
the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no
checksums specified.
Returns a dict of checksums that can be stored in a done stamp file and
passed in as precomputed parameter in a later call to avoid re-computing
the checksums from the file. This allows verifying the checksums of the
file against those in the recipe each time, rather than only after
downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
"""
if ud.ignore_checksums or not ud.method.supports_checksum(ud):
return
_MD5_KEY = "md5"
_SHA256_KEY = "sha256"
md5data = bb.utils.md5_file(ud.localpath)
sha256data = bb.utils.sha256_file(ud.localpath)
if ud.ignore_checksums or not ud.method.supports_checksum(ud):
return {}
if _MD5_KEY in precomputed:
md5data = precomputed[_MD5_KEY]
else:
md5data = bb.utils.md5_file(ud.localpath)
if _SHA256_KEY in precomputed:
sha256data = precomputed[_SHA256_KEY]
else:
sha256data = bb.utils.sha256_file(ud.localpath)
if ud.method.recommends_checksum(ud):
# If strict checking enabled and neither sum defined, raise error
@@ -589,6 +611,72 @@ def verify_checksum(ud, d):
if len(msg):
raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data)
return {
_MD5_KEY: md5data,
_SHA256_KEY: sha256data
}
def verify_donestamp(ud, d):
"""
Check whether the done stamp file has the right checksums (if the fetch
method supports them). If it doesn't, delete the done stamp and force
a re-download.
Returns True, if the donestamp exists and is valid, False otherwise. When
returning False, any existing done stamps are removed.
"""
if not os.path.exists(ud.donestamp):
return False
if not ud.method.supports_checksum(ud):
# done stamp exists, checksums not supported; assume the local file is
# current
return True
if not os.path.exists(ud.localpath):
# done stamp exists, but the downloaded file does not; the done stamp
# must be incorrect, re-trigger the download
bb.utils.remove(ud.donestamp)
return False
precomputed_checksums = {}
# Only re-use the precomputed checksums if the donestamp is newer than the
# file. Do not rely on the mtime of directories, though. If ud.localpath is
# a directory, there will probably not be any checksums anyway.
if (os.path.isdir(ud.localpath) or
os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)):
try:
with open(ud.donestamp, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
precomputed_checksums.update(pickled.load())
except Exception as e:
# Avoid the warnings on the upgrade path from emtpy done stamp
# files to those containing the checksums.
if not isinstance(e, EOFError):
# Ignore errors, they aren't fatal
logger.warn("Couldn't load checksums from donestamp %s: %s "
"(msg: %s)" % (ud.donestamp, type(e).__name__,
str(e)))
try:
checksums = verify_checksum(ud, d, precomputed_checksums)
# If the cache file did not have the checksums, compute and store them
# as an upgrade path from the previous done stamp file format.
if checksums != precomputed_checksums:
with open(ud.donestamp, "wb") as cachefile:
p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
p.dump(checksums)
return True
except ChecksumError as e:
# Checksums failed to verify, trigger re-download and remove the
# incorrect stamp file.
logger.warn("Checksum mismatch for local file %s\n"
"Cleaning and trying again." % ud.localpath)
rename_bad_checksum(ud, e.checksum)
bb.utils.remove(ud.donestamp)
return False
def update_stamp(ud, d):
"""
@@ -603,8 +691,11 @@ def update_stamp(ud, d):
# Errors aren't fatal here
pass
else:
verify_checksum(ud, d)
open(ud.donestamp, 'w').close()
checksums = verify_checksum(ud, d)
# Store the checksums for later re-verification against the recipe
with open(ud.donestamp, "wb") as cachefile:
p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
p.dump(checksums)
def subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
@@ -805,7 +896,7 @@ def try_mirror_url(origud, ud, ld, check = False):
os.chdir(ld.getVar("DL_DIR", True))
if not os.path.exists(ud.donestamp) or ud.method.need_update(ud, ld):
if not verify_donestamp(ud, ld) or ud.method.need_update(ud, ld):
ud.method.download(ud, ld)
if hasattr(ud.method,"build_mirror_data"):
ud.method.build_mirror_data(ud, ld)
@@ -821,12 +912,13 @@ def try_mirror_url(origud, ud, ld, check = False):
dldir = ld.getVar("DL_DIR", True)
if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
# Create donestamp in old format to avoid triggering a re-download
bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
open(ud.donestamp, 'w').close()
dest = os.path.join(dldir, os.path.basename(ud.localpath))
if not os.path.exists(dest):
os.symlink(ud.localpath, dest)
if not os.path.exists(origud.donestamp) or origud.method.need_update(origud, ld):
if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
origud.method.download(origud, ld)
if hasattr(origud.method,"build_mirror_data"):
origud.method.build_mirror_data(origud, ld)
@@ -1422,7 +1514,7 @@ class Fetch(object):
try:
self.d.setVar("BB_NO_NETWORK", network)
if os.path.exists(ud.donestamp) and not m.need_update(ud, self.d):
if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
localpath = ud.localpath
elif m.try_premirror(ud, self.d):
logger.debug(1, "Trying PREMIRRORS")
@@ -1435,7 +1527,7 @@ class Fetch(object):
os.chdir(self.d.getVar("DL_DIR", True))
firsterr = None
if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(ud, self.d)):
if not localpath and ((not verify_donestamp(ud, self.d)) or m.need_update(ud, self.d)):
try:
logger.debug(1, "Trying Upstream")
m.download(ud, self.d)