mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-05-31 12:49:46 +00:00
Code Issues Projects Releases Wiki Activity

bitbake: fetch2: Handle missing donestamp file when content is valid

Browse Source 
In order to allow users to manually populate the download directory with
valid content change the assumption that missing the donestamp file
means unfetched content.

This allows users to populate the download dir without needing to create
dummy .done files such that a user does not need a PREMIRROR when using
BB_NO_NETWORK to provide valid content files in the download directory.

To ensure the correct result this change also fails first if the
localpath does not exist. This prevents further parts of the function
attempting to calculating the checksum on non-existent files. This also
fixes some edge conditions around where if the donestamp exists but the
localpath does not it returns, and did not remove the donestamp.

Also added test cases to cover this use case and additional use cases
where for example the fetcher does not support checksums.

(Bitbake rev: a335dbbb65d5b56e71d98cf3e4fa9bfbec1dcde6)

Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Nathan Rossi
2018-02-02 23:51:15 +10:00
committed by Richard Purdie
parent bb3a634226
commit f2d5967982
2 changed files with 114 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbake/lib/bb/fetch2/__init__.py
+10 -11
View File
@@ -643,26 +643,25 @@ def verify_donestamp(ud, d, origud=None):
if not ud.needdonestamp or (origud and not origud.needdonestamp):
return True
if not os.path.exists(ud.donestamp):
if not os.path.exists(ud.localpath):
# local path does not exist
if os.path.exists(ud.donestamp):
# done stamp exists, but the downloaded file does not; the done stamp
# must be incorrect, re-trigger the download
bb.utils.remove(ud.donestamp)
return False
if (not ud.method.supports_checksum(ud) or
(origud and not origud.method.supports_checksum(origud))):
# done stamp exists, checksums not supported; assume the local file is
# current
return True
if not os.path.exists(ud.localpath):
# done stamp exists, but the downloaded file does not; the done stamp
# must be incorrect, re-trigger the download
bb.utils.remove(ud.donestamp)
return False
# if done stamp exists and checksums not supported; assume the local
# file is current
return os.path.exists(ud.donestamp)
precomputed_checksums = {}
# Only re-use the precomputed checksums if the donestamp is newer than the
# file. Do not rely on the mtime of directories, though. If ud.localpath is
# a directory, there will probably not be any checksums anyway.
if (os.path.isdir(ud.localpath) or
if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or
os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)):
try:
with open(ud.donestamp, "rb") as cachefile:
bitbake/lib/bb/tests/fetch.py
+104
View File
@@ -20,6 +20,7 @@
#
import unittest
import hashlib
import tempfile
import subprocess
import collections
@@ -522,6 +523,109 @@ class FetcherLocalTest(FetcherTest):
with self.assertRaises(bb.fetch2.UnpackError):
self.fetchUnpack(['file://a;subdir=/bin/sh'])
class FetcherNoNetworkTest(FetcherTest):
def setUp(self):
super().setUp()
# all test cases are based on not having network
self.d.setVar("BB_NO_NETWORK", "1")
def test_missing(self):
string = "this is a test file\n".encode("utf-8")
self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
with self.assertRaises(bb.fetch2.NetworkAccess):
fetcher.download()
def test_valid_missing_donestamp(self):
# create the file in the download directory with correct hash
string = "this is a test file\n".encode("utf-8")
with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
f.write(string)
self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
fetcher.download()
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
def test_invalid_missing_donestamp(self):
# create an invalid file in the download directory with incorrect hash
string = "this is a test file\n".encode("utf-8")
with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
pass
self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
with self.assertRaises(bb.fetch2.NetworkAccess):
fetcher.download()
# the existing file should not exist or should have be moved to "bad-checksum"
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
def test_nochecksums_missing(self):
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
# ssh fetch does not support checksums
fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
# attempts to download with missing donestamp
with self.assertRaises(bb.fetch2.NetworkAccess):
fetcher.download()
def test_nochecksums_missing_donestamp(self):
# create a file in the download directory
with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
pass
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
# ssh fetch does not support checksums
fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
# attempts to download with missing donestamp
with self.assertRaises(bb.fetch2.NetworkAccess):
fetcher.download()
def test_nochecksums_has_donestamp(self):
# create a file in the download directory with the donestamp
with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
pass
with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
pass
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
# ssh fetch does not support checksums
fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
# should not fetch
fetcher.download()
# both files should still exist
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
def test_nochecksums_missing_has_donestamp(self):
# create a file in the download directory with the donestamp
with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
pass
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
# ssh fetch does not support checksums
fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
with self.assertRaises(bb.fetch2.NetworkAccess):
fetcher.download()
# both files should still exist
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_fetch(self):