mirror of
https://git.yoctoproject.org/poky
synced 2026-05-09 17:39:31 +00:00
Stefan Herbrechtsmeier
3e543e8eaa
The fetch data class already expands the type, host, path, user, pswd and parm variables. The fetcher classes already expand the localfile variable. The getVar function expands the returned string per default. Remove unnecessary expand function calls to simplify the code. (Bitbake rev: 1b1eb037b861fbf20491ac17e519e9eaf232b858) Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
103 lines
3.4 KiB
Python
103 lines
3.4 KiB
Python
"""
|
|
BitBake 'Fetch' implementation for Google Cloup Platform Storage.
|
|
|
|
Class for fetching files from Google Cloud Storage using the
|
|
Google Cloud Storage Python Client. The GCS Python Client must
|
|
be correctly installed, configured and authenticated prior to use.
|
|
Additionally, gsutil must also be installed.
|
|
|
|
"""
|
|
|
|
# Copyright (C) 2023, Snap Inc.
|
|
#
|
|
# Based in part on bb.fetch2.s3:
|
|
# Copyright (C) 2017 Andre McCurdy
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
|
|
|
|
import os
|
|
import bb
|
|
import urllib.parse, urllib.error
|
|
from bb.fetch2 import FetchMethod
|
|
from bb.fetch2 import FetchError
|
|
from bb.fetch2 import logger
|
|
|
|
class GCP(FetchMethod):
|
|
"""
|
|
Class to fetch urls via GCP's Python API.
|
|
"""
|
|
def __init__(self):
|
|
self.gcp_client = None
|
|
|
|
def supports(self, ud, d):
|
|
"""
|
|
Check to see if a given url can be fetched with GCP.
|
|
"""
|
|
return ud.type in ['gs']
|
|
|
|
def recommends_checksum(self, urldata):
|
|
return True
|
|
|
|
def urldata_init(self, ud, d):
|
|
if 'downloadfilename' in ud.parm:
|
|
ud.basename = ud.parm['downloadfilename']
|
|
else:
|
|
ud.basename = os.path.basename(ud.path)
|
|
|
|
ud.localfile = urllib.parse.unquote(ud.basename)
|
|
|
|
def get_gcp_client(self):
|
|
from google.cloud import storage
|
|
self.gcp_client = storage.Client(project=None)
|
|
|
|
def download(self, ud, d):
|
|
"""
|
|
Fetch urls using the GCP API.
|
|
Assumes localpath was called first.
|
|
"""
|
|
from google.api_core.exceptions import NotFound
|
|
logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
|
|
if self.gcp_client is None:
|
|
self.get_gcp_client()
|
|
|
|
bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}")
|
|
|
|
# Path sometimes has leading slash, so strip it
|
|
path = ud.path.lstrip("/")
|
|
blob = self.gcp_client.bucket(ud.host).blob(path)
|
|
try:
|
|
blob.download_to_filename(ud.localpath)
|
|
except NotFound:
|
|
raise FetchError("The GCP API threw a NotFound exception")
|
|
|
|
# Additional sanity checks copied from the wget class (although there
|
|
# are no known issues which mean these are required, treat the GCP API
|
|
# tool with a little healthy suspicion).
|
|
if not os.path.exists(ud.localpath):
|
|
raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
|
|
|
|
if os.path.getsize(ud.localpath) == 0:
|
|
os.remove(ud.localpath)
|
|
raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
|
|
|
|
return True
|
|
|
|
def checkstatus(self, fetch, ud, d):
|
|
"""
|
|
Check the status of a URL.
|
|
"""
|
|
logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
|
|
if self.gcp_client is None:
|
|
self.get_gcp_client()
|
|
|
|
bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}")
|
|
|
|
# Path sometimes has leading slash, so strip it
|
|
path = ud.path.lstrip("/")
|
|
if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
|
|
raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
|
|
else:
|
|
return True
|