mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-07-15 11:57:59 +00:00
feat: add GCS publish backend with config, API, docs, and tests
This commit is contained in:
Pierig Le Saux
committed by
André Roth
André Roth
parent
ed4af9a0f6
commit
d5fbf0f795
@@ -0,0 +1,101 @@
|
||||
from lib import BaseTest
|
||||
import os
|
||||
import uuid
|
||||
|
||||
try:
|
||||
from google.cloud import storage
|
||||
|
||||
gcs_project = os.environ.get('GCS_PROJECT')
|
||||
|
||||
if gcs_project:
|
||||
gcs_client = storage.Client(project=gcs_project)
|
||||
else:
|
||||
print('GCS tests disabled: GCS_PROJECT is not set')
|
||||
gcs_client = None
|
||||
except ImportError as e:
|
||||
print("GCS tests disabled: can't import google.cloud.storage", e)
|
||||
gcs_client = None
|
||||
except Exception as e:
|
||||
print('GCS tests disabled: unable to initialize GCS client', e)
|
||||
gcs_client = None
|
||||
|
||||
|
||||
class GCSTest(BaseTest):
|
||||
"""
|
||||
BaseTest + support for GCS
|
||||
"""
|
||||
|
||||
gcsOverrides = {}
|
||||
|
||||
def __init__(self) -> None:
|
||||
super(GCSTest, self).__init__()
|
||||
self.bucket_name = None
|
||||
self.bucket = None
|
||||
self.bucket_contents = None
|
||||
|
||||
def fixture_available(self):
|
||||
return super(GCSTest, self).fixture_available() and gcs_client is not None
|
||||
|
||||
def prepare(self):
|
||||
# GCS bucket names must be globally unique and lower-case.
|
||||
self.bucket_name = 'aptly-sys-test-' + str(uuid.uuid4()).replace('_', '-').lower()
|
||||
self.bucket = gcs_client.create_bucket(self.bucket_name)
|
||||
|
||||
self.configOverride = {
|
||||
'GcsPublishEndpoints': {
|
||||
'test1': {
|
||||
'bucket': self.bucket_name,
|
||||
'project': gcs_project,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
self.configOverride['GcsPublishEndpoints']['test1'].update(**self.gcsOverrides)
|
||||
|
||||
super(GCSTest, self).prepare()
|
||||
|
||||
def shutdown(self):
|
||||
if self.bucket is not None:
|
||||
for blob in self.bucket.list_blobs():
|
||||
blob.delete()
|
||||
self.bucket.delete(force=True)
|
||||
|
||||
super(GCSTest, self).shutdown()
|
||||
|
||||
def _normalize_path(self, path):
|
||||
if path.startswith('public/'):
|
||||
return path[7:]
|
||||
return path
|
||||
|
||||
def check_path(self, path):
|
||||
if self.bucket_contents is None:
|
||||
self.bucket_contents = [blob.name for blob in self.bucket.list_blobs()]
|
||||
|
||||
path = self._normalize_path(path)
|
||||
|
||||
if path in self.bucket_contents:
|
||||
return True
|
||||
|
||||
if not path.endswith('/'):
|
||||
path = path + '/'
|
||||
|
||||
for item in self.bucket_contents:
|
||||
if item.startswith(path):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def check_exists(self, path):
|
||||
if not self.check_path(path):
|
||||
raise Exception("path %s doesn't exist" % (path, ))
|
||||
|
||||
def check_not_exists(self, path):
|
||||
if self.check_path(path):
|
||||
raise Exception("path %s exists" % (path, ))
|
||||
|
||||
def read_file(self, path, mode=''):
|
||||
assert not mode
|
||||
|
||||
path = self._normalize_path(path)
|
||||
blob = self.bucket.blob(path)
|
||||
return blob.download_as_text()
|
||||
@@ -1,4 +1,5 @@
|
||||
azure-storage-blob
|
||||
google-cloud-storage
|
||||
boto
|
||||
requests==2.33.0
|
||||
requests-unixsocket
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
from gcs_lib import GCSTest
|
||||
|
||||
|
||||
def strip_processor(output):
|
||||
return '\n'.join(
|
||||
[
|
||||
l
|
||||
for l in output.split('\n')
|
||||
if not l.startswith(' ') and not l.startswith('Date:')
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class GCSPublish1Test(GCSTest):
|
||||
"""
|
||||
publish to GCS: from repo
|
||||
"""
|
||||
|
||||
fixtureCmds = [
|
||||
'aptly repo create -distribution=maverick local-repo',
|
||||
'aptly repo add local-repo ${files}',
|
||||
'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
|
||||
]
|
||||
runCmd = 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:'
|
||||
|
||||
def check(self):
|
||||
self.check_exists('public/dists/maverick/InRelease')
|
||||
self.check_exists('public/dists/maverick/Release')
|
||||
self.check_exists('public/dists/maverick/Release.gpg')
|
||||
|
||||
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
|
||||
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
|
||||
self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
|
||||
self.check_exists('public/dists/maverick/main/source/Sources')
|
||||
self.check_exists('public/dists/maverick/main/source/Sources.gz')
|
||||
self.check_exists('public/dists/maverick/main/source/Sources.bz2')
|
||||
|
||||
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
|
||||
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
|
||||
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
|
||||
self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
|
||||
self.check_exists(
|
||||
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
|
||||
)
|
||||
|
||||
# verify contents except sums/date chunks
|
||||
self.check_file_contents(
|
||||
'public/dists/maverick/Release', 'release', match_prepare=strip_processor
|
||||
)
|
||||
self.check_file_contents(
|
||||
'public/dists/maverick/main/source/Sources',
|
||||
'sources',
|
||||
match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
|
||||
)
|
||||
self.check_file_contents(
|
||||
'public/dists/maverick/main/binary-i386/Packages',
|
||||
'binary',
|
||||
match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
|
||||
)
|
||||
|
||||
|
||||
class GCSPublish2Test(GCSTest):
|
||||
"""
|
||||
publish to GCS: update after removing package from repo
|
||||
"""
|
||||
|
||||
fixtureCmds = [
|
||||
'aptly repo create -distribution=maverick local-repo',
|
||||
'aptly repo add local-repo ${files}/',
|
||||
'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
|
||||
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:',
|
||||
'aptly repo remove local-repo pyspi',
|
||||
]
|
||||
runCmd = 'aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick gcs:test1:'
|
||||
|
||||
def check(self):
|
||||
self.check_exists('public/dists/maverick/InRelease')
|
||||
self.check_exists('public/dists/maverick/Release')
|
||||
self.check_exists('public/dists/maverick/Release.gpg')
|
||||
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
|
||||
self.check_exists(
|
||||
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
|
||||
)
|
||||
|
||||
|
||||
class GCSPublish3Test(GCSTest):
|
||||
"""
|
||||
publish to GCS: publish drop performs cleanup
|
||||
"""
|
||||
|
||||
fixtureCmds = [
|
||||
'aptly repo create local1',
|
||||
'aptly repo create local2',
|
||||
'aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb',
|
||||
'aptly repo add local2 ${files}',
|
||||
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 gcs:test1:',
|
||||
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 gcs:test1:',
|
||||
]
|
||||
runCmd = 'aptly publish drop sq2 gcs:test1:'
|
||||
|
||||
def check(self):
|
||||
self.check_exists('public/dists/sq1')
|
||||
self.check_not_exists('public/dists/sq2')
|
||||
self.check_exists('public/pool/main/')
|
||||
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
|
||||
self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
|
||||
self.check_exists(
|
||||
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
|
||||
)
|
||||
Reference in New Issue
Block a user