diff --git a/api/api_test.go b/api/api_test.go index f9f3b7b1..afb5e65b 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "sort" "strings" "testing" @@ -41,6 +42,17 @@ func createTestConfig() *os.File { jsonString, err := json.Marshal(gin.H{ "architectures": []string{}, "enableMetricsEndpoint": true, + "S3PublishEndpoints": map[string]map[string]string{ + "test-s3": { + "region": "us-east-1", + "bucket": "bucket-s3", + }, + }, + "GcsPublishEndpoints": map[string]map[string]string{ + "test-gcs": { + "bucket": "bucket-gcs", + }, + }, }) if err != nil { return nil @@ -173,3 +185,27 @@ func (s *APISuite) TestTruthy(c *C) { c.Check(truthy(-1), Equals, true) c.Check(truthy(gin.H{}), Equals, true) } + +func (s *APISuite) TestGetS3Endpoints(c *C) { + response, err := s.HTTPRequest("GET", "/api/s3", nil) + c.Assert(err, IsNil) + c.Check(response.Code, Equals, 200) + + var endpoints []string + err = json.Unmarshal(response.Body.Bytes(), &endpoints) + c.Assert(err, IsNil) + sort.Strings(endpoints) + c.Check(endpoints, DeepEquals, []string{"test-s3"}) +} + +func (s *APISuite) TestGetGCSEndpoints(c *C) { + response, err := s.HTTPRequest("GET", "/api/gcs", nil) + c.Assert(err, IsNil) + c.Check(response.Code, Equals, 200) + + var endpoints []string + err = json.Unmarshal(response.Body.Bytes(), &endpoints) + c.Assert(err, IsNil) + sort.Strings(endpoints) + c.Check(endpoints, DeepEquals, []string{"test-gcs"}) +} diff --git a/debian/aptly.conf b/debian/aptly.conf index 54360870..a2c7cf24 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -256,6 +256,54 @@ s3_publish_endpoints: # # Enables detailed request/response dump for each S3 operation # debug: false +# GCS Endpoint Support +# +# aptly can be configured to publish repositories directly to Google Cloud +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has a name and associated settings. +# +# In order to publish to GCS, specify endpoint as `gcs:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main gcs:test:` +# +gcs_publish_endpoints: + # # Endpoint Name + # test: + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Credentials File (optional) + # # Path to a service account credentials JSON file + # credentials_file: "" + # # Service Account JSON (optional) + # # Inline service account credentials JSON payload + # service_account_json: "" + # # Project (optional) + # # Quota project used for GCS requests + # project: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Storage Class (optional) + # # GCS storage class, e.g. `STANDARD` + # storage_class: STANDARD + # # Encryption Key (optional) + # # Customer-supplied encryption key (32-byte AES-256 key) + # encryption_key: "" + # # Disable MultiDel (optional) + # # Kept for parity with S3 settings; GCS deletes are one-by-one + # disable_multidel: false + # # Debug (optional) + # # Enables detailed logs for each GCS operation + # debug: false + # Swift Endpoint Support # # aptly can publish a repository directly to OpenStack Swift. diff --git a/gcs/gcs_test.go b/gcs/gcs_test.go new file mode 100644 index 00000000..b31ecfbe --- /dev/null +++ b/gcs/gcs_test.go @@ -0,0 +1,12 @@ +package gcs + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +// Launch gocheck tests. +func Test(t *testing.T) { + TestingT(t) +} diff --git a/gcs/public_test.go b/gcs/public_test.go new file mode 100644 index 00000000..4cf3d83f --- /dev/null +++ b/gcs/public_test.go @@ -0,0 +1,75 @@ +package gcs + +import ( + "path/filepath" + + "github.com/aptly-dev/aptly/utils" + . "gopkg.in/check.v1" +) + +type PublishedStorageSuite struct{} + +var _ = Suite(&PublishedStorageSuite{}) + +func (s *PublishedStorageSuite) TestString(c *C) { + storage := &PublishedStorage{bucketName: "bucket-1", prefix: "prefix/a"} + c.Check(storage.String(), Equals, "GCS: bucket-1/prefix/a") +} + +func (s *PublishedStorageSuite) TestObjectPath(c *C) { + storage := &PublishedStorage{prefix: "root"} + c.Check(storage.objectPath("dists/stable/Release"), Equals, filepath.Join("root", "dists/stable/Release")) +} + +func (s *PublishedStorageSuite) TestApplyACLNoOpModes(c *C) { + for _, acl := range []string{"", "none", "private"} { + storage := &PublishedStorage{acl: acl} + err := storage.applyACL(nil) + c.Check(err, IsNil) + } +} + +func (s *PublishedStorageSuite) TestApplyACLUnsupported(c *C) { + storage := &PublishedStorage{acl: "bucket-owner-full-control"} + err := storage.applyACL(nil) + c.Assert(err, NotNil) + c.Check(err, ErrorMatches, "unsupported GCS ACL value: bucket-owner-full-control") +} + +func (s *PublishedStorageSuite) TestLinkFromPoolMissingMD5(c *C) { + publishedPrefix := "repo" + publishedRelPath := "pool/main/a/aptly" + fileName := "pkg.deb" + relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName) + + storage := &PublishedStorage{pathCache: map[string]string{relPath: "0123456789abcdef0123456789abcdef"}} + + err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{}, false) + c.Assert(err, NotNil) + c.Check(err, ErrorMatches, "unable to compare object, MD5 checksum missing") +} + +func (s *PublishedStorageSuite) TestLinkFromPoolDifferentMD5NoForce(c *C) { + publishedPrefix := "repo" + publishedRelPath := "pool/main/a/aptly" + fileName := "pkg.deb" + relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName) + + storage := &PublishedStorage{pathCache: map[string]string{relPath: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}} + + err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{MD5: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, false) + c.Assert(err, NotNil) + c.Check(err, ErrorMatches, ".*file already exists and is different.*") +} + +func (s *PublishedStorageSuite) TestLinkFromPoolSameMD5NoUpload(c *C) { + publishedPrefix := "repo" + publishedRelPath := "pool/main/a/aptly" + fileName := "pkg.deb" + relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName) + + storage := &PublishedStorage{pathCache: map[string]string{relPath: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}} + + err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{MD5: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, false) + c.Check(err, IsNil) +} diff --git a/man/aptly.1 b/man/aptly.1 index 8939dc5f..67351d7a 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -307,6 +307,66 @@ The legacy json configuration is still supported (and also supports comments): // } }, + // GCS Endpoint Support + // + // aptly can be configured to publish repositories directly to Google Cloud + // Storage\. First, publishing endpoints should be described in the aptly + // configuration file\. Each endpoint has a name and associated settings\. + // + // In order to publish to GCS, specify endpoint as `gcs:endpoint\-name:` before + // publishing prefix on the command line, e\.g\.: + // + // `aptly publish snapshot wheezy\-main gcs:test:` + // + "GcsPublishEndpoints": { + // // Endpoint Name + // "test": { + + // // Bucket name + // "bucket": "test\-bucket", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Credentials file (optional) + // // Path to a service account credentials JSON file\. If omitted, + // // Application Default Credentials are used\. + // "credentialsFile": "", + + // // Service Account JSON (optional) + // // Inline service account credentials JSON content\. + // "serviceAccountJSON": "", + + // // Project (optional) + // // Quota project to bill requests to\. + // "project": "", + + // // Default ACLs (optional) + // // assign ACL to published files\. Useful values: `private` (default), + // // `public\-read` (public repository), or `none` (don't set ACL)\. + // "acl": "private", + + // // Storage Class (optional) + // // GCS storage class, e\.g\. `STANDARD`\. + // "storageClass": "STANDARD", + + // // Encryption Key (optional) + // // Customer-supplied encryption key (32-byte AES-256 key) for object operations\. + // "encryptionKey": "", + + // // Disable MultiDel (optional) + // // Kept for parity with S3 settings\. + // // GCS deletes are currently performed one object at a time\. + // "disableMultiDel": false, + + // // Debug (optional) + // // Enables detailed GCS operation logs + // "debug": false + // } + }, + // Swift Endpoint Support // // aptly could be configured to publish repository directly to OpenStack Swift\. First, diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 708fdbb6..04364310 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -296,6 +296,66 @@ The legacy json configuration is still supported (and also supports comments): // } }, + // GCS Endpoint Support + // + // aptly can be configured to publish repositories directly to Google Cloud + // Storage. First, publishing endpoints should be described in the aptly + // configuration file. Each endpoint has a name and associated settings. + // + // In order to publish to GCS, specify endpoint as `gcs:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main gcs:test:` + // + "GcsPublishEndpoints": { + // // Endpoint Name + // "test": { + + // // Bucket name + // "bucket": "test-bucket", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Credentials file (optional) + // // Path to a service account credentials JSON file. If omitted, + // // Application Default Credentials are used. + // "credentialsFile": "", + + // // Service Account JSON (optional) + // // Inline service account credentials JSON content. + // "serviceAccountJSON": "", + + // // Project (optional) + // // Quota project to bill requests to. + // "project": "", + + // // Default ACLs (optional) + // // assign ACL to published files. Useful values: `private` (default), + // // `public-read` (public repository), or `none` (don't set ACL). + // "acl": "private", + + // // Storage Class (optional) + // // GCS storage class, e.g. `STANDARD`. + // "storageClass": "STANDARD", + + // // Encryption Key (optional) + // // Customer-supplied encryption key (32-byte AES-256 key) for object operations. + // "encryptionKey": "", + + // // Disable MultiDel (optional) + // // Kept for parity with S3 settings. + // // GCS deletes are currently performed one object at a time. + // "disableMultiDel": false, + + // // Debug (optional) + // // Enables detailed GCS operation logs + // "debug": false + // } + }, + // Swift Endpoint Support // // aptly could be configured to publish repository directly to OpenStack Swift. First, diff --git a/system/gcs_lib.py b/system/gcs_lib.py new file mode 100644 index 00000000..9cc4c5c0 --- /dev/null +++ b/system/gcs_lib.py @@ -0,0 +1,101 @@ +from lib import BaseTest +import os +import uuid + +try: + from google.cloud import storage + + gcs_project = os.environ.get('GCS_PROJECT') + + if gcs_project: + gcs_client = storage.Client(project=gcs_project) + else: + print('GCS tests disabled: GCS_PROJECT is not set') + gcs_client = None +except ImportError as e: + print("GCS tests disabled: can't import google.cloud.storage", e) + gcs_client = None +except Exception as e: + print('GCS tests disabled: unable to initialize GCS client', e) + gcs_client = None + + +class GCSTest(BaseTest): + """ + BaseTest + support for GCS + """ + + gcsOverrides = {} + + def __init__(self) -> None: + super(GCSTest, self).__init__() + self.bucket_name = None + self.bucket = None + self.bucket_contents = None + + def fixture_available(self): + return super(GCSTest, self).fixture_available() and gcs_client is not None + + def prepare(self): + # GCS bucket names must be globally unique and lower-case. + self.bucket_name = 'aptly-sys-test-' + str(uuid.uuid4()).replace('_', '-').lower() + self.bucket = gcs_client.create_bucket(self.bucket_name) + + self.configOverride = { + 'GcsPublishEndpoints': { + 'test1': { + 'bucket': self.bucket_name, + 'project': gcs_project, + }, + }, + } + + self.configOverride['GcsPublishEndpoints']['test1'].update(**self.gcsOverrides) + + super(GCSTest, self).prepare() + + def shutdown(self): + if self.bucket is not None: + for blob in self.bucket.list_blobs(): + blob.delete() + self.bucket.delete(force=True) + + super(GCSTest, self).shutdown() + + def _normalize_path(self, path): + if path.startswith('public/'): + return path[7:] + return path + + def check_path(self, path): + if self.bucket_contents is None: + self.bucket_contents = [blob.name for blob in self.bucket.list_blobs()] + + path = self._normalize_path(path) + + if path in self.bucket_contents: + return True + + if not path.endswith('/'): + path = path + '/' + + for item in self.bucket_contents: + if item.startswith(path): + return True + + return False + + def check_exists(self, path): + if not self.check_path(path): + raise Exception("path %s doesn't exist" % (path, )) + + def check_not_exists(self, path): + if self.check_path(path): + raise Exception("path %s exists" % (path, )) + + def read_file(self, path, mode=''): + assert not mode + + path = self._normalize_path(path) + blob = self.bucket.blob(path) + return blob.download_as_text() diff --git a/system/requirements.txt b/system/requirements.txt index a00a2a65..40ea29af 100644 --- a/system/requirements.txt +++ b/system/requirements.txt @@ -1,4 +1,5 @@ azure-storage-blob +google-cloud-storage boto requests==2.33.0 requests-unixsocket diff --git a/system/t06_publish/gcs.py b/system/t06_publish/gcs.py new file mode 100644 index 00000000..d60c5553 --- /dev/null +++ b/system/t06_publish/gcs.py @@ -0,0 +1,116 @@ +from gcs_lib import GCSTest + + +def strip_processor(output): + return '\n'.join( + [ + l + for l in output.split('\n') + if not l.startswith(' ') and not l.startswith('Date:') + ] + ) + + +class GCSPublish1Test(GCSTest): + """ + publish to GCS: from repo + """ + + fixtureCmds = [ + 'aptly repo create -distribution=maverick local-repo', + 'aptly repo add local-repo ${files}', + 'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386', + ] + runCmd = 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:' + + def check(self): + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/source/Sources') + self.check_exists('public/dists/maverick/main/source/Sources.gz') + self.check_exists('public/dists/maverick/main/source/Sources.bz2') + + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + ) + + # verify contents except sums/date chunks + self.check_file_contents( + 'public/dists/maverick/Release', 'release', match_prepare=strip_processor + ) + self.check_file_contents( + 'public/dists/maverick/main/source/Sources', + 'sources', + match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))), + ) + self.check_file_contents( + 'public/dists/maverick/main/binary-i386/Packages', + 'binary', + match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))), + ) + + +class GCSPublish2Test(GCSTest): + """ + publish to GCS: update after removing package from repo + """ + + fixtureCmds = [ + 'aptly repo create -distribution=maverick local-repo', + 'aptly repo add local-repo ${files}/', + 'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:', + 'aptly repo remove local-repo pyspi', + ] + runCmd = 'aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick gcs:test1:' + + def check(self): + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + ) + + +class GCSPublish3Test(GCSTest): + """ + publish to GCS: publish drop performs cleanup + """ + + fixtureCmds = [ + 'aptly repo create local1', + 'aptly repo create local2', + 'aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb', + 'aptly repo add local2 ${files}', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 gcs:test1:', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 gcs:test1:', + ] + runCmd = 'aptly publish drop sq2 gcs:test1:' + + def check(self): + self.check_exists('public/dists/sq1') + self.check_not_exists('public/dists/sq2') + self.check_exists('public/pool/main/') + + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + )