feat: add GCS publish backend with config, API, docs, and tests

This commit is contained in:
Pierig Le Saux
2026-03-27 20:18:16 -04:00
committed by André Roth
parent ed4af9a0f6
commit d5fbf0f795
9 changed files with 509 additions and 0 deletions
+36
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"net/http/httptest"
"os"
"sort"
"strings"
"testing"
@@ -41,6 +42,17 @@ func createTestConfig() *os.File {
jsonString, err := json.Marshal(gin.H{
"architectures": []string{},
"enableMetricsEndpoint": true,
"S3PublishEndpoints": map[string]map[string]string{
"test-s3": {
"region": "us-east-1",
"bucket": "bucket-s3",
},
},
"GcsPublishEndpoints": map[string]map[string]string{
"test-gcs": {
"bucket": "bucket-gcs",
},
},
})
if err != nil {
return nil
@@ -173,3 +185,27 @@ func (s *APISuite) TestTruthy(c *C) {
c.Check(truthy(-1), Equals, true)
c.Check(truthy(gin.H{}), Equals, true)
}
func (s *APISuite) TestGetS3Endpoints(c *C) {
response, err := s.HTTPRequest("GET", "/api/s3", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var endpoints []string
err = json.Unmarshal(response.Body.Bytes(), &endpoints)
c.Assert(err, IsNil)
sort.Strings(endpoints)
c.Check(endpoints, DeepEquals, []string{"test-s3"})
}
func (s *APISuite) TestGetGCSEndpoints(c *C) {
response, err := s.HTTPRequest("GET", "/api/gcs", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var endpoints []string
err = json.Unmarshal(response.Body.Bytes(), &endpoints)
c.Assert(err, IsNil)
sort.Strings(endpoints)
c.Check(endpoints, DeepEquals, []string{"test-gcs"})
}
+48
View File
@@ -256,6 +256,54 @@ s3_publish_endpoints:
# # Enables detailed request/response dump for each S3 operation
# debug: false
# GCS Endpoint Support
#
# aptly can be configured to publish repositories directly to Google Cloud
# Storage. First, publishing endpoints should be described in the aptly
# configuration file. Each endpoint has a name and associated settings.
#
# In order to publish to GCS, specify endpoint as `gcs:endpoint-name:` before
# publishing prefix on the command line, e.g.:
#
# `aptly publish snapshot wheezy-main gcs:test:`
#
gcs_publish_endpoints:
# # Endpoint Name
# test:
# # Bucket name
# bucket: test-bucket
# # Prefix (optional)
# # publishing under specified prefix in the bucket, defaults to
# # no prefix (bucket root)
# prefix: ""
# # Credentials File (optional)
# # Path to a service account credentials JSON file
# credentials_file: ""
# # Service Account JSON (optional)
# # Inline service account credentials JSON payload
# service_account_json: ""
# # Project (optional)
# # Quota project used for GCS requests
# project: ""
# # Default ACLs (optional)
# # assign ACL to published files:
# # * private (default)
# # * public-read (public repository)
# # * none (don't set ACL)
# acl: private
# # Storage Class (optional)
# # GCS storage class, e.g. `STANDARD`
# storage_class: STANDARD
# # Encryption Key (optional)
# # Customer-supplied encryption key (32-byte AES-256 key)
# encryption_key: ""
# # Disable MultiDel (optional)
# # Kept for parity with S3 settings; GCS deletes are one-by-one
# disable_multidel: false
# # Debug (optional)
# # Enables detailed logs for each GCS operation
# debug: false
# Swift Endpoint Support
#
# aptly can publish a repository directly to OpenStack Swift.
+12
View File
@@ -0,0 +1,12 @@
package gcs
import (
"testing"
. "gopkg.in/check.v1"
)
// Launch gocheck tests.
func Test(t *testing.T) {
TestingT(t)
}
+75
View File
@@ -0,0 +1,75 @@
package gcs
import (
"path/filepath"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
type PublishedStorageSuite struct{}
var _ = Suite(&PublishedStorageSuite{})
func (s *PublishedStorageSuite) TestString(c *C) {
storage := &PublishedStorage{bucketName: "bucket-1", prefix: "prefix/a"}
c.Check(storage.String(), Equals, "GCS: bucket-1/prefix/a")
}
func (s *PublishedStorageSuite) TestObjectPath(c *C) {
storage := &PublishedStorage{prefix: "root"}
c.Check(storage.objectPath("dists/stable/Release"), Equals, filepath.Join("root", "dists/stable/Release"))
}
func (s *PublishedStorageSuite) TestApplyACLNoOpModes(c *C) {
for _, acl := range []string{"", "none", "private"} {
storage := &PublishedStorage{acl: acl}
err := storage.applyACL(nil)
c.Check(err, IsNil)
}
}
func (s *PublishedStorageSuite) TestApplyACLUnsupported(c *C) {
storage := &PublishedStorage{acl: "bucket-owner-full-control"}
err := storage.applyACL(nil)
c.Assert(err, NotNil)
c.Check(err, ErrorMatches, "unsupported GCS ACL value: bucket-owner-full-control")
}
func (s *PublishedStorageSuite) TestLinkFromPoolMissingMD5(c *C) {
publishedPrefix := "repo"
publishedRelPath := "pool/main/a/aptly"
fileName := "pkg.deb"
relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName)
storage := &PublishedStorage{pathCache: map[string]string{relPath: "0123456789abcdef0123456789abcdef"}}
err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{}, false)
c.Assert(err, NotNil)
c.Check(err, ErrorMatches, "unable to compare object, MD5 checksum missing")
}
func (s *PublishedStorageSuite) TestLinkFromPoolDifferentMD5NoForce(c *C) {
publishedPrefix := "repo"
publishedRelPath := "pool/main/a/aptly"
fileName := "pkg.deb"
relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName)
storage := &PublishedStorage{pathCache: map[string]string{relPath: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}
err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{MD5: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, false)
c.Assert(err, NotNil)
c.Check(err, ErrorMatches, ".*file already exists and is different.*")
}
func (s *PublishedStorageSuite) TestLinkFromPoolSameMD5NoUpload(c *C) {
publishedPrefix := "repo"
publishedRelPath := "pool/main/a/aptly"
fileName := "pkg.deb"
relPath := filepath.Join(filepath.Join(publishedPrefix, publishedRelPath), fileName)
storage := &PublishedStorage{pathCache: map[string]string{relPath: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}
err := storage.LinkFromPool(publishedPrefix, publishedRelPath, fileName, nil, "", utils.ChecksumInfo{MD5: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, false)
c.Check(err, IsNil)
}
+60
View File
@@ -307,6 +307,66 @@ The legacy json configuration is still supported (and also supports comments):
// }
},
// GCS Endpoint Support
//
// aptly can be configured to publish repositories directly to Google Cloud
// Storage\. First, publishing endpoints should be described in the aptly
// configuration file\. Each endpoint has a name and associated settings\.
//
// In order to publish to GCS, specify endpoint as `gcs:endpoint\-name:` before
// publishing prefix on the command line, e\.g\.:
//
// `aptly publish snapshot wheezy\-main gcs:test:`
//
"GcsPublishEndpoints": {
// // Endpoint Name
// "test": {
// // Bucket name
// "bucket": "test\-bucket",
// // Prefix (optional)
// // publishing under specified prefix in the bucket, defaults to
// // no prefix (bucket root)
// "prefix": "",
// // Credentials file (optional)
// // Path to a service account credentials JSON file\. If omitted,
// // Application Default Credentials are used\.
// "credentialsFile": "",
// // Service Account JSON (optional)
// // Inline service account credentials JSON content\.
// "serviceAccountJSON": "",
// // Project (optional)
// // Quota project to bill requests to\.
// "project": "",
// // Default ACLs (optional)
// // assign ACL to published files\. Useful values: `private` (default),
// // `public\-read` (public repository), or `none` (don't set ACL)\.
// "acl": "private",
// // Storage Class (optional)
// // GCS storage class, e\.g\. `STANDARD`\.
// "storageClass": "STANDARD",
// // Encryption Key (optional)
// // Customer-supplied encryption key (32-byte AES-256 key) for object operations\.
// "encryptionKey": "",
// // Disable MultiDel (optional)
// // Kept for parity with S3 settings\.
// // GCS deletes are currently performed one object at a time\.
// "disableMultiDel": false,
// // Debug (optional)
// // Enables detailed GCS operation logs
// "debug": false
// }
},
// Swift Endpoint Support
//
// aptly could be configured to publish repository directly to OpenStack Swift\. First,
+60
View File
@@ -296,6 +296,66 @@ The legacy json configuration is still supported (and also supports comments):
// }
},
// GCS Endpoint Support
//
// aptly can be configured to publish repositories directly to Google Cloud
// Storage. First, publishing endpoints should be described in the aptly
// configuration file. Each endpoint has a name and associated settings.
//
// In order to publish to GCS, specify endpoint as `gcs:endpoint-name:` before
// publishing prefix on the command line, e.g.:
//
// `aptly publish snapshot wheezy-main gcs:test:`
//
"GcsPublishEndpoints": {
// // Endpoint Name
// "test": {
// // Bucket name
// "bucket": "test-bucket",
// // Prefix (optional)
// // publishing under specified prefix in the bucket, defaults to
// // no prefix (bucket root)
// "prefix": "",
// // Credentials file (optional)
// // Path to a service account credentials JSON file. If omitted,
// // Application Default Credentials are used.
// "credentialsFile": "",
// // Service Account JSON (optional)
// // Inline service account credentials JSON content.
// "serviceAccountJSON": "",
// // Project (optional)
// // Quota project to bill requests to.
// "project": "",
// // Default ACLs (optional)
// // assign ACL to published files. Useful values: `private` (default),
// // `public-read` (public repository), or `none` (don't set ACL).
// "acl": "private",
// // Storage Class (optional)
// // GCS storage class, e.g. `STANDARD`.
// "storageClass": "STANDARD",
// // Encryption Key (optional)
// // Customer-supplied encryption key (32-byte AES-256 key) for object operations.
// "encryptionKey": "",
// // Disable MultiDel (optional)
// // Kept for parity with S3 settings.
// // GCS deletes are currently performed one object at a time.
// "disableMultiDel": false,
// // Debug (optional)
// // Enables detailed GCS operation logs
// "debug": false
// }
},
// Swift Endpoint Support
//
// aptly could be configured to publish repository directly to OpenStack Swift. First,
+101
View File
@@ -0,0 +1,101 @@
from lib import BaseTest
import os
import uuid
try:
from google.cloud import storage
gcs_project = os.environ.get('GCS_PROJECT')
if gcs_project:
gcs_client = storage.Client(project=gcs_project)
else:
print('GCS tests disabled: GCS_PROJECT is not set')
gcs_client = None
except ImportError as e:
print("GCS tests disabled: can't import google.cloud.storage", e)
gcs_client = None
except Exception as e:
print('GCS tests disabled: unable to initialize GCS client', e)
gcs_client = None
class GCSTest(BaseTest):
"""
BaseTest + support for GCS
"""
gcsOverrides = {}
def __init__(self) -> None:
super(GCSTest, self).__init__()
self.bucket_name = None
self.bucket = None
self.bucket_contents = None
def fixture_available(self):
return super(GCSTest, self).fixture_available() and gcs_client is not None
def prepare(self):
# GCS bucket names must be globally unique and lower-case.
self.bucket_name = 'aptly-sys-test-' + str(uuid.uuid4()).replace('_', '-').lower()
self.bucket = gcs_client.create_bucket(self.bucket_name)
self.configOverride = {
'GcsPublishEndpoints': {
'test1': {
'bucket': self.bucket_name,
'project': gcs_project,
},
},
}
self.configOverride['GcsPublishEndpoints']['test1'].update(**self.gcsOverrides)
super(GCSTest, self).prepare()
def shutdown(self):
if self.bucket is not None:
for blob in self.bucket.list_blobs():
blob.delete()
self.bucket.delete(force=True)
super(GCSTest, self).shutdown()
def _normalize_path(self, path):
if path.startswith('public/'):
return path[7:]
return path
def check_path(self, path):
if self.bucket_contents is None:
self.bucket_contents = [blob.name for blob in self.bucket.list_blobs()]
path = self._normalize_path(path)
if path in self.bucket_contents:
return True
if not path.endswith('/'):
path = path + '/'
for item in self.bucket_contents:
if item.startswith(path):
return True
return False
def check_exists(self, path):
if not self.check_path(path):
raise Exception("path %s doesn't exist" % (path, ))
def check_not_exists(self, path):
if self.check_path(path):
raise Exception("path %s exists" % (path, ))
def read_file(self, path, mode=''):
assert not mode
path = self._normalize_path(path)
blob = self.bucket.blob(path)
return blob.download_as_text()
+1
View File
@@ -1,4 +1,5 @@
azure-storage-blob
google-cloud-storage
boto
requests==2.33.0
requests-unixsocket
+116
View File
@@ -0,0 +1,116 @@
from gcs_lib import GCSTest
def strip_processor(output):
return '\n'.join(
[
l
for l in output.split('\n')
if not l.startswith(' ') and not l.startswith('Date:')
]
)
class GCSPublish1Test(GCSTest):
"""
publish to GCS: from repo
"""
fixtureCmds = [
'aptly repo create -distribution=maverick local-repo',
'aptly repo add local-repo ${files}',
'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
]
runCmd = 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:'
def check(self):
self.check_exists('public/dists/maverick/InRelease')
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/Release.gpg')
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/source/Sources')
self.check_exists('public/dists/maverick/main/source/Sources.gz')
self.check_exists('public/dists/maverick/main/source/Sources.bz2')
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
self.check_exists(
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
)
# verify contents except sums/date chunks
self.check_file_contents(
'public/dists/maverick/Release', 'release', match_prepare=strip_processor
)
self.check_file_contents(
'public/dists/maverick/main/source/Sources',
'sources',
match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
)
self.check_file_contents(
'public/dists/maverick/main/binary-i386/Packages',
'binary',
match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
)
class GCSPublish2Test(GCSTest):
"""
publish to GCS: update after removing package from repo
"""
fixtureCmds = [
'aptly repo create -distribution=maverick local-repo',
'aptly repo add local-repo ${files}/',
'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo gcs:test1:',
'aptly repo remove local-repo pyspi',
]
runCmd = 'aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick gcs:test1:'
def check(self):
self.check_exists('public/dists/maverick/InRelease')
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/Release.gpg')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
self.check_exists(
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
)
class GCSPublish3Test(GCSTest):
"""
publish to GCS: publish drop performs cleanup
"""
fixtureCmds = [
'aptly repo create local1',
'aptly repo create local2',
'aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb',
'aptly repo add local2 ${files}',
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 gcs:test1:',
'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 gcs:test1:',
]
runCmd = 'aptly publish drop sq2 gcs:test1:'
def check(self):
self.check_exists('public/dists/sq1')
self.check_not_exists('public/dists/sq2')
self.check_exists('public/pool/main/')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
self.check_exists(
'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
)