vincent/aptly
1
0 Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-11-13 14:37:18 +00:00
Code Issues Projects Releases Wiki Activity
Files
0fcc80db7d7dbba0a77cb09813cf0973905ebda6
aptly/utils/config_test.go
T
Martin Simon e58bab9587 Make the set of Acquire-By-Hash algorithms configurable 
Acquire-By-Hash entries are written for MD5Sum, SHA1, SHA256 and SHA512,
hardcoded in two places in deb/index_files.go. Every algorithm is a full
set of index copies on each publish, and on S3-like storage each of those
is a network round trip rather than a link.

Measured on an R2-backed archive of 25 packages across three suites, one
`aptly publish update` per suite:

  Acquire-By-Hash off   38 S3 requests
  Acquire-By-Hash on   222 S3 requests   (88 HeadObject, 83 CopyObject,
                                          35 DeleteObject, 13 PutObject)

Modern apt verifies with SHA256; MD5Sum and SHA1 are deprecated and
Debian has been phasing them out of Release files for years. Publishers
who do not need them are paying three quarters of that for nothing, with
no way to say so.

acquireByHashAlgorithms in aptly.conf now selects the set. An empty or
absent value keeps all four, so existing configurations are unaffected.
The order entries are written in comes from the code rather than the
config, so the directories created in one loop and the entries written in
another cannot disagree. An unrecognised name is an error rather than
something to skip: skipping a typo would produce "no by-hash entries for
that algorithm", which looks exactly like having configured it away.

Set per publish invocation from the config, the way SkipBz2 already is,
rather than persisted on the publish point.

Pre-existing test failures on this tree are unchanged by this commit:
deb goes 173->177 passed with the same 11 failures, utils keeps its one
(TestCompress), s3 stays green.
2026-11-03 13:13:18 +01:00

442 lines
11 KiB
Go
Raw Blame History

package utils
import (
"os"
"path/filepath"
. "gopkg.in/check.v1"
)
type ConfigSuite struct {
config ConfigStructure
}
var _ = Suite(&ConfigSuite{})
func (s *ConfigSuite) TestLoadConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.json1")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFile)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err, IsNil)
c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/")
c.Check(s.config.DownloadConcurrency, Equals, 33)
c.Check(s.config.DatabaseOpenAttempts, Equals, 33)
}
func (s *ConfigSuite) TestSaveConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.json2")
// start with empty config
s.config = ConfigStructure{}
s.config.RootDir = "/tmp/aptly"
s.config.DownloadConcurrency = 5
s.config.DatabaseOpenAttempts = 5
s.config.GpgProvider = "gpg"
s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"}
s.config.FileSystemPublishRoots = map[string]FileSystemPublishRoot{"test": {
RootDir: "/opt/aptly-publish"}}
s.config.JFrogPublishRoots = map[string]JFrogPublishRoot{"test": {
Repository: "repo",
URL: "jfrog.example.com"}}
s.config.S3PublishRoots = map[string]S3PublishRoot{"test": {
Region: "us-east-1",
Bucket: "repo"}}
s.config.GCSPublishRoots = map[string]GCSPublishRoot{"test": {
Bucket: "repo"}}
s.config.SwiftPublishRoots = map[string]SwiftPublishRoot{"test": {
Container: "repo"}}
s.config.AzurePublishRoots = map[string]AzureEndpoint{"test": {
Container: "repo"}}
s.config.LogLevel = "info"
s.config.LogFormat = "json"
err := SaveConfig(configname, &s.config)
c.Assert(err, IsNil)
f, _ := os.Open(configname)
defer func() {
_ = f.Close()
}()
st, _ := f.Stat()
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, `{
"rootDir": "/tmp/aptly",
"logLevel": "info",
"logFormat": "json",
"databaseOpenAttempts": 5,
"architectures": null,
"acquireByHashAlgorithms": null,
"skipLegacyPool": false,
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"ppaDistributorID": "",
"ppaCodename": "",
"ppaBaseURL": "",
"serveInAPIMode": false,
"enableMetricsEndpoint": false,
"enableSwaggerEndpoint": false,
"enablePprofEndpoint": false,
"AsyncAPI": false,
"databaseBackend": {
"type": "",
"dbPath": "",
"url": ""
},
"downloader": "",
"downloadConcurrency": 5,
"downloadSpeedLimit": 0,
"downloadRetries": 0,
"downloadSourcePackages": false,
"gpgProvider": "gpg",
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgKeys": null,
"skipContentsPublishing": false,
"skipBz2Publishing": false,
"FileSystemPublishEndpoints": {
"test": {
"rootDir": "/opt/aptly-publish",
"linkMethod": "",
"verifyMethod": ""
}
},
"JFrogPublishEndpoints": {
"test": {
"repository": "repo",
"url": "jfrog.example.com",
"user": "",
"password": "",
"apiKey": "",
"accessToken": "",
"prefix": "",
"plusWorkaround": false,
"debug": false
}
},
"S3PublishEndpoints": {
"test": {
"region": "us-east-1",
"bucket": "repo",
"prefix": "",
"acl": "",
"awsAccessKeyID": "",
"awsSecretAccessKey": "",
"awsSessionToken": "",
"endpoint": "",
"storageClass": "",
"encryptionMethod": "",
"plusWorkaround": false,
"disableMultiDel": false,
"forceSigV2": false,
"forceVirtualHostedStyle": false,
"debug": false
}
},
"GcsPublishEndpoints": {
"test": {
"bucket": "repo",
"prefix": "",
"credentialsFile": "",
"serviceAccountJSON": "",
"project": "",
"endpoint": "",
"acl": "",
"storageClass": "",
"encryptionKey": "",
"disableMultiDel": false,
"debug": false
}
},
"SwiftPublishEndpoints": {
"test": {
"container": "repo",
"prefix": "",
"osname": "",
"password": "",
"tenant": "",
"tenantid": "",
"domain": "",
"domainid": "",
"tenantdomain": "",
"tenantdomainid": "",
"authurl": ""
}
},
"AzurePublishEndpoints": {
"test": {
"container": "repo",
"prefix": "",
"accountName": "",
"accountKey": "",
"endpoint": ""
}
},
"packagePoolStorage": {
"type": "local",
"path": "/tmp/aptly-pool"
}
}`)
}
func (s *ConfigSuite) TestLoadYAMLConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml1")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAML)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err, IsNil)
c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/")
c.Check(s.config.DownloadConcurrency, Equals, 40)
c.Check(s.config.DatabaseOpenAttempts, Equals, 10)
}
func (s *ConfigSuite) TestLoadYAMLErrorConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml2")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAMLError)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err.Error(), Equals, "invalid yaml (unknown pool storage type: invalid) or json (invalid character 'p' looking for beginning of value)")
}
func (s *ConfigSuite) TestSaveYAMLConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml3")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAML)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err, IsNil)
err = SaveConfigYAML(configname, &s.config)
c.Assert(err, IsNil)
f, _ = os.Open(configname)
defer func() {
_ = f.Close()
}()
st, _ := f.Stat()
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, configFileYAML)
}
func (s *ConfigSuite) TestSaveYAML2Config(c *C) {
// start with empty config
s.config = ConfigStructure{}
s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"}
s.config.PackagePoolStorage.Azure = nil
configname := filepath.Join(c.MkDir(), "aptly.yaml4")
err := SaveConfigYAML(configname, &s.config)
c.Assert(err, IsNil)
f, _ := os.Open(configname)
defer func() {
_ = f.Close()
}()
st, _ := f.Stat()
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, ""+
"root_dir: \"\"\n"+
"log_level: \"\"\n"+
"log_format: \"\"\n"+
"database_open_attempts: 0\n"+
"architectures: []\n"+
"acquire_by_hash_algorithms: []\n"+
"skip_legacy_pool: false\n"+
"dep_follow_suggests: false\n"+
"dep_follow_recommends: false\n"+
"dep_follow_all_variants: false\n"+
"dep_follow_source: false\n"+
"dep_verboseresolve: false\n"+
"ppa_distributor_id: \"\"\n"+
"ppa_codename: \"\"\n"+
"ppa_baseurl: \"\"\n"+
"serve_in_api_mode: false\n"+
"enable_metrics_endpoint: false\n"+
"enable_swagger_endpoint: false\n"+
"enable_pprof_endpoint: false\n"+
"async_api: false\n"+
"database_backend:\n"+
" type: \"\"\n"+
" db_path: \"\"\n"+
" url: \"\"\n"+
"downloader: \"\"\n"+
"download_concurrency: 0\n"+
"download_limit: 0\n"+
"download_retries: 0\n"+
"download_sourcepackages: false\n"+
"gpg_provider: \"\"\n"+
"gpg_disable_sign: false\n"+
"gpg_disable_verify: false\n"+
"gpg_keys: []\n"+
"skip_contents_publishing: false\n"+
"skip_bz2_publishing: false\n"+
"filesystem_publish_endpoints: {}\n"+
"jfrog_publish_endpoints: {}\n"+
"s3_publish_endpoints: {}\n"+
"gcs_publish_endpoints: {}\n"+
"swift_publish_endpoints: {}\n"+
"azure_publish_endpoints: {}\n"+
"packagepool_storage:\n"+
" type: local\n"+
" path: /tmp/aptly-pool\n")
}
func (s *ConfigSuite) TestLoadEmptyConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml5")
f, _ := os.Create(configname)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err.Error(), Equals, "invalid yaml (EOF) or json (EOF)")
}
const configFile = `{"rootDir": "/opt/aptly/", "downloadConcurrency": 33, "databaseOpenAttempts": 33}`
const configFileYAML = `root_dir: /opt/aptly/
log_level: error
log_format: json
database_open_attempts: 10
architectures:
- amd64
- arm64
acquire_by_hash_algorithms: []
skip_legacy_pool: true
dep_follow_suggests: true
dep_follow_recommends: true
dep_follow_all_variants: true
dep_follow_source: true
dep_verboseresolve: true
ppa_distributor_id: Ubuntu
ppa_codename: code
ppa_baseurl: http://ppa.launchpad.net
serve_in_api_mode: true
enable_metrics_endpoint: true
enable_swagger_endpoint: true
enable_pprof_endpoint: false
async_api: true
database_backend:
type: etcd
db_path: ""
url: 127.0.0.1:2379
downloader: grab
download_concurrency: 40
download_limit: 100
download_retries: 10
download_sourcepackages: true
gpg_provider: gpg
gpg_disable_sign: true
gpg_disable_verify: true
gpg_keys: []
skip_contents_publishing: true
skip_bz2_publishing: true
filesystem_publish_endpoints:
test1:
root_dir: /opt/srv/aptly_public
link_method: hardlink
verify_method: md5
jfrog_publish_endpoints: {}
s3_publish_endpoints:
test:
region: us-east-1
bucket: test-bucket
prefix: prfx
acl: public-read
access_key_id: "2"
secret_access_key: secret
session_token: none
endpoint: endpoint
storage_class: STANDARD
encryption_method: AES256
plus_workaround: true
disable_multidel: true
force_sigv2: true
force_virtualhosted_style: true
debug: true
gcs_publish_endpoints:
test:
bucket: gcs-bucket
prefix: pre-gcs
credentials_file: /tmp/creds.json
service_account_json: '{"type":"service_account"}'
project: test-project
endpoint: ""
acl: public-read
storage_class: STANDARD
encryption_key: "12345678901234567890123456789012"
disable_multidel: true
debug: true
swift_publish_endpoints:
test:
container: c1
prefix: pre
username: user
password: pass
tenant: t
tenant_id: "2"
domain: pop
domain_id: "1"
tenant_domain: td
tenant_domain_id: "3"
auth_url: http://auth.url
azure_publish_endpoints:
test:
container: container1
prefix: pre2
account_name: aname
account_key: akey
endpoint: https://end.point
packagepool_storage:
type: azure
container: test-pool1
prefix: pre3
account_name: a name
account_key: a key
endpoint: ep
`
const configFileYAMLError = `packagepool_storage:
type: invalid
`
Reference in new issue View Git Blame Copy Permalink