mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-11-12 14:31:02 +00:00
Martin Simon
e58bab9587
Acquire-By-Hash entries are written for MD5Sum, SHA1, SHA256 and SHA512,
hardcoded in two places in deb/index_files.go. Every algorithm is a full
set of index copies on each publish, and on S3-like storage each of those
is a network round trip rather than a link.
Measured on an R2-backed archive of 25 packages across three suites, one
`aptly publish update` per suite:
Acquire-By-Hash off 38 S3 requests
Acquire-By-Hash on 222 S3 requests (88 HeadObject, 83 CopyObject,
35 DeleteObject, 13 PutObject)
Modern apt verifies with SHA256; MD5Sum and SHA1 are deprecated and
Debian has been phasing them out of Release files for years. Publishers
who do not need them are paying three quarters of that for nothing, with
no way to say so.
acquireByHashAlgorithms in aptly.conf now selects the set. An empty or
absent value keeps all four, so existing configurations are unaffected.
The order entries are written in comes from the code rather than the
config, so the directories created in one loop and the entries written in
another cannot disagree. An unrecognised name is an error rather than
something to skip: skipping a typo would produce "no by-hash entries for
that algorithm", which looks exactly like having configured it away.
Set per publish invocation from the config, the way SkipBz2 already is,
rather than persisted on the publish point.
Pre-existing test failures on this tree are unchanged by this commit:
deb goes 173->177 passed with the same 11 failures, utils keeps its one
(TestCompress), s3 stays green.
369 lines
14 KiB
Go
369 lines
14 KiB
Go
package utils
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/DisposaBoy/JsonConfigReader"
|
|
yaml "gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// ConfigStructure is structure of main configuration
|
|
type ConfigStructure struct { // nolint: maligned
|
|
// General
|
|
RootDir string `json:"rootDir" yaml:"root_dir"`
|
|
LogLevel string `json:"logLevel" yaml:"log_level"`
|
|
LogFormat string `json:"logFormat" yaml:"log_format"`
|
|
DatabaseOpenAttempts int `json:"databaseOpenAttempts" yaml:"database_open_attempts"`
|
|
Architectures []string `json:"architectures" yaml:"architectures"`
|
|
// Hash algorithms to write Acquire-By-Hash entries for. Each one costs a
|
|
// full set of index copies on every publish, which is felt most on S3-like
|
|
// storage where every write is a round trip. Empty means all four, which is
|
|
// what aptly did before this was configurable.
|
|
AcquireByHashAlgorithms []string `json:"acquireByHashAlgorithms" yaml:"acquire_by_hash_algorithms"`
|
|
SkipLegacyPool bool `json:"skipLegacyPool" yaml:"skip_legacy_pool"` // OBSOLETE
|
|
|
|
// Dependency following
|
|
DepFollowSuggests bool `json:"dependencyFollowSuggests" yaml:"dep_follow_suggests"`
|
|
DepFollowRecommends bool `json:"dependencyFollowRecommends" yaml:"dep_follow_recommends"`
|
|
DepFollowAllVariants bool `json:"dependencyFollowAllVariants" yaml:"dep_follow_all_variants"`
|
|
DepFollowSource bool `json:"dependencyFollowSource" yaml:"dep_follow_source"`
|
|
DepVerboseResolve bool `json:"dependencyVerboseResolve" yaml:"dep_verboseresolve"`
|
|
|
|
// PPA
|
|
PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"`
|
|
PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"`
|
|
PpaBaseURL string `json:"ppaBaseURL" yaml:"ppa_baseurl"`
|
|
|
|
// Server
|
|
ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"`
|
|
EnableMetricsEndpoint bool `json:"enableMetricsEndpoint" yaml:"enable_metrics_endpoint"`
|
|
EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint" yaml:"enable_swagger_endpoint"`
|
|
EnablePprofEndpoint bool `json:"enablePprofEndpoint" yaml:"enable_pprof_endpoint"`
|
|
AsyncAPI bool `json:"AsyncAPI" yaml:"async_api"` // OBSOLETE
|
|
|
|
// Database
|
|
DatabaseBackend DBConfig `json:"databaseBackend" yaml:"database_backend"`
|
|
|
|
// Mirroring
|
|
Downloader string `json:"downloader" yaml:"downloader"`
|
|
DownloadConcurrency int `json:"downloadConcurrency" yaml:"download_concurrency"`
|
|
DownloadLimit int64 `json:"downloadSpeedLimit" yaml:"download_limit"`
|
|
DownloadRetries int `json:"downloadRetries" yaml:"download_retries"`
|
|
DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"`
|
|
|
|
// Signing
|
|
GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"`
|
|
GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"`
|
|
GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"`
|
|
GpgKeys []string `json:"gpgKeys" yaml:"gpg_keys"`
|
|
|
|
// Publishing
|
|
SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"`
|
|
SkipBz2Publishing bool `json:"skipBz2Publishing" yaml:"skip_bz2_publishing"`
|
|
|
|
// Storage
|
|
FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints" yaml:"filesystem_publish_endpoints"`
|
|
JFrogPublishRoots map[string]JFrogPublishRoot `json:"JFrogPublishEndpoints" yaml:"jfrog_publish_endpoints"`
|
|
S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints" yaml:"s3_publish_endpoints"`
|
|
GCSPublishRoots map[string]GCSPublishRoot `json:"GcsPublishEndpoints" yaml:"gcs_publish_endpoints"`
|
|
SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints" yaml:"swift_publish_endpoints"`
|
|
AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints" yaml:"azure_publish_endpoints"`
|
|
PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage" yaml:"packagepool_storage"`
|
|
}
|
|
|
|
// DBConfig structure
|
|
type DBConfig struct {
|
|
Type string `json:"type" yaml:"type"`
|
|
DBPath string `json:"dbPath" yaml:"db_path"`
|
|
URL string `json:"url" yaml:"url"`
|
|
}
|
|
|
|
type LocalPoolStorage struct {
|
|
Path string `json:"path,omitempty" yaml:"path,omitempty"`
|
|
}
|
|
|
|
type PackagePoolStorage struct {
|
|
Local *LocalPoolStorage
|
|
Azure *AzureEndpoint
|
|
}
|
|
|
|
var AZURE = "azure"
|
|
var LOCAL = "local"
|
|
|
|
func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error {
|
|
var discriminator struct {
|
|
Type string `json:"type"`
|
|
}
|
|
|
|
if err := json.Unmarshal(data, &discriminator); err != nil {
|
|
return err
|
|
}
|
|
|
|
switch discriminator.Type {
|
|
case AZURE:
|
|
pool.Azure = &AzureEndpoint{}
|
|
return json.Unmarshal(data, &pool.Azure)
|
|
case LOCAL, "":
|
|
pool.Local = &LocalPoolStorage{}
|
|
return json.Unmarshal(data, &pool.Local)
|
|
default:
|
|
return fmt.Errorf("unknown pool storage type: %s", discriminator.Type)
|
|
}
|
|
}
|
|
|
|
func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
var discriminator struct {
|
|
Type string `yaml:"type"`
|
|
}
|
|
if err := unmarshal(&discriminator); err != nil {
|
|
return err
|
|
}
|
|
|
|
switch discriminator.Type {
|
|
case AZURE:
|
|
pool.Azure = &AzureEndpoint{}
|
|
return unmarshal(&pool.Azure)
|
|
case LOCAL, "":
|
|
pool.Local = &LocalPoolStorage{}
|
|
return unmarshal(&pool.Local)
|
|
default:
|
|
return fmt.Errorf("unknown pool storage type: %s", discriminator.Type)
|
|
}
|
|
}
|
|
|
|
func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) {
|
|
var wrapper struct {
|
|
Type string `json:"type,omitempty"`
|
|
*LocalPoolStorage
|
|
*AzureEndpoint
|
|
}
|
|
|
|
if pool.Azure != nil {
|
|
wrapper.Type = "azure"
|
|
wrapper.AzureEndpoint = pool.Azure
|
|
} else if pool.Local.Path != "" {
|
|
wrapper.Type = "local"
|
|
wrapper.LocalPoolStorage = pool.Local
|
|
}
|
|
|
|
return json.Marshal(wrapper)
|
|
}
|
|
|
|
func (pool PackagePoolStorage) MarshalYAML() (interface{}, error) {
|
|
var wrapper struct {
|
|
Type string `yaml:"type,omitempty"`
|
|
*LocalPoolStorage `yaml:",inline"`
|
|
*AzureEndpoint `yaml:",inline"`
|
|
}
|
|
|
|
if pool.Azure != nil {
|
|
wrapper.Type = "azure"
|
|
wrapper.AzureEndpoint = pool.Azure
|
|
} else if pool.Local.Path != "" {
|
|
wrapper.Type = "local"
|
|
wrapper.LocalPoolStorage = pool.Local
|
|
}
|
|
|
|
return wrapper, nil
|
|
}
|
|
|
|
// FileSystemPublishRoot describes single filesystem publishing entry point
|
|
type FileSystemPublishRoot struct {
|
|
RootDir string `json:"rootDir" yaml:"root_dir"`
|
|
LinkMethod string `json:"linkMethod" yaml:"link_method"`
|
|
VerifyMethod string `json:"verifyMethod" yaml:"verify_method"`
|
|
}
|
|
|
|
// S3PublishRoot describes single S3 publishing entry point
|
|
|
|
type JFrogPublishRoot struct {
|
|
Repository string `json:"repository" yaml:"repository"`
|
|
URL string `json:"url" yaml:"url"`
|
|
User string `json:"user" yaml:"user"`
|
|
Password string `json:"password" yaml:"password"`
|
|
APIKey string `json:"apiKey" yaml:"api_key"`
|
|
AccessToken string `json:"accessToken" yaml:"access_token"`
|
|
Prefix string `json:"prefix" yaml:"prefix"`
|
|
PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"`
|
|
Debug bool `json:"debug" yaml:"debug"`
|
|
}
|
|
|
|
type S3PublishRoot struct {
|
|
Region string `json:"region" yaml:"region"`
|
|
Bucket string `json:"bucket" yaml:"bucket"`
|
|
Prefix string `json:"prefix" yaml:"prefix"`
|
|
ACL string `json:"acl" yaml:"acl"`
|
|
AccessKeyID string `json:"awsAccessKeyID" yaml:"access_key_id"`
|
|
SecretAccessKey string `json:"awsSecretAccessKey" yaml:"secret_access_key"`
|
|
SessionToken string `json:"awsSessionToken" yaml:"session_token"`
|
|
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
|
StorageClass string `json:"storageClass" yaml:"storage_class"`
|
|
EncryptionMethod string `json:"encryptionMethod" yaml:"encryption_method"`
|
|
PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"`
|
|
DisableMultiDel bool `json:"disableMultiDel" yaml:"disable_multidel"`
|
|
ForceSigV2 bool `json:"forceSigV2" yaml:"force_sigv2"`
|
|
ForceVirtualHostedStyle bool `json:"forceVirtualHostedStyle" yaml:"force_virtualhosted_style"`
|
|
Debug bool `json:"debug" yaml:"debug"`
|
|
}
|
|
|
|
// GCSPublishRoot describes single GCS publishing entry point
|
|
type GCSPublishRoot struct {
|
|
Bucket string `json:"bucket" yaml:"bucket"`
|
|
Prefix string `json:"prefix" yaml:"prefix"`
|
|
CredentialsFile string `json:"credentialsFile" yaml:"credentials_file"`
|
|
ServiceAccountJSON string `json:"serviceAccountJSON" yaml:"service_account_json"`
|
|
Project string `json:"project" yaml:"project"`
|
|
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
|
ACL string `json:"acl" yaml:"acl"`
|
|
StorageClass string `json:"storageClass" yaml:"storage_class"`
|
|
EncryptionKey string `json:"encryptionKey" yaml:"encryption_key"`
|
|
DisableMultiDel bool `json:"disableMultiDel" yaml:"disable_multidel"`
|
|
Debug bool `json:"debug" yaml:"debug"`
|
|
}
|
|
|
|
// SwiftPublishRoot describes single OpenStack Swift publishing entry point
|
|
type SwiftPublishRoot struct {
|
|
Container string `json:"container" yaml:"container"`
|
|
Prefix string `json:"prefix" yaml:"prefix"`
|
|
UserName string `json:"osname" yaml:"username"`
|
|
Password string `json:"password" yaml:"password"`
|
|
Tenant string `json:"tenant" yaml:"tenant"`
|
|
TenantID string `json:"tenantid" yaml:"tenant_id"`
|
|
Domain string `json:"domain" yaml:"domain"`
|
|
DomainID string `json:"domainid" yaml:"domain_id"`
|
|
TenantDomain string `json:"tenantdomain" yaml:"tenant_domain"`
|
|
TenantDomainID string `json:"tenantdomainid" yaml:"tenant_domain_id"`
|
|
AuthURL string `json:"authurl" yaml:"auth_url"`
|
|
}
|
|
|
|
// AzureEndpoint describes single Azure publishing entry point
|
|
type AzureEndpoint struct {
|
|
Container string `json:"container" yaml:"container"`
|
|
Prefix string `json:"prefix" yaml:"prefix"`
|
|
AccountName string `json:"accountName" yaml:"account_name"`
|
|
AccountKey string `json:"accountKey" yaml:"account_key"`
|
|
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
|
}
|
|
|
|
// Config is configuration for aptly, shared by all modules
|
|
var Config = ConfigStructure{
|
|
RootDir: filepath.Join(os.Getenv("HOME"), ".aptly"),
|
|
AcquireByHashAlgorithms: []string{"MD5Sum", "SHA1", "SHA256", "SHA512"},
|
|
DownloadConcurrency: 4,
|
|
DownloadLimit: 0,
|
|
Downloader: "default",
|
|
DatabaseOpenAttempts: -1,
|
|
Architectures: []string{},
|
|
DepFollowSuggests: false,
|
|
DepFollowRecommends: false,
|
|
DepFollowAllVariants: false,
|
|
DepFollowSource: false,
|
|
GpgProvider: "gpg",
|
|
GpgDisableSign: false,
|
|
GpgDisableVerify: false,
|
|
GpgKeys: []string{},
|
|
DownloadSourcePackages: false,
|
|
PackagePoolStorage: PackagePoolStorage{
|
|
Local: &LocalPoolStorage{Path: ""},
|
|
},
|
|
SkipLegacyPool: false,
|
|
PpaDistributorID: "ubuntu",
|
|
PpaCodename: "",
|
|
PpaBaseURL: "http://ppa.launchpad.net",
|
|
FileSystemPublishRoots: map[string]FileSystemPublishRoot{},
|
|
S3PublishRoots: map[string]S3PublishRoot{},
|
|
GCSPublishRoots: map[string]GCSPublishRoot{},
|
|
SwiftPublishRoots: map[string]SwiftPublishRoot{},
|
|
AzurePublishRoots: map[string]AzureEndpoint{},
|
|
AsyncAPI: false,
|
|
EnableMetricsEndpoint: false,
|
|
LogLevel: "info",
|
|
LogFormat: "default",
|
|
ServeInAPIMode: false,
|
|
EnableSwaggerEndpoint: false,
|
|
}
|
|
|
|
// LoadConfig loads configuration from json file
|
|
func LoadConfig(filename string, config *ConfigStructure) error {
|
|
f, err := os.Open(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = f.Close()
|
|
}()
|
|
|
|
decJSON := json.NewDecoder(JsonConfigReader.New(f))
|
|
if err = decJSON.Decode(&config); err != nil {
|
|
_, _ = f.Seek(0, 0)
|
|
decYAML := yaml.NewDecoder(f)
|
|
if err2 := decYAML.Decode(&config); err2 != nil {
|
|
err = fmt.Errorf("invalid yaml (%s) or json (%s)", err2, err)
|
|
} else {
|
|
err = nil
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// SaveConfig write configuration to json file
|
|
func SaveConfig(filename string, config *ConfigStructure) error {
|
|
f, err := os.Create(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = f.Close()
|
|
}()
|
|
|
|
encoded, err := json.MarshalIndent(&config, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = f.Write(encoded)
|
|
return err
|
|
}
|
|
|
|
// SaveConfigRaw write configuration to file
|
|
func SaveConfigRaw(filename string, conf []byte) error {
|
|
f, err := os.Create(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = f.Close()
|
|
}()
|
|
|
|
_, err = f.Write(conf)
|
|
return err
|
|
}
|
|
|
|
// SaveConfigYAML write configuration to yaml file
|
|
func SaveConfigYAML(filename string, config *ConfigStructure) error {
|
|
f, err := os.Create(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = f.Close()
|
|
}()
|
|
|
|
yamlData, err := yaml.Marshal(&config)
|
|
if err != nil {
|
|
return fmt.Errorf("error marshaling to YAML: %s", err)
|
|
}
|
|
|
|
_, err = f.Write(yamlData)
|
|
return err
|
|
}
|
|
|
|
// GetRootDir returns the RootDir with expanded ~ as home directory
|
|
func (conf *ConfigStructure) GetRootDir() string {
|
|
return strings.Replace(conf.RootDir, "~", os.Getenv("HOME"), 1)
|
|
}
|