[adapted4debian] publish: check if storage exists

This commit is contained in:
André Roth
2026-06-18 13:36:17 +02:00
parent 2ae5830239
commit f488412dc0
9 changed files with 69 additions and 40 deletions
+8 -4
View File
@@ -220,7 +220,8 @@ func (s *PublishedFileMissingSuite) TestPublishedFileGoMissing(c *C) {
c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String())) c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String()))
// Now check if the file is actually accessible in the published location // Now check if the file is actually accessible in the published location
publishedStorage := s.context.GetPublishedStorage("") publishedStorage, err := s.context.GetPublishedStorage("")
c.Assert(err, IsNil)
publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
// Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb // Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb
@@ -332,7 +333,8 @@ func (s *PublishedFileMissingSuite) TestConcurrentPublishRace(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
// Check published files // Check published files
publishedStorage := s.context.GetPublishedStorage("") publishedStorage, err := s.context.GetPublishedStorage("")
c.Assert(err, IsNil)
publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
missingFiles := []string{} missingFiles := []string{}
@@ -446,7 +448,8 @@ func (s *PublishedFileMissingSuite) TestIdenticalPackageRace(c *C) {
c.Logf("[iter %d] All operations complete", iter) c.Logf("[iter %d] All operations complete", iter)
// Check the shared pool location // Check the shared pool location
publishedStorage := s.context.GetPublishedStorage("") publishedStorage, err := s.context.GetPublishedStorage("")
c.Assert(err, IsNil)
publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
poolSubdir := string(packageName[0]) poolSubdir := string(packageName[0])
@@ -663,7 +666,8 @@ func (s *PublishedFileMissingSuite) TestConcurrentSnapshotPublishToSamePrefix(c
c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed")) c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed"))
// Verify ALL package files exist in the published pool // Verify ALL package files exist in the published pool
publishedStorage := s.context.GetPublishedStorage("") publishedStorage, err := s.context.GetPublishedStorage("")
c.Assert(err, IsNil)
publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
missingFiles := []string{} missingFiles := []string{}
+6 -1
View File
@@ -60,7 +60,12 @@ func reposServeInAPIMode(c *gin.Context) {
storage = "filesystem:" + storage storage = "filesystem:" + storage
} }
publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath() ps, err := context.GetPublishedStorage(storage)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
c.FileFromFS(pkgpath, http.Dir(publicPath)) c.FileFromFS(pkgpath, http.Dir(publicPath))
} }
+2 -2
View File
@@ -95,8 +95,8 @@ type FileSystemPublishedStorage interface {
// PublishedStorageProvider is a thing that returns PublishedStorage by name // PublishedStorageProvider is a thing that returns PublishedStorage by name
type PublishedStorageProvider interface { type PublishedStorageProvider interface {
// GetPublishedStorage returns PublishedStorage by name // GetPublishedStorage returns PublishedStorage by name, or an error if the storage is not configured
GetPublishedStorage(name string) PublishedStorage GetPublishedStorage(name string) (PublishedStorage, error)
} }
// BarType used to differentiate between different progress bars // BarType used to differentiate between different progress bars
+5 -3
View File
@@ -190,9 +190,11 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
context.Progress().Printf("\n%s been successfully published.\n", message) context.Progress().Printf("\n%s been successfully published.\n", message)
if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok { if ps, err := context.GetPublishedStorage(storage); err == nil {
context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n", if localStorage, ok := ps.(aptly.FileSystemPublishedStorage); ok {
localStorage.PublicPath()) context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
localStorage.PublicPath())
}
} }
context.Progress().Printf("Now you can add following line to apt sources:\n") context.Progress().Printf("Now you can add following line to apt sources:\n")
+5 -1
View File
@@ -97,7 +97,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
} }
} }
publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath() ps, err := context.GetPublishedStorage("")
if err != nil {
return err
}
publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
ShutdownContext() ShutdownContext()
fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen) fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+17 -12
View File
@@ -116,7 +116,12 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation) fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf) defaultConfig := aptly.AptlyConf
if len(defaultConfig) == 0 {
defaultConfig = []byte("root_dir: \"\"")
}
_ = utils.SaveConfigRaw(homeLocation, defaultConfig)
err = utils.LoadConfig(homeLocation, &utils.Config) err = utils.LoadConfig(homeLocation, &utils.Config)
if err != nil { if err != nil {
Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err)) Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -407,8 +412,8 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
return context.packagePool return context.packagePool
} }
// GetPublishedStorage returns instance of PublishedStorage // GetPublishedStorage returns instance of PublishedStorage, or an error if the storage is not configured
func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage { func (context *AptlyContext) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
context.Lock() context.Lock()
defer context.Unlock() defer context.Unlock()
@@ -419,14 +424,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
} else if strings.HasPrefix(name, "filesystem:") { } else if strings.HasPrefix(name, "filesystem:") {
params, ok := context.config().FileSystemPublishRoots[name[11:]] params, ok := context.config().FileSystemPublishRoots[name[11:]]
if !ok { if !ok {
Fatal(fmt.Errorf("published local storage %v not configured", name[11:])) return nil, fmt.Errorf("published local storage %v not configured", name[11:])
} }
publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod) publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
} else if strings.HasPrefix(name, "s3:") { } else if strings.HasPrefix(name, "s3:") {
params, ok := context.config().S3PublishRoots[name[3:]] params, ok := context.config().S3PublishRoots[name[3:]]
if !ok { if !ok {
Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:])) return nil, fmt.Errorf("published S3 storage %v not configured", name[3:])
} }
var err error var err error
@@ -436,39 +441,39 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel, params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug) params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
if err != nil { if err != nil {
Fatal(err) return nil, err
} }
} else if strings.HasPrefix(name, "swift:") { } else if strings.HasPrefix(name, "swift:") {
params, ok := context.config().SwiftPublishRoots[name[6:]] params, ok := context.config().SwiftPublishRoots[name[6:]]
if !ok { if !ok {
Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:])) return nil, fmt.Errorf("published Swift storage %v not configured", name[6:])
} }
var err error var err error
publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password, publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix) params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
if err != nil { if err != nil {
Fatal(err) return nil, err
} }
} else if strings.HasPrefix(name, "azure:") { } else if strings.HasPrefix(name, "azure:") {
params, ok := context.config().AzurePublishRoots[name[6:]] params, ok := context.config().AzurePublishRoots[name[6:]]
if !ok { if !ok {
Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:])) return nil, fmt.Errorf("published Azure storage %v not configured", name[6:])
} }
var err error var err error
publishedStorage, err = azure.NewPublishedStorage( publishedStorage, err = azure.NewPublishedStorage(
params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint) params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint)
if err != nil { if err != nil {
Fatal(err) return nil, err
} }
} else { } else {
Fatal(fmt.Errorf("unknown published storage format: %v", name)) return nil, fmt.Errorf("unknown published storage format: %v", name)
} }
context.publishedStorages[name] = publishedStorage context.publishedStorages[name] = publishedStorage
} }
return publishedStorage return publishedStorage, nil
} }
// UploadPath builds path to upload storage // UploadPath builds path to upload storage
+5 -8
View File
@@ -1,8 +1,6 @@
package context package context
import ( import (
"fmt"
"os"
"reflect" "reflect"
"testing" "testing"
@@ -80,10 +78,9 @@ func (s *AptlyContextSuite) SetUpTest(c *C) {
func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
// https://github.com/aptly-dev/aptly/issues/711 // https://github.com/aptly-dev/aptly/issues/711
// This will fail on account of us not having a config, so the // https://github.com/aptly-dev/aptly/issues/1477
// storage never exists. // GetPublishedStorage must return an error (not panic) when the
c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, // requested storage is not configured.
FatalErrorPanicMatches, _, err := s.context.GetPublishedStorage("filesystem:fuji")
&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)", c.Assert(err, NotNil)
os.Getenv("HOME"))})
} }
+18 -6
View File
@@ -824,9 +824,12 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error { collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error {
publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
if err != nil {
return err
}
err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) err = publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
if err != nil { if err != nil {
return err return err
} }
@@ -1200,7 +1203,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
// It can remove prefix fully, and part of pool (for specific component) // It can remove prefix fully, and part of pool (for specific component)
func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool, func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool,
removePoolComponents []string, progress aptly.Progress) error { removePoolComponents []string, progress aptly.Progress) error {
publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
if err != nil {
return err
}
// I. Easy: remove whole prefix (meta+packages) // I. Easy: remove whole prefix (meta+packages)
if removePrefix { if removePrefix {
@@ -1213,7 +1219,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
} }
// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique // II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress) err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
if err != nil { if err != nil {
return err return err
} }
@@ -1573,7 +1579,10 @@ func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(published
} }
// true→false: directly remove the per-distribution pool directories. // true→false: directly remove the per-distribution pool directories.
publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage) publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
if err != nil {
return err
}
for _, component := range cleanComponents { for _, component := range cleanComponents {
poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component) poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component)
if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil { if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil {
@@ -1599,7 +1608,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
distribution := published.Distribution distribution := published.Distribution
rootPath := filepath.Join(prefix, "dists", distribution) rootPath := filepath.Join(prefix, "dists", distribution)
publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage) publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
if err != nil {
return err
}
sort.Strings(cleanComponents) sort.Strings(cleanComponents)
publishedComponents := published.Components() publishedComponents := published.Components()
+3 -3
View File
@@ -61,12 +61,12 @@ type FakeStorageProvider struct {
storages map[string]aptly.PublishedStorage storages map[string]aptly.PublishedStorage
} }
func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage { func (p *FakeStorageProvider) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
storage, ok := p.storages[name] storage, ok := p.storages[name]
if !ok { if !ok {
panic(fmt.Sprintf("unknown storage: %#v", name)) return nil, fmt.Errorf("unknown storage: %#v", name)
} }
return storage return storage, nil
} }
type PublishedRepoSuite struct { type PublishedRepoSuite struct {