From bc090e1dce0a6c4f9882002aef73ee8639f1957c Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Tue, 30 May 2023 15:25:40 +0200 Subject: [PATCH 01/12] add GetAddonDir to context --- context/context.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/context/context.go b/context/context.go index 7e468b0f..bf8b5cf4 100644 --- a/context/context.go +++ b/context/context.go @@ -527,6 +527,11 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier { return pgp.NewGpgVerifier(context.getGPGFinder()) } +// AddonPath builds the local addon folder +func (context *AptlyContext) AddonPath() string { + return filepath.Join(context.config().RootDir, "addon") +} + // UpdateFlags sets internal copy of flags in the context func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) { context.Lock() From 2aac7baf52fcf6b43f63eb9fd94c82b24c3c8090 Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Tue, 30 May 2023 15:26:45 +0200 Subject: [PATCH 02/12] add AddonIndex to index_files I had to remove "signable: false" (line 399), since that property doesn't exist. --- deb/index_files.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/deb/index_files.go b/deb/index_files.go index 01de3dd5..17f180cd 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -389,6 +389,27 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile return file } +func (files *indexFiles) AddonIndex(component, path string) *indexFile { + key := fmt.Sprintf("ai-%s-%s", component, path) + file, ok := files.indexes[key] + + if !ok { + relativePath := filepath.Join(component, path) + + file = &indexFile{ + parent: files, + discardable: false, + compressable: false, + onlyGzip: false, + relativePath: relativePath, + } + + files.indexes[key] = file + } + + return file +} + func (files *indexFiles) ReleaseFile() *indexFile { return &indexFile{ parent: files, From 17186b0c739f6c95ab94cddaf1554d91f63a9d2d Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Tue, 30 May 2023 15:29:30 +0200 Subject: [PATCH 03/12] add GetAddonPaths to publish file --- deb/publish.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/deb/publish.go b/deb/publish.go index 9143e610..53a4e775 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -773,6 +773,44 @@ func (p *PublishedRepo) GetCodename() string { return p.Codename } +// GetAddonFiles returns a map of files to be added to a repo. Key being the relative +// path from component folder, and value being the full local FS path. +func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[string]string, error) { + files := make(map[string]string) + + if addonDir == "" { + return files, nil + } + + fsPath := filepath.Join(addonDir, p.Prefix, "dists", p.Distribution, component) + if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + stat, err := os.Stat(path) + if err != nil { + return err + } + + if !stat.Mode().IsRegular() { + return nil + } + + relativePath, err := filepath.Rel(fsPath, path) + if err != nil { + return err + } + + files[relativePath] = path + return nil + }); err != nil && !os.IsNotExist(err) { + return files, err + } + + return files, nil +} + // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error { From c75c2c759462d8ec923805cee5cb11b372ed8214 Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Wed, 31 May 2023 07:13:15 +0200 Subject: [PATCH 04/12] pass down addonpath from api and cmd context --- api/publish.go | 4 ++-- cmd/publish_snapshot.go | 2 +- cmd/publish_switch.go | 2 +- cmd/publish_update.go | 2 +- deb/publish.go | 26 +++++++++++++++++++++++++- deb/publish_test.go | 10 +++++----- man/aptly.1.ronn.tmpl | 3 ++- 7 files changed, 37 insertions(+), 12 deletions(-) diff --git a/api/publish.go b/api/publish.go index b2f4ffad..fbdb3451 100644 --- a/api/publish.go +++ b/api/publish.go @@ -340,7 +340,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.AddonPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } @@ -1018,7 +1018,7 @@ func apiPublishUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.AddonPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 6312c4d4..19753d16 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -170,7 +170,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 12d73f08..c0c47dcc 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -103,7 +103,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 4b38282f..f59bb65e 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -64,7 +64,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/deb/publish.go b/deb/publish.go index 53a4e775..319ae635 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -813,7 +813,7 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error { + collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, addonDirectory string) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -1022,6 +1022,30 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } + for component := range p.sourceItems { + addonFiles, err := p.GetAddonFiles(addonDirectory, component) + if err != nil { + return fmt.Errorf("unable to get addon files: %v", err) + } + + for relPath, absPath := range addonFiles { + bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() + if err != nil { + return fmt.Errorf("unable to generate addon index: %v", err) + } + + file, err := os.Open(absPath) + if err != nil { + return fmt.Errorf("unable to read addon file: %v", err) + } + + _, err = bufio.NewReader(file).WriteTo(bufWriter) + if err != nil { + return fmt.Errorf("unable to write addon file: %v", err) + } + } + } + udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) diff --git a/deb/publish_test.go b/deb/publish_test.go index eec1601f..4ce9c4ce 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -380,7 +380,7 @@ func (s *PublishedRepoSuite) TestUpdate(c *C) { } func (s *PublishedRepoSuite) TestPublish(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") c.Assert(err, IsNil) c.Check(s.repo.Architectures, DeepEquals, []string{"i386"}) @@ -427,7 +427,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { } func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists) @@ -435,7 +435,7 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { - err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -443,7 +443,7 @@ func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { - err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -451,7 +451,7 @@ func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) { - err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists) diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index e88ba35e..0585ad3c 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -121,7 +121,8 @@ Options: * `rootDir`: is root of directory storage to store database (`rootDir`/db), the default for downloaded packages (`rootDir`/pool) and - the default for published repositories (`rootDir`/public) + the default for published repositories (`rootDir`/public) and + addon files (`rootDir`/addon) * `databaseBackend`: the database config; if this config is empty, use levledb backend by default From 8d8f4714c32504bfc97b2c640889aace56997cd2 Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Wed, 31 May 2023 07:13:52 +0200 Subject: [PATCH 05/12] add name to AUTHORS list --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 0abdc0b0..1fd68974 100644 --- a/AUTHORS +++ b/AUTHORS @@ -65,3 +65,4 @@ List of contributors, in chronological order: * Cookie Fei (https://github.com/wuhuang26) * Andrey Loukhnov (https://github.com/aol-nnov) * Christoph Fiehe (https://github.com/cfiehe) +* Blake Kostner (https://github.com/btkostner) From 03f189b62cfe5c61f9f40e8e9897a1add759d50a Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Wed, 31 May 2023 07:26:48 +0200 Subject: [PATCH 06/12] add first test of addon files --- system/lib.py | 9 ++++ system/t06_publish/PublishRepo34Test_gold | 14 ++++++ system/t06_publish/repo.py | 58 +++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 system/t06_publish/PublishRepo34Test_gold diff --git a/system/lib.py b/system/lib.py index 6f76213f..2918d805 100644 --- a/system/lib.py +++ b/system/lib.py @@ -402,6 +402,15 @@ class BaseTest(object): else: raise + def write_file(self, path, content): + full_path = os.path.join(os.environ["HOME"], ".aptly", path) + + if not os.path.exists(os.path.dirname(full_path)): + os.makedirs(os.path.dirname(full_path), 0o755) + + with open(full_path, "w") as f: + f.write(content) + def read_file(self, path, mode=''): with open(os.path.join(os.environ["HOME"], self.aptlyDir, path), "r" + mode) as f: return f.read() diff --git a/system/t06_publish/PublishRepo34Test_gold b/system/t06_publish/PublishRepo34Test_gold new file mode 100644 index 00000000..365295fa --- /dev/null +++ b/system/t06_publish/PublishRepo34Test_gold @@ -0,0 +1,14 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Local repo local-repo has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ maverick main + deb-src http://your-server/ maverick main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 7e4b10c5..e469d9a1 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -888,3 +888,61 @@ class PublishRepo33Test(BaseTest): self.check_exists('public/dists/maverick/main/binary-amd64/Packages') self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz') self.check_not_exists('public/dists/maverick/main/binary-amd64/Packages.bz2') + + +class PublishRepo34Test(BaseTest): + """ + publish repo: addon files + """ + fixtureCmds = [ + "aptly repo create local-repo", + "aptly repo add local-repo ${files}" + ] + runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-contents local-repo" + gold_processor = BaseTest.expand_environ + + def prepare_fixture(self): + super(PublishRepo34Test, self).prepare_fixture() + + self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + + def check(self): + super(PublishRepo34Test, self).check() + + self.check_exists('public/dists/maverick/main/dep11/README') + + self.check_exists('public/dists/maverick/Release') + + readme = self.read_file('public/dists/maverick/main/dep11/README') + if readme != 'README test file': + raise Exception("README file not copied on publish") + + release = self.read_file('public/dists/maverick/Release').split("\n") + release = [l for l in release if l.startswith(" ")] + pathsSeen = set() + for l in release: + fileHash, fileSize, path = l.split() + pathsSeen.add(path) + + fileSize = int(fileSize) + + st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path)) + if fileSize != st.st_size: + raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size)) + + if len(fileHash) == 32: + h = hashlib.md5() + elif len(fileHash) == 40: + h = hashlib.sha1() + elif len(fileHash) == 64: + h = hashlib.sha256() + else: + h = hashlib.sha512() + + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) + + if h.hexdigest() != fileHash: + raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) + + if 'main/dep11/README' not in pathsSeen: + raise Exception("README file not included in release file") From 1357d246d811962f236719b42f129d26ac304fdc Mon Sep 17 00:00:00 2001 From: Mauro Regli Date: Wed, 31 May 2023 07:39:29 +0200 Subject: [PATCH 07/12] rename addon files to skel files --- api/publish.go | 5 +++-- cmd/publish_snapshot.go | 2 +- cmd/publish_switch.go | 2 +- cmd/publish_update.go | 2 +- context/context.go | 6 +++--- deb/index_files.go | 4 ++-- deb/publish.go | 24 ++++++++++++------------ man/aptly.1.ronn.tmpl | 2 +- system/t06_publish/repo.py | 4 ++-- 9 files changed, 26 insertions(+), 25 deletions(-) diff --git a/api/publish.go b/api/publish.go index fbdb3451..aa45750a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -340,7 +340,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.AddonPath()) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } @@ -477,6 +477,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { } } +<<<<<<< HEAD result, err := published.Update(collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) @@ -1018,7 +1019,7 @@ func apiPublishUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.AddonPath()) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 19753d16..eac4c3b3 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -170,7 +170,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index c0c47dcc..f39269a1 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -103,7 +103,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index f59bb65e..6ea638d4 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -64,7 +64,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.AddonPath()) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/context/context.go b/context/context.go index bf8b5cf4..296d67b9 100644 --- a/context/context.go +++ b/context/context.go @@ -527,9 +527,9 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier { return pgp.NewGpgVerifier(context.getGPGFinder()) } -// AddonPath builds the local addon folder -func (context *AptlyContext) AddonPath() string { - return filepath.Join(context.config().RootDir, "addon") +// SkelPath builds the local skeleton folder +func (context *AptlyContext) SkelPath() string { + return filepath.Join(context.config().RootDir, "skel") } // UpdateFlags sets internal copy of flags in the context diff --git a/deb/index_files.go b/deb/index_files.go index 17f180cd..27a19470 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -389,8 +389,8 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile return file } -func (files *indexFiles) AddonIndex(component, path string) *indexFile { - key := fmt.Sprintf("ai-%s-%s", component, path) +func (files *indexFiles) SkelIndex(component, path string) *indexFile { + key := fmt.Sprintf("si-%s-%s", component, path) file, ok := files.indexes[key] if !ok { diff --git a/deb/publish.go b/deb/publish.go index 319ae635..7205a176 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -773,16 +773,16 @@ func (p *PublishedRepo) GetCodename() string { return p.Codename } -// GetAddonFiles returns a map of files to be added to a repo. Key being the relative +// GetSkelFiles returns a map of files to be added to a repo. Key being the relative // path from component folder, and value being the full local FS path. -func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[string]string, error) { +func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[string]string, error) { files := make(map[string]string) - if addonDir == "" { + if skelDir == "" { return files, nil } - fsPath := filepath.Join(addonDir, p.Prefix, "dists", p.Distribution, component) + fsPath := filepath.Join(skelDir, p.Prefix, "dists", p.Distribution, component) if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -813,7 +813,7 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, addonDirectory string) error { + collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, skelDir string) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -1023,25 +1023,25 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } for component := range p.sourceItems { - addonFiles, err := p.GetAddonFiles(addonDirectory, component) + skelFiles, err := p.GetSkelFiles(skelDir, component) if err != nil { - return fmt.Errorf("unable to get addon files: %v", err) + return fmt.Errorf("unable to get skeleton files: %v", err) } - for relPath, absPath := range addonFiles { - bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() + for relPath, absPath := range skelFiles { + bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter() if err != nil { - return fmt.Errorf("unable to generate addon index: %v", err) + return fmt.Errorf("unable to generate skeleton index: %v", err) } file, err := os.Open(absPath) if err != nil { - return fmt.Errorf("unable to read addon file: %v", err) + return fmt.Errorf("unable to read skeleton file: %v", err) } _, err = bufio.NewReader(file).WriteTo(bufWriter) if err != nil { - return fmt.Errorf("unable to write addon file: %v", err) + return fmt.Errorf("unable to write skeleton file: %v", err) } } } diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 0585ad3c..0b319c74 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -122,7 +122,7 @@ Options: is root of directory storage to store database (`rootDir`/db), the default for downloaded packages (`rootDir`/pool) and the default for published repositories (`rootDir`/public) and - addon files (`rootDir`/addon) + skeleton files (`rootDir`/skel) * `databaseBackend`: the database config; if this config is empty, use levledb backend by default diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index e469d9a1..20fbdf1b 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -892,7 +892,7 @@ class PublishRepo33Test(BaseTest): class PublishRepo34Test(BaseTest): """ - publish repo: addon files + publish repo: skeleton files """ fixtureCmds = [ "aptly repo create local-repo", @@ -904,7 +904,7 @@ class PublishRepo34Test(BaseTest): def prepare_fixture(self): super(PublishRepo34Test, self).prepare_fixture() - self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + self.write_file(os.path.join('skel', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') def check(self): super(PublishRepo34Test, self).check() From 9ca9569714d32e600dbe00cb4c5a4c3ee820130f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 24 Apr 2024 22:06:11 +0200 Subject: [PATCH 08/12] fix build and golangci-lint --- api/publish.go | 3 +-- deb/publish.go | 4 ++-- deb/publish_test.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/api/publish.go b/api/publish.go index aa45750a..9b10e65f 100644 --- a/api/publish.go +++ b/api/publish.go @@ -477,13 +477,12 @@ func apiPublishUpdateSwitch(c *gin.Context) { } } -<<<<<<< HEAD result, err := published.Update(collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) } diff --git a/deb/publish.go b/deb/publish.go index 7205a176..df8ad10f 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -783,7 +783,7 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri } fsPath := filepath.Join(skelDir, p.Prefix, "dists", p.Distribution, component) - if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(fsPath, func(path string, _ os.FileInfo, err error) error { if err != nil { return err } @@ -813,7 +813,7 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, skelDir string) error { + collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) diff --git a/deb/publish_test.go b/deb/publish_test.go index 4ce9c4ce..c2d228ac 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -197,7 +197,7 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) { func (s *PublishedRepoSuite) TestMultiDistPool(c *C) { repo, err := NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, true) c.Assert(err, IsNil) - err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) + err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") c.Assert(err, IsNil) publishedStorage := files.NewPublishedStorage(s.root, "", "") From 5ddb718eab8a064c3dbcf5be3d381666c3f0253c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 8 Nov 2024 20:22:53 +0100 Subject: [PATCH 09/12] support ~ in rootDir --- context/context.go | 6 +++--- utils/config_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/context/context.go b/context/context.go index 296d67b9..6e0dfe67 100644 --- a/context/context.go +++ b/context/context.go @@ -297,7 +297,7 @@ func (context *AptlyContext) _database() (database.Storage, error) { if len(context.config().DatabaseBackend.DbPath) == 0 { return nil, errors.New("leveldb databaseBackend config invalid") } - dbPath := filepath.Join(context.config().RootDir, context.config().DatabaseBackend.DbPath) + dbPath := filepath.Join(context.config().GetRootDir(), context.config().DatabaseBackend.DbPath) context.database, err = goleveldb.NewDB(dbPath) case "etcd": context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL) @@ -388,7 +388,7 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool { } else { poolRoot := context.config().PackagePoolStorage.Local.Path if poolRoot == "" { - poolRoot = filepath.Join(context.config().RootDir, "pool") + poolRoot = filepath.Join(context.config().GetRootDir(), "pool") } context.packagePool = files.NewPackagePool(poolRoot, !context.config().SkipLegacyPool) @@ -529,7 +529,7 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier { // SkelPath builds the local skeleton folder func (context *AptlyContext) SkelPath() string { - return filepath.Join(context.config().RootDir, "skel") + return filepath.Join(context.config().GetRootDir(), "skel") } // UpdateFlags sets internal copy of flags in the context diff --git a/utils/config_test.go b/utils/config_test.go index badff769..ddfa4ba9 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -21,7 +21,7 @@ func (s *ConfigSuite) TestLoadConfig(c *C) { err := LoadConfig(configname, &s.config) c.Assert(err, IsNil) - c.Check(s.config.RootDir, Equals, "/opt/aptly/") + c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/") c.Check(s.config.DownloadConcurrency, Equals, 33) c.Check(s.config.DatabaseOpenAttempts, Equals, 33) } From d2b9adf6f2c79fc8dd80ea7450a21244cde162cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 17 Nov 2024 14:08:49 +0100 Subject: [PATCH 10/12] Add a test to confirm that skel files added with the same path as a repository file do not override the repository file. Co-authored-by: iofq --- system/t06_publish/repo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 20fbdf1b..f882a428 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -905,6 +905,7 @@ class PublishRepo34Test(BaseTest): super(PublishRepo34Test, self).prepare_fixture() self.write_file(os.path.join('skel', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + self.write_file(os.path.join('skel', 'dists', 'maverick', 'Release'), 'Release test file') def check(self): super(PublishRepo34Test, self).check() @@ -917,6 +918,10 @@ class PublishRepo34Test(BaseTest): if readme != 'README test file': raise Exception("README file not copied on publish") + release = self.read_file('public/dists/maverick/Release') + if release == 'Release test file': + raise Exception("Release file was copied on publish") + release = self.read_file('public/dists/maverick/Release').split("\n") release = [l for l in release if l.startswith(" ")] pathsSeen = set() From e2cbd637b82a153a6756f2af0519e8fe769ee9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 11 Nov 2024 15:03:02 +0100 Subject: [PATCH 11/12] use new azure-sdk --- azure/azure.go | 85 +++++++++++++------------ azure/package_pool.go | 45 +++++++------ azure/package_pool_test.go | 8 ++- azure/public.go | 125 ++++++++++++++++++++----------------- azure/public_test.go | 49 ++++++++------- context/context.go | 1 - deb/list.go | 1 - go.mod | 8 +-- go.sum | 49 ++++++--------- 9 files changed, 187 insertions(+), 184 deletions(-) diff --git a/azure/azure.go b/azure/azure.go index b313f904..3f12678b 100644 --- a/azure/azure.go +++ b/azure/azure.go @@ -5,28 +5,35 @@ package azure import ( "context" "encoding/hex" + "errors" "fmt" "io" - "net/url" + "os" "path/filepath" "time" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/aptly-dev/aptly/aptly" ) func isBlobNotFound(err error) bool { - storageError, ok := err.(azblob.StorageError) - return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 404 // BlobNotFound + } + return false } type azContext struct { - container azblob.ContainerURL + client *azblob.Client + container string prefix string } func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) { - credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return nil, err } @@ -35,15 +42,14 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) ( endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName) } - url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container)) + serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil) if err != nil { return nil, err } - containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{})) - result := &azContext{ - container: containerURL, + client: serviceClient, + container: container, prefix: prefix, } @@ -54,10 +60,6 @@ func (az *azContext) blobPath(path string) string { return filepath.Join(az.prefix, path) } -func (az *azContext) blobURL(path string) azblob.BlobURL { - return az.container.NewBlobURL(az.blobPath(path)) -} - func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) { const delimiter = "/" paths = make([]string, 0, 1024) @@ -67,27 +69,33 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p prefix += delimiter } - for marker := (azblob.Marker{}); marker.NotDone(); { - listBlob, err := az.container.ListBlobsFlatSegment( - context.Background(), marker, azblob.ListBlobsSegmentOptions{ - Prefix: prefix, - MaxResults: 1, - Details: azblob.BlobListingDetails{Metadata: true}}) + ctx := context.Background() + maxResults := int32(1) + pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + MaxResults: &maxResults, + Include: azblob.ListBlobsInclude{Metadata: true}, + }) + + // Iterate over each page + for pager.More() { + page, err := pager.NextPage(ctx) if err != nil { return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err) } - marker = listBlob.NextMarker - - for _, blob := range listBlob.Segment.BlobItems { + for _, blob := range page.Segment.BlobItems { if prefix == "" { - paths = append(paths, blob.Name) + paths = append(paths, *blob.Name) } else { - paths = append(paths, blob.Name[len(prefix):]) + name := *blob.Name + paths = append(paths, name[len(prefix):]) } - md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5)) - } + b := *blob + md5 := b.Properties.ContentMD5 + md5s = append(md5s, fmt.Sprintf("%x", md5)) + } if progress != nil { time.Sleep(time.Duration(500) * time.Millisecond) progress.AddBar(1) @@ -97,28 +105,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p return paths, md5s, nil } -func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error { - uploadOptions := azblob.UploadStreamToBlockBlobOptions{ - BufferSize: 4 * 1024 * 1024, - MaxBuffers: 8, +func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error { + uploadOptions := &azblob.UploadFileOptions{ + BlockSize: 4 * 1024 * 1024, + Concurrency: 8, } + path := az.blobPath(blobName) if len(sourceMD5) > 0 { decodedMD5, err := hex.DecodeString(sourceMD5) if err != nil { return err } - uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{ - ContentMD5: decodedMD5, + uploadOptions.HTTPHeaders = &blob.HTTPHeaders{ + BlobContentMD5: decodedMD5, } } - _, err := azblob.UploadStreamToBlockBlob( - context.Background(), - source, - blob.ToBlockBlobURL(), - uploadOptions, - ) + var err error + if file, ok := source.(*os.File); ok { + _, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions) + } return err } diff --git a/azure/package_pool.go b/azure/package_pool.go index 167ab6be..ad32a11f 100644 --- a/azure/package_pool.go +++ b/azure/package_pool.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" - "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" "github.com/pkg/errors" @@ -41,10 +40,7 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename) } -func (pool *PackagePool) ensureChecksums( - poolPath string, - checksumStorage aptly.ChecksumStorage, -) (*utils.ChecksumInfo, error) { +func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) { targetChecksums, err := checksumStorage.Get(poolPath) if err != nil { return nil, err @@ -52,8 +48,7 @@ func (pool *PackagePool) ensureChecksums( if targetChecksums == nil { // we don't have checksums stored yet for this file - blob := pool.az.blobURL(poolPath) - download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil) if err != nil { if isBlobNotFound(err) { return nil, nil @@ -63,7 +58,7 @@ func (pool *PackagePool) ensureChecksums( } targetChecksums = &utils.ChecksumInfo{} - *targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{})) + *targetChecksums, err = utils.ChecksumsForReader(download.Body) if err != nil { return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath) } @@ -92,46 +87,49 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er } func (pool *PackagePool) Size(path string) (int64, error) { - blob := pool.az.blobURL(path) - props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := pool.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(pool.az.container) + blobClient := containerClient.NewBlobClient(path) + + props, err := blobClient.GetProperties(context.TODO(), nil) if err != nil { return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) } - return props.ContentLength(), nil + return *props.ContentLength, nil } func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { - blob := pool.az.blobURL(path) - temp, err := os.CreateTemp("", "blob-download") if err != nil { - return nil, errors.Wrap(err, "error creating temporary file for blob download") + return nil, errors.Wrapf(err, "error creating tempfile for %s", path) } - defer os.Remove(temp.Name()) - err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{}) + _, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil) if err != nil { - return nil, errors.Wrapf(err, "error downloading blob at %s", path) + return nil, errors.Wrapf(err, "error downloading blob %s", path) } return temp, nil } func (pool *PackagePool) Remove(path string) (int64, error) { - blob := pool.az.blobURL(path) - props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := pool.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(pool.az.container) + blobClient := containerClient.NewBlobClient(path) + + props, err := blobClient.GetProperties(context.TODO(), nil) if err != nil { - return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool) + return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) } - _, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + _, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil) if err != nil { return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool) } - return props.ContentLength(), nil + return *props.ContentLength, nil } func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) { @@ -145,7 +143,6 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } path := pool.buildPoolPath(basename, checksums) - blob := pool.az.blobURL(path) targetChecksums, err := pool.ensureChecksums(path, checksumStorage) if err != nil { return "", err @@ -161,7 +158,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } defer source.Close() - err = pool.az.putFile(blob, source, checksums.MD5) + err = pool.az.putFile(path, source, checksums.MD5) if err != nil { return "", err } diff --git a/azure/package_pool_test.go b/azure/package_pool_test.go index 58949ec6..a32bbff7 100644 --- a/azure/package_pool_test.go +++ b/azure/package_pool_test.go @@ -7,7 +7,7 @@ import ( "path/filepath" "runtime" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" @@ -50,8 +50,10 @@ func (s *PackagePoolSuite) SetUpTest(c *C) { s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - cnt := s.pool.az.container - _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) + publicAccessType := azblob.PublicAccessTypeContainer + _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{ + Access: &publicAccessType, + }) c.Assert(err, IsNil) s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint) diff --git a/azure/public.go b/azure/public.go index 50a82029..6bda2599 100644 --- a/azure/public.go +++ b/azure/public.go @@ -3,20 +3,21 @@ package azure import ( "context" "fmt" - "net/http" "os" "path/filepath" "time" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" + "github.com/pborman/uuid" "github.com/pkg/errors" ) // PublishedStorage abstract file system with published files (actually hosted on Azure) type PublishedStorage struct { - container azblob.ContainerURL prefix string az *azContext pathCache map[string]map[string]string @@ -66,7 +67,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err } defer source.Close() - err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5) + err = storage.az.putFile(path, source, sourceMD5) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) } @@ -76,14 +77,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err // RemoveDirs removes directory structure under public path func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error { + path = storage.az.blobPath(path) filelist, err := storage.Filelist(path) if err != nil { return err } for _, filename := range filelist { - blob := storage.az.blobURL(filepath.Join(path, filename)) - _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + blob := filepath.Join(path, filename) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil) if err != nil { return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err) } @@ -94,8 +96,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // Remove removes single file under public path func (storage *PublishedStorage) Remove(path string) error { - blob := storage.az.blobURL(path) - _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + path = storage.az.blobPath(path) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err)) } @@ -114,9 +116,8 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { relFilePath := filepath.Join(publishedRelPath, fileName) - // prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) - // FIXME: check how to integrate publishedPrefix: - poolPath := storage.az.blobPath(fileName) + prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) + poolPath := storage.az.blobPath(prefixRelFilePath) if storage.pathCache == nil { storage.pathCache = make(map[string]map[string]string) @@ -159,7 +160,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } defer source.Close() - err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5) + err = storage.az.putFile(relFilePath, source, sourceMD5) if err == nil { pathCache[relFilePath] = sourceMD5 } else { @@ -176,57 +177,58 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { } // Internal copy or move implementation -func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error { +func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error { const leaseDuration = 30 + leaseID := uuid.NewRandom().String() - dstBlobURL := storage.az.blobURL(dst) - srcBlobURL := storage.az.blobURL(src) - leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{}) - if err != nil || leaseResp.StatusCode() != http.StatusCreated { - return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + srcBlobClient := containerClient.NewBlobClient(src) + blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)}) + if err != nil { + return fmt.Errorf("error acquiring lease on source blob %s", src) } - defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) - srcBlobLeaseID := leaseResp.LeaseID() - copyResp, err := dstBlobURL.StartCopyFromURL( - context.Background(), - srcBlobURL.URL(), - metadata, - azblob.ModifiedAccessConditions{}, - azblob.BlobAccessConditions{}, - azblob.DefaultAccessTier, - nil) + _, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil) + if err != nil { + return fmt.Errorf("error acquiring lease on source blob %s", src) + } + defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))}) + + dstBlobClient := containerClient.NewBlobClient(dst) + copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{ + Metadata: metadata, + }) + if err != nil { return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err) } - copyStatus := copyResp.CopyStatus() + copyStatus := *copyResp.CopyStatus for { - if copyStatus == azblob.CopyStatusSuccess { + if copyStatus == blob.CopyStatusTypeSuccess { if move { - _, err = srcBlobURL.Delete( - context.Background(), - azblob.DeleteSnapshotsOptionNone, - azblob.BlobAccessConditions{ - LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}, - }) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{ + AccessConditions: &blob.AccessConditions{ + LeaseAccessConditions: &blob.LeaseAccessConditions{ + LeaseID: &leaseID, + }, + }, + }) return err } return nil - } else if copyStatus == azblob.CopyStatusPending { + } else if copyStatus == blob.CopyStatusTypePending { time.Sleep(1 * time.Second) - blobPropsResp, err := dstBlobURL.GetProperties( - context.Background(), - azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}}, - azblob.ClientProvidedKeyOptions{}) + getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil) if err != nil { - return fmt.Errorf("error getting destination blob properties %s", dstBlobURL) + return fmt.Errorf("error getting copy progress %s", dst) } - copyStatus = blobPropsResp.CopyStatus() + copyStatus = *getMetadata.CopyStatus - _, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{}) + _, err = blobLeaseClient.RenewLease(context.Background(), nil) if err != nil { - return fmt.Errorf("error renewing source blob lease %s", srcBlobURL) + return fmt.Errorf("error renewing source blob lease %s", src) } } else { return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus) @@ -241,7 +243,9 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { // SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { - return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */) + metadata := make(map[string]*string) + metadata["SymLink"] = &src + return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */) } // HardLink using symlink functionality as hard links do not exist @@ -251,28 +255,33 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) (bool, error) { - blob := storage.az.blobURL(path) - resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + blobClient := containerClient.NewBlobClient(path) + _, err := blobClient.GetProperties(context.Background(), nil) if err != nil { if isBlobNotFound(err) { return false, nil } - return false, err - } else if resp.StatusCode() == http.StatusOK { - return true, nil + return false, fmt.Errorf("error checking if blob %s exists: %v", path, err) } - return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) + return true, nil } // ReadLink returns the symbolic link pointed to by path. // This simply reads text file created with SymLink func (storage *PublishedStorage) ReadLink(path string) (string, error) { - blob := storage.az.blobURL(path) - resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + blobClient := containerClient.NewBlobClient(path) + props, err := blobClient.GetProperties(context.Background(), nil) if err != nil { - return "", err - } else if resp.StatusCode() != http.StatusOK { - return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) + return "", fmt.Errorf("failed to get blob properties: %v", err) } - return resp.NewMetadata()["SymLink"], nil + + metadata := props.Metadata + if originalBlob, exists := metadata["original_blob"]; exists { + return *originalBlob, nil + } + return "", fmt.Errorf("error reading link %s: %v", path, err) } diff --git a/azure/public_test.go b/azure/public_test.go index 767f2495..3acd767f 100644 --- a/azure/public_test.go +++ b/azure/public_test.go @@ -7,8 +7,11 @@ import ( "io/ioutil" "os" "path/filepath" + "bytes" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" . "gopkg.in/check.v1" @@ -66,8 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - cnt := s.storage.az.container - _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) + publicAccessType := azblob.PublicAccessTypeContainer + _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{ + Access: &publicAccessType, + }) c.Assert(err, IsNil) s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint) @@ -75,41 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { } func (s *PublishedStorageSuite) TearDownTest(c *C) { - cnt := s.storage.az.container - _, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{}) + _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil) c.Assert(err, IsNil) } func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { - blob := s.storage.az.container.NewBlobURL(path) - resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil) c.Assert(err, IsNil) - body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}) - data, err := ioutil.ReadAll(body) + data, err := ioutil.ReadAll(resp.Body) c.Assert(err, IsNil) return data } func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { - _, err := s.storage.az.container.NewBlobURL(path).GetProperties( - context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := s.storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(s.storage.az.container) + blobClient := containerClient.NewBlobClient(path) + _, err := blobClient.GetProperties(context.Background(), nil) c.Assert(err, NotNil) - storageError, ok := err.(azblob.StorageError) + + storageError, ok := err.(*azcore.ResponseError) c.Assert(ok, Equals, true) - c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound))) + c.Assert(storageError.StatusCode, Equals, 404) } func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { hash := md5.Sum(data) - _, err := azblob.UploadBufferToBlockBlob( - context.Background(), - data, - s.storage.az.container.NewBlockBlobURL(path), - azblob.UploadToBlockBlobOptions{ - BlobHTTPHeaders: azblob.BlobHTTPHeaders{ - ContentMD5: hash[:], - }, - }) + uploadOptions := &azblob.UploadStreamOptions{ + HTTPHeaders: &blob.HTTPHeaders{ + BlobContentMD5: hash[:], + }, + } + reader := bytes.NewReader(data) + _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions) c.Assert(err, IsNil) } @@ -330,7 +333,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { // 2nd link from pool, providing wrong path for source file // - // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) + // this test should check that file already exists in Azure and skip upload (which would fail if not skipped) s.prefixedStorage.pathCache = nil err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) diff --git a/context/context.go b/context/context.go index 6e0dfe67..e599102d 100644 --- a/context/context.go +++ b/context/context.go @@ -101,7 +101,6 @@ func (context *AptlyContext) config() *utils.ConfigStructure { configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"} for _, configLocation := range configLocations { - // FIXME: check if exists, check if readable err = utils.LoadConfig(configLocation, &utils.Config) if os.IsPermission(err) || os.IsNotExist(err) { continue diff --git a/deb/list.go b/deb/list.go index ada6fab7..4f863ccf 100644 --- a/deb/list.go +++ b/deb/list.go @@ -549,7 +549,6 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie // // when follow-all-variants is enabled, we need to try to expand anyway, // as even if dependency is satisfied now, there might be other ways to satisfy dependency - // FIXME: do not search twice if result.Search(dep, false, true) != nil { if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) diff --git a/go.mod b/go.mod index d053f1df..b1e030c0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.22.7 require ( github.com/AlekSi/pointer v1.1.0 - github.com/Azure/azure-storage-blob-go v0.15.0 github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 github.com/awalterschulze/gographviz v2.0.1+incompatible github.com/cavaliergopher/grab/v3 v3.0.1 @@ -43,7 +42,7 @@ require ( ) require ( - github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -89,7 +88,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-ieproxy v0.0.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -99,7 +97,7 @@ require ( github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/etcd/api/v3 v3.5.15 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect @@ -119,6 +117,8 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 github.com/ProtonMail/go-crypto v1.0.0 github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.25.1 diff --git a/go.sum b/go.sum index ac476e31..43c4b545 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,17 @@ github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= -github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= -github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 h1:jbGlDKdzAZ92NzK65hUP98ri0/r50vVVvmZsFP/nIqo= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -91,8 +88,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -125,6 +120,8 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -144,7 +141,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -190,9 +186,6 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-ieproxy v0.0.9 h1:RvVbLiMv/Hbjf1gRaC2AQyzwbdVhdId7D2vPnXIml4k= -github.com/mattn/go-ieproxy v0.0.9/go.mod h1:eF30/rfdQUO9EnzNIZQr0r9HiLMlZNCpJkHbmMuOAE0= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -228,6 +221,8 @@ github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -245,8 +240,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= @@ -306,8 +301,6 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -322,15 +315,12 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -352,7 +342,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -360,12 +349,10 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From d80b905945171ca16544ac21adefee8093848221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 13 Nov 2024 13:04:45 +0100 Subject: [PATCH 12/12] run azure tests in docker --- Makefile | 23 +++++++++++++++++++++-- system/Dockerfile | 4 +++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5294db9a..632d7066 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,13 @@ swagger-install: # Install swag @test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest +azurite-start: + azurite & \ + echo $$! > ~/.azurite.pid + +azurite-stop: + kill `cat ~/.azurite.pid` + swagger: swagger-install # Generate swagger docs @PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs @@ -160,10 +167,22 @@ docker-deb: ## Build debian packages in docker container @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 docker-unit-test: ## Run unit tests in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper test + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + azurite-start \ + AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ + AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ + AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + test \ + azurite-stop docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper system-test TEST=$(TEST) + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + azurite-start \ + AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ + AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ + AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + system-test TEST=$(TEST) \ + azurite-stop docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true diff --git a/system/Dockerfile b/system/Dockerfile index b2da26e3..f2435244 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports \ make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage \ g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu \ - binutils-arm-linux-gnueabihf bash-completion zip ruby3.1-dev lintian \ + binutils-arm-linux-gnueabihf bash-completion zip ruby3.1-dev lintian npm \ libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \ gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu && \ apt-get clean && rm -rf /var/lib/apt/lists/* @@ -18,6 +18,7 @@ RUN mkdir /work WORKDIR /work/src RUN chown aptly /work +# install test dependencies RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-db.git RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-pool.git RUN cd /var/lib/aptly; curl -O http://repo.aptly.info/system-tests/etcd.db.xz && xz -d etcd.db.xz @@ -25,3 +26,4 @@ RUN echo "aptly ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/aptly ADD system/t13_etcd/install-etcd.sh /src/ RUN /src/install-etcd.sh RUN rm -rf /src +RUN npm install -g azurite