From b4cd86aa14e0fd7386b286406560e217b6defecb Mon Sep 17 00:00:00 2001 From: Noa Resare Date: Sat, 18 May 2024 17:59:55 +0100 Subject: [PATCH] Introduce option multi-dist to the publish commands This change makes it possible to publish multiple distributions with packages named the same but with different content by changing structure of the generated pool hierarchy. The option not enabled by default as this changes the structure of the output which could break the expectations of other tools. --- AUTHORS | 1 + api/publish.go | 6 ++-- cmd/publish_repo.go | 1 + cmd/publish_snapshot.go | 4 ++- cmd/publish_switch.go | 4 ++- cmd/publish_update.go | 4 ++- completion.d/aptly | 2 +- deb/publish.go | 9 ++++-- deb/publish_test.go | 62 +++++++++++++++++++++++++++++++++++++---- 9 files changed, 80 insertions(+), 13 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5f398d66..af67c810 100644 --- a/AUTHORS +++ b/AUTHORS @@ -59,3 +59,4 @@ List of contributors, in chronological order: * Paul Cacheux (https://github.com/paulcacheux) * Nic Waller (https://github.com/sf-nwaller) * iofq (https://github.com/iofq) +* Noa Resare (https://github.com/nresare) diff --git a/api/publish.go b/api/publish.go index c158a3a5..adce3c84 100644 --- a/api/publish.go +++ b/api/publish.go @@ -101,6 +101,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { Architectures []string Signing SigningOptions AcquireByHash *bool + MultiDist bool } if c.Bind(&b) != nil { @@ -226,7 +227,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite) + err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, b.MultiDist) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } @@ -257,6 +258,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { Name string `binding:"required"` } AcquireByHash *bool + MultiDist bool } if c.Bind(&b) != nil { @@ -341,7 +343,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { resources = append(resources, string(published.Key())) taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", ")) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/publish_repo.go b/cmd/publish_repo.go index f2f30472..91971022 100644 --- a/cmd/publish_repo.go +++ b/cmd/publish_repo.go @@ -51,6 +51,7 @@ Example: cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") return cmd } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 3f115541..1e79fac2 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -116,6 +116,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { origin := context.Flags().Lookup("origin").Value.String() notAutomatic := context.Flags().Lookup("notautomatic").Value.String() butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String() + multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool) published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory) if err != nil { @@ -165,7 +166,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist) if err != nil { return fmt.Errorf("unable to publish: %s", err) } @@ -242,6 +243,7 @@ Example: cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") return cmd } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 0784fba3..58a1195c 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -14,6 +14,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { var err error components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") + multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool) if len(args) < len(components)+1 || len(args) > len(components)+2 { cmd.Usage() @@ -100,7 +101,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist) if err != nil { return fmt.Errorf("unable to publish: %s", err) } @@ -161,6 +162,7 @@ This command would switch published repository (with one component) named ppa/wh cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") return cmd } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index fcdea8ed..9bceda50 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -17,6 +17,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { distribution := args[0] param := "." + multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool) if len(args) == 2 { param = args[1] @@ -64,7 +65,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist) if err != nil { return fmt.Errorf("unable to publish: %s", err) } @@ -119,6 +120,7 @@ Example: cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") return cmd } diff --git a/completion.d/aptly b/completion.d/aptly index c0276ad3..dd2e8040 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -503,7 +503,7 @@ _aptly() "snapshot"|"repo") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur})) + COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur})) else if [[ "$subcmd" == "snapshot" ]]; then COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur})) diff --git a/deb/publish.go b/deb/publish.go index eb890e70..0098bb21 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -544,7 +544,7 @@ func (p *PublishedRepo) GetCodename() string { // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error { + collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, multiDist bool) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -656,7 +656,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP if err2 != nil { return err2 } - relPath = filepath.Join("pool", component, poolDir) + if multiDist { + relPath = filepath.Join("pool", p.Distribution, component, poolDir) + } else { + relPath = filepath.Join("pool", component, poolDir) + } + } else { if p.Distribution == aptly.DistributionFocal { relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "legacy-images") diff --git a/deb/publish_test.go b/deb/publish_test.go index 94ff0fa5..6fdad759 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -193,6 +193,58 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) { c.Check(err, IsNil) } +func (s *PublishedRepoSuite) TestMultiDistPool(c *C) { + repo, err := NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory) + c.Assert(err, IsNil) + err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, true) + c.Assert(err, IsNil) + + publishedStorage := files.NewPublishedStorage(s.root, "", "") + + c.Check(repo.Architectures, DeepEquals, []string{"i386"}) + + rf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/Release")) + c.Assert(err, IsNil) + + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() + c.Assert(err, IsNil) + + c.Check(st["Origin"], Equals, "ppa squeeze") + c.Check(st["Components"], Equals, "main") + c.Check(st["Architectures"], Equals, "i386") + + pf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages")) + c.Assert(err, IsNil) + + cfr = NewControlFileReader(pf, false, false) + + for i := 0; i < 3; i++ { + st, err = cfr.ReadStanza() + c.Assert(err, IsNil) + + c.Check(st["Filename"], Equals, "pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb") + } + + st, err = cfr.ReadStanza() + c.Assert(err, IsNil) + c.Assert(st, IsNil) + + drf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release")) + c.Assert(err, IsNil) + + cfr = NewControlFileReader(drf, true, false) + st, err = cfr.ReadStanza() + c.Assert(err, IsNil) + + c.Check(st["Archive"], Equals, "squeeze") + c.Check(st["Architecture"], Equals, "i386") + + _, err = os.Stat(filepath.Join(publishedStorage.PublicPath(), "ppa/pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")) + c.Assert(err, IsNil) + +} + func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) { for _, t := range []struct { @@ -308,7 +360,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) { } func (s *PublishedRepoSuite) TestPublish(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, false) c.Assert(err, IsNil) c.Check(s.repo.Architectures, DeepEquals, []string{"i386"}) @@ -355,7 +407,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { } func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists) @@ -363,7 +415,7 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { - err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -371,7 +423,7 @@ func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { - err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -379,7 +431,7 @@ func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) { - err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)