From d6a3917141d27df4aeda639ed52898840f08a265 Mon Sep 17 00:00:00 2001 From: Ludovico Cavedon Date: Mon, 14 Aug 2017 15:49:50 -0700 Subject: [PATCH] Add -skip-cleanup option for publish commands. Allow skipping unreferenced files cleanup on publish switch/update/drop via the -skip-cleanup command line option. Also support API SkipCleanup parameter. Fixes #570. --- AUTHORS | 1 + api/publish.go | 16 +- bash_completion.d/aptly | 6 +- cmd/publish_drop.go | 5 +- cmd/publish_switch.go | 12 +- cmd/publish_update.go | 12 +- deb/publish.go | 4 +- deb/publish_test.go | 48 ++++- system/t06_publish/PublishDrop8Test_gold | 3 + system/t06_publish/PublishDrop9Test_gold | 4 + system/t06_publish/PublishSwitch14Test_binary | 29 +++ system/t06_publish/PublishSwitch14Test_gold | 7 + .../t06_publish/PublishSwitch14Test_release | 11 + system/t06_publish/PublishUpdate12Test_binary | 27 +++ system/t06_publish/PublishUpdate12Test_gold | 7 + .../t06_publish/PublishUpdate12Test_release | 11 + .../t06_publish/PublishUpdate12Test_sources | 0 system/t06_publish/drop.py | 63 ++++++ system/t06_publish/switch.py | 82 ++++++++ system/t06_publish/update.py | 80 +++++++ system/t12_api/publish.py | 198 ++++++++++++++++++ 21 files changed, 601 insertions(+), 25 deletions(-) create mode 100644 system/t06_publish/PublishDrop8Test_gold create mode 100644 system/t06_publish/PublishDrop9Test_gold create mode 100644 system/t06_publish/PublishSwitch14Test_binary create mode 100644 system/t06_publish/PublishSwitch14Test_gold create mode 100644 system/t06_publish/PublishSwitch14Test_release create mode 100644 system/t06_publish/PublishUpdate12Test_binary create mode 100644 system/t06_publish/PublishUpdate12Test_gold create mode 100644 system/t06_publish/PublishUpdate12Test_release create mode 100644 system/t06_publish/PublishUpdate12Test_sources diff --git a/AUTHORS b/AUTHORS index 0be46dc4..7655f3f6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -28,3 +28,4 @@ List of contributors, in chronological order: * Charles Hsu (https://github.com/charz) * Clemens Rabe (https://github.com/seeraven) * TJ Merritt (https://github.com/tjmerritt) +* Ludovico Cavedon (https://github.com/cavedon) diff --git a/api/publish.go b/api/publish.go index 6210d252..016fa4ce 100644 --- a/api/publish.go +++ b/api/publish.go @@ -233,6 +233,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { ForceOverwrite bool Signing SigningOptions SkipContents *bool + SkipCleanup *bool Snapshots []struct { Component string `binding:"required"` Name string `binding:"required"` @@ -328,11 +329,13 @@ func apiPublishUpdateSwitch(c *gin.Context) { return } - err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, - context.GetPublishedStorage(storage), context.CollectionFactory(), nil) - if err != nil { - c.Fail(500, fmt.Errorf("unable to update: %s", err)) - return + if b.SkipCleanup == nil || !*b.SkipCleanup { + err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, + context.GetPublishedStorage(storage), context.CollectionFactory(), nil) + if err != nil { + c.Fail(500, fmt.Errorf("unable to update: %s", err)) + return + } } c.JSON(200, published) @@ -341,6 +344,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { // DELETE /publish/:prefix/:distribution func apiPublishDrop(c *gin.Context) { force := c.Request.URL.Query().Get("force") == "1" + skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1" param := parseEscapedPath(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) @@ -356,7 +360,7 @@ func apiPublishDrop(c *gin.Context) { defer collection.Unlock() err := collection.Remove(context, storage, prefix, distribution, - context.CollectionFactory(), context.Progress(), force) + context.CollectionFactory(), context.Progress(), force, skipCleanup) if err != nil { c.Fail(500, fmt.Errorf("unable to drop: %s", err)) return diff --git a/bash_completion.d/aptly b/bash_completion.d/aptly index 2bd4b0ce..cb9d9d5d 100644 --- a/bash_completion.d/aptly +++ b/bash_completion.d/aptly @@ -524,7 +524,7 @@ _aptly() "update") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur})) + COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur})) fi @@ -539,7 +539,7 @@ _aptly() "switch") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur})) + COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur})) fi @@ -559,7 +559,7 @@ _aptly() "drop") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-force-drop" -- ${cur})) + COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur})) fi diff --git a/cmd/publish_drop.go b/cmd/publish_drop.go index 95bc6ac7..6fab0352 100644 --- a/cmd/publish_drop.go +++ b/cmd/publish_drop.go @@ -24,7 +24,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error { storage, prefix := deb.ParsePrefix(param) err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution, - context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool)) + context.CollectionFactory(), context.Progress(), + context.Flags().Lookup("force-drop").Value.Get().(bool), + context.Flags().Lookup("skip-cleanup").Value.Get().(bool)) if err != nil { return fmt.Errorf("unable to remove: %s", err) } @@ -50,6 +52,7 @@ Example: } cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up") + cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") return cmd } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 86c3f2a8..c71b0c39 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -105,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to save to DB: %s", err) } - err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, - context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress()) - if err != nil { - return fmt.Errorf("unable to update: %s", err) + skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) + if !skipCleanup { + err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, + context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } } context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String()) @@ -151,6 +154,7 @@ This command would switch published repository (with one component) named ppa/wh cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes") cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") return cmd } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 587a6399..0db3cdb2 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -69,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to save to DB: %s", err) } - err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, - context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress()) - if err != nil { - return fmt.Errorf("unable to update: %s", err) + skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) + if !skipCleanup { + err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, + context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } } context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String()) @@ -109,6 +112,7 @@ Example: cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG") cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") return cmd } diff --git a/deb/publish.go b/deb/publish.go index 59d722aa..4e588540 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1097,7 +1097,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st // Remove removes published repository, cleaning up directories, files func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider, storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress, - force bool) error { + force, skipCleanup bool) error { repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { return err @@ -1134,7 +1134,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list = nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1] - if len(cleanComponents) > 0 { + if !skipCleanup && len(cleanComponents) > 0 { err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents, publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress) if err != nil { diff --git a/deb/publish_test.go b/deb/publish_test.go index eb0ff55b..c28d1f5f 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -756,7 +756,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { - err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false) + err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda") @@ -776,10 +776,48 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists) - err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false) + err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false) c.Check(err, ErrorMatches, ".*not found") - err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false) + err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, false) + c.Check(err, IsNil) + + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists)) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists)) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists) + c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists) + c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists) +} + +func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2SkipCleanup(c *C) { + err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true) + c.Check(err, IsNil) + + _, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda") + c.Check(err, ErrorMatches, ".*not found") + + collection := NewPublishedRepoCollection(s.db) + _, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda") + c.Check(err, ErrorMatches, ".*not found") + + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists) + c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists) + c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists) + c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists) + + err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true) + c.Check(err, ErrorMatches, ".*not found") + + err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, true) c.Check(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) @@ -794,7 +832,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) { - err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false) + err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda") @@ -816,7 +854,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) { - err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false) + err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog") diff --git a/system/t06_publish/PublishDrop8Test_gold b/system/t06_publish/PublishDrop8Test_gold new file mode 100644 index 00000000..fd9e6c56 --- /dev/null +++ b/system/t06_publish/PublishDrop8Test_gold @@ -0,0 +1,3 @@ +Removing ${HOME}/.aptly/public/dists/sq2... + +Published repository has been removed successfully. diff --git a/system/t06_publish/PublishDrop9Test_gold b/system/t06_publish/PublishDrop9Test_gold new file mode 100644 index 00000000..eb8ce9c0 --- /dev/null +++ b/system/t06_publish/PublishDrop9Test_gold @@ -0,0 +1,4 @@ +Removing ${HOME}/.aptly/public/dists/sq1... +Cleaning up prefix "." components main... + +Published repository has been removed successfully. diff --git a/system/t06_publish/PublishSwitch14Test_binary b/system/t06_publish/PublishSwitch14Test_binary new file mode 100644 index 00000000..70d9c276 --- /dev/null +++ b/system/t06_publish/PublishSwitch14Test_binary @@ -0,0 +1,29 @@ + + . + . + C-like language. Can perform smoothing, spline-fitting, or nonlinear fits, + Data files and self-defined functions can be manipulated by the internal + Gnuplot is a portable command-line driven interactive data and function + This package contains the terminal driver that enables gnuplot to plot + and can work with complex numbers. + for many printers, (La)TeX, (x)fig, Postscript, and so on. The X11-output + gnuplot. + images interactively under X11. Most users will want this, it is however + is packaged in gnuplot-x11. + packaged separately so that low-end systems don't need X installed to use + plotting utility that supports lots of output formats, including drivers +Architecture: i386 +Depends: gnuplot-nox (>= 4.6.1-1~maverick2), libc6 (>= 2.11), libcairo2 (>= 1.6.0), libedit2 (>= 2.5.cvs.20010821-1), libgcc1 (>= 1:4.1.1), libgd2-noxpm (>= 2.0.36~rc1~dfsg) | libgd2-xpm (>= 2.0.36~rc1~dfsg), libglib2.0-0 (>= 2.12.0), liblua5.1-0, libpango1.0-0 (>= 1.14.0), libstdc++6 (>= 4.1.1), libwxbase2.8-0 (>= 2.8.11.0), libwxgtk2.8-0 (>= 2.8.11.0), libx11-6 +Description: Command-line driven interactive plotting program +Filename: pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb +Installed-Size: 1604 +MD5sum: fcad938905d0ace50a6ce0c73b2c6583 +Maintainer: Debian Science Team +Package: gnuplot-x11 +Priority: optional +Replaces: gnuplot (<< 4.0.0) +SHA1: 02f9a93097a8f798a054e26154dbe5789088c069 +Section: math +Size: 724388 +Source: gnuplot +Version: 4.6.1-1~maverick2 diff --git a/system/t06_publish/PublishSwitch14Test_gold b/system/t06_publish/PublishSwitch14Test_gold new file mode 100644 index 00000000..9b8ce8f1 --- /dev/null +++ b/system/t06_publish/PublishSwitch14Test_gold @@ -0,0 +1,7 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. diff --git a/system/t06_publish/PublishSwitch14Test_release b/system/t06_publish/PublishSwitch14Test_release new file mode 100644 index 00000000..91ff3759 --- /dev/null +++ b/system/t06_publish/PublishSwitch14Test_release @@ -0,0 +1,11 @@ +Origin: LP-PPA-gladky-anton-gnuplot +Label: . maverick +Suite: maverick +Codename: maverick +Architectures: amd64 i386 +Components: main +Description: Generated by aptly +MD5Sum: +SHA1: +SHA256: +SHA512: diff --git a/system/t06_publish/PublishUpdate12Test_binary b/system/t06_publish/PublishUpdate12Test_binary new file mode 100644 index 00000000..1fd182e4 --- /dev/null +++ b/system/t06_publish/PublishUpdate12Test_binary @@ -0,0 +1,27 @@ + + + (name, value) pairs from the user, via conventional methods such as + . + . + Boost version (currently 1.49). + Library to let program developers obtain program options, that is + This package forms part of the Boost C++ Libraries collection. + This package is a dependency package, which depends on Debian's default + command line and config file. +Architecture: i386 +Depends: libboost-program-options1.49-dev +Description: program options library for C++ (default version) +Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb +Homepage: http://www.boost.org/libs/program_options/ +Installed-Size: 26 +MD5sum: 0035d7822b2f8f0ec4013f270fd650c2 +Maintainer: Debian Boost Team +Package: libboost-program-options-dev +Priority: optional +SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b +SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12 +SHA512: d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c +Section: libdevel +Size: 2738 +Source: boost-defaults +Version: 1.49.0.1 \ No newline at end of file diff --git a/system/t06_publish/PublishUpdate12Test_gold b/system/t06_publish/PublishUpdate12Test_gold new file mode 100644 index 00000000..3b7a5709 --- /dev/null +++ b/system/t06_publish/PublishUpdate12Test_gold @@ -0,0 +1,7 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate12Test_release b/system/t06_publish/PublishUpdate12Test_release new file mode 100644 index 00000000..78154c5b --- /dev/null +++ b/system/t06_publish/PublishUpdate12Test_release @@ -0,0 +1,11 @@ +Origin: . maverick +Label: . maverick +Suite: maverick +Codename: maverick +Architectures: i386 +Components: main +Description: Generated by aptly +MD5Sum: +SHA1: +SHA256: +SHA512: diff --git a/system/t06_publish/PublishUpdate12Test_sources b/system/t06_publish/PublishUpdate12Test_sources new file mode 100644 index 00000000..e69de29b diff --git a/system/t06_publish/drop.py b/system/t06_publish/drop.py index 8660df10..8c4978e7 100644 --- a/system/t06_publish/drop.py +++ b/system/t06_publish/drop.py @@ -143,3 +143,66 @@ class PublishDrop7Test(BaseTest): self.check_not_exists('public/ppa/smira/dists/') self.check_not_exists('public/ppa/smira/pool/') self.check_exists('public/ppa/smira/') + + +class PublishDrop8Test(BaseTest): + """ + publish drop: skip component cleanup + """ + fixtureCmds = [ + "aptly repo create local1", + "aptly repo create local2", + "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb", + "aptly repo add local2 ${files}", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2", + ] + runCmd = "aptly publish drop -skip-cleanup sq2" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishDrop8Test, self).check() + + self.check_exists('public/dists/sq1') + self.check_not_exists('public/dists/sq2') + self.check_exists('public/pool/main/') + + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') + + +class PublishDrop9Test(BaseTest): + """ + publish drop: component cleanup after first cleanup skipped + """ + fixtureCmds = [ + "aptly repo create local1", + "aptly repo create local2", + "aptly repo create local3", + "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb", + "aptly repo add local2 ${files}", + "aptly repo add local3 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq3 local3", + "aptly publish drop -skip-cleanup sq2" + ] + runCmd = "aptly publish drop sq1" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishDrop9Test, self).check() + + self.check_not_exists('public/dists/sq1') + self.check_not_exists('public/dists/sq2') + self.check_exists('public/dists/sq3') + self.check_exists('public/pool/main/') + + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') diff --git a/system/t06_publish/switch.py b/system/t06_publish/switch.py index 586ace7a..21b697f6 100644 --- a/system/t06_publish/switch.py +++ b/system/t06_publish/switch.py @@ -451,3 +451,85 @@ class PublishSwitch13Test(BaseTest): self.check_not_exists('public/dists/maverick/main/Contents-i386.gz') self.check_exists('public/dists/maverick/main/binary-amd64/Packages') self.check_not_exists('public/dists/maverick/main/Contents-amd64.gz') + + +class PublishSwitch14Test(BaseTest): + """ + publish switch: removed some packages skipping cleanup + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1", + ] + runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick snap3" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSwitch14Test, self).check() + + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/Contents-i386.gz') + self.check_exists('public/dists/maverick/main/binary-amd64/Packages') + self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-amd64/Packages.bz2') + self.check_exists('public/dists/maverick/main/Contents-amd64.gz') + + self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb') + self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_amd64.deb') + self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_i386.deb') + self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_amd64.deb') + + # verify contents except of sums + self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor) + self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) + + # verify signatures + self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), + "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')]) + self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), + "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'), + os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')]) + + # verify sums + release = self.read_file('public/dists/maverick/Release').split("\n") + release = [l for l in release if l.startswith(" ")] + pathsSeen = set() + for l in release: + fileHash, fileSize, path = l.split() + pathsSeen.add(path) + + fileSize = int(fileSize) + + st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path)) + if fileSize != st.st_size: + raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size)) + + if len(fileHash) == 32: + h = hashlib.md5() + elif len(fileHash) == 40: + h = hashlib.sha1() + elif len(fileHash) == 64: + h = hashlib.sha256() + else: + h = hashlib.sha512() + + h.update(self.read_file(os.path.join('public/dists/maverick', path))) + + if h.hexdigest() != fileHash: + raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) + + if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz', + 'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2', + 'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz', + 'main/Contents-i386.gz']): + raise Exception("path seen wrong: %r" % (pathsSeen, )) diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 0c4c3eb8..480b4057 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -339,3 +339,83 @@ class PublishUpdate11Test(BaseTest): self.check_exists('public/dists/maverick/main/binary-i386/Packages') self.check_not_exists('public/dists/maverick/main/Contents-i386.gz') + + +class PublishUpdate12Test(BaseTest): + """ + publish update: removed some packages skipping cleanup + """ + fixtureCmds = [ + "aptly repo create local-repo", + "aptly repo add local-repo ${files}/", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", + "aptly repo remove local-repo pyspi" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishUpdate12Test, self).check() + + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/Contents-i386.gz') + self.check_exists('public/dists/maverick/main/source/Sources') + self.check_exists('public/dists/maverick/main/source/Sources.gz') + self.check_exists('public/dists/maverick/main/source/Sources.bz2') + + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') + + # verify contents except of sums + self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor) + self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) + self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) + + # verify signatures + self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), + "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')]) + self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), + "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'), + os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')]) + + # verify sums + release = self.read_file('public/dists/maverick/Release').split("\n") + release = [l for l in release if l.startswith(" ")] + pathsSeen = set() + for l in release: + fileHash, fileSize, path = l.split() + pathsSeen.add(path) + + fileSize = int(fileSize) + + st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path)) + if fileSize != st.st_size: + raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size)) + + if len(fileHash) == 32: + h = hashlib.md5() + elif len(fileHash) == 40: + h = hashlib.sha1() + elif len(fileHash) == 64: + h = hashlib.sha256() + else: + h = hashlib.sha512() + + h.update(self.read_file(os.path.join('public/dists/maverick', path))) + + if h.hexdigest() != fileHash: + raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) + + if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz', + 'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2', + 'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']): + raise Exception("path seen wrong: %r" % (pathsSeen, )) diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index a18f8ee3..82ecfcef 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -214,6 +214,88 @@ class PublishUpdateAPITestRepo(APITest): self.check_not_exists("public/" + prefix + "dists/") +class PublishUpdateSkipCleanupAPITestRepo(APITest): + """ + PUT /publish/:prefix/:distribution (local repos), DELETE /publish/:prefix/:distribution + """ + fixtureGpg = True + + def check(self): + repo_name = self.random_name() + self.check_equal(self.post("/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal( + self.upload("/api/files/" + d, + "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + prefix = self.random_name() + resp = self.post("/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "local", + "Sources": [{"Name": repo_name}], + "Signing": DefaultSigningOptions, + }) + + self.check_equal(resp.status_code, 201) + + self.check_not_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + # Publish two repos, so that deleting one while skipping cleanup will + # not delete the whole prefix. + resp = self.post("/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "Distribution": "otherdist", + "SourceKind": "local", + "Sources": [{"Name": repo_name}], + "Signing": DefaultSigningOptions, + }) + + self.check_equal(resp.status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + self.check_equal(self.delete("/api/repos/" + repo_name + "/packages/", + json={"PackageRefs": ['Psource pyspi 0.6.1-1.4 f8f1daa806004e89']}).status_code, 200) + + resp = self.put("/api/publish/" + prefix + "/wheezy", + json={ + "Signing": DefaultSigningOptions, + "SkipCleanup": True, + }) + repo_expected = { + 'Architectures': ['i386', 'source'], + 'Distribution': 'wheezy', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Prefix': prefix, + 'SkipContents': False, + 'SourceKind': 'local', + 'Sources': [{'Component': 'main', 'Name': repo_name}], + 'Storage': ''} + + self.check_equal(resp.status_code, 200) + self.check_equal(resp.json(), repo_expected) + + self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy", params={"SkipCleanup": "1"}).status_code, 200) + self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + class PublishSwitchAPITestRepo(APITest): """ PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution @@ -300,3 +382,119 @@ class PublishSwitchAPITestRepo(APITest): self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy").status_code, 200) self.check_not_exists("public/" + prefix + "dists/") + + +class PublishSwitchAPISkipCleanupTestRepo(APITest): + """ + PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution + """ + fixtureGpg = True + + def check(self): + repo_name = self.random_name() + self.check_equal(self.post("/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal( + self.upload("/api/files/" + d, + "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + snapshot1_name = self.random_name() + self.check_equal(self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot1_name}).status_code, 201) + + prefix = self.random_name() + resp = self.post("/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "snapshot", + "Sources": [{"Name": snapshot1_name}], + "Signing": DefaultSigningOptions, + }) + + self.check_equal(resp.status_code, 201) + repo_expected = { + 'Architectures': ['i386', 'source'], + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': '', + 'Prefix': prefix, + 'SkipContents': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot1_name}], + 'Storage': ''} + self.check_equal(resp.json(), repo_expected) + + self.check_not_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + # Publish two snapshots, so that deleting one while skipping cleanup will + # not delete the whole prefix. + resp = self.post("/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "Distribution": "otherdist", + "SourceKind": "snapshot", + "Sources": [{"Name": snapshot1_name}], + "Signing": DefaultSigningOptions, + }) + + self.check_equal(resp.status_code, 201) + repo_expected = { + 'Architectures': ['i386', 'source'], + 'Distribution': 'otherdist', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': '', + 'Prefix': prefix, + 'SkipContents': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot1_name}], + 'Storage': ''} + self.check_equal(resp.json(), repo_expected) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + self.check_equal(self.delete("/api/repos/" + repo_name + "/packages/", + json={"PackageRefs": ['Psource pyspi 0.6.1-1.4 f8f1daa806004e89']}).status_code, 200) + + snapshot2_name = self.random_name() + self.check_equal(self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot2_name}).status_code, 201) + + resp = self.put("/api/publish/" + prefix + "/wheezy", + json={ + "Snapshots": [{"Component": "main", "Name": snapshot2_name}], + "Signing": DefaultSigningOptions, + "SkipCleanup": True, + "SkipContents": True, + }) + repo_expected = { + 'Architectures': ['i386', 'source'], + 'Distribution': 'wheezy', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Prefix': prefix, + 'SkipContents': True, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot2_name}], + 'Storage': ''} + + self.check_equal(resp.status_code, 200) + self.check_equal(resp.json(), repo_expected) + + self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy", params={"SkipCleanup": "1"}).status_code, 200) + self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")