Merge pull request #620 from cavedon/skipCleanup

Add -skip-cleanup option for publish commands.

AUTHORS

@@ -29,3 +29,4 @@ List of contributors, in chronological order:
 * Clemens Rabe (https://github.com/seeraven)
 * TJ Merritt (https://github.com/tjmerritt)
 * Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)

api/publish.go

@@ -233,6 +233,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		ForceOverwrite bool
 		Signing        SigningOptions
 		SkipContents   *bool
+		SkipCleanup    *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
@@ -328,11 +329,13 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
-	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+	if b.SkipCleanup == nil || !*b.SkipCleanup {
-		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
-	if err != nil {
+			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		if err != nil {
-		return
+			c.Fail(500, fmt.Errorf("unable to update: %s", err))
+			return
+		}
 	}
 
 	c.JSON(200, published)
@@ -341,6 +344,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
 
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
@@ -356,7 +360,7 @@ func apiPublishDrop(c *gin.Context) {
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), force)
+		context.CollectionFactory(), context.Progress(), force, skipCleanup)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
 		return

bash_completion.d/aptly

@@ -524,7 +524,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -539,7 +539,7 @@ _aptly()
           "switch")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -559,7 +559,7 @@ _aptly()
           "drop")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force-drop" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi

cmd/publish_drop.go

@@ -24,7 +24,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
+		context.CollectionFactory(), context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -50,6 +52,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -105,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -151,6 +154,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -69,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -109,6 +112,7 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

deb/publish.go

@@ -1097,7 +1097,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
-	force bool) error {
+	force, skipCleanup bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1134,7 +1134,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
-	if len(cleanComponents) > 0 {
+	if !skipCleanup && len(cleanComponents) > 0 {
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {

deb/publish_test.go

@@ -756,7 +756,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -776,10 +776,48 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, false)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2SkipCleanup(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, true)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -794,7 +832,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -816,7 +854,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

system/t06_publish/PublishDrop8Test_gold

@@ -0,0 +1,3 @@
+Removing ${HOME}/.aptly/public/dists/sq2...
+
+Published repository has been removed successfully.

system/t06_publish/PublishDrop9Test_gold

@@ -0,0 +1,4 @@
+Removing ${HOME}/.aptly/public/dists/sq1...
+Cleaning up prefix "." components main...
+
+Published repository has been removed successfully.

system/t06_publish/PublishSwitch14Test_binary

@@ -0,0 +1,29 @@
+
+ .
+ .
+ C-like language. Can perform smoothing, spline-fitting, or nonlinear fits,
+ Data files and self-defined functions can be manipulated by the internal
+ Gnuplot is a portable command-line driven interactive data and function
+ This package contains the terminal driver that enables gnuplot to plot
+ and can work with complex numbers.
+ for many printers, (La)TeX, (x)fig, Postscript, and so on. The X11-output
+ gnuplot.
+ images interactively under X11. Most users will want this, it is however
+ is packaged in gnuplot-x11.
+ packaged separately so that low-end systems don't need X installed to use
+ plotting utility that supports lots of output formats, including drivers
+Architecture: i386
+Depends: gnuplot-nox (>= 4.6.1-1~maverick2), libc6 (>= 2.11), libcairo2 (>= 1.6.0), libedit2 (>= 2.5.cvs.20010821-1), libgcc1 (>= 1:4.1.1), libgd2-noxpm (>= 2.0.36~rc1~dfsg) | libgd2-xpm (>= 2.0.36~rc1~dfsg), libglib2.0-0 (>= 2.12.0), liblua5.1-0, libpango1.0-0 (>= 1.14.0), libstdc++6 (>= 4.1.1), libwxbase2.8-0 (>= 2.8.11.0), libwxgtk2.8-0 (>= 2.8.11.0), libx11-6
+Description: Command-line driven interactive plotting program
+Filename: pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb
+Installed-Size: 1604
+MD5sum: fcad938905d0ace50a6ce0c73b2c6583
+Maintainer: Debian Science Team <debian-science-maintainers@lists.alioth.debian.org>
+Package: gnuplot-x11
+Priority: optional
+Replaces: gnuplot (<< 4.0.0)
+SHA1: 02f9a93097a8f798a054e26154dbe5789088c069
+Section: math
+Size: 724388
+Source: gnuplot
+Version: 4.6.1-1~maverick2

system/t06_publish/PublishSwitch14Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot.

system/t06_publish/PublishSwitch14Test_release

@@ -0,0 +1,11 @@
+Origin: LP-PPA-gladky-anton-gnuplot
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Architectures: amd64 i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/PublishUpdate12Test_binary

@@ -0,0 +1,27 @@
+
+
+ (name, value) pairs from the user, via conventional methods such as
+ .
+ .
+ Boost version (currently 1.49).
+ Library to let program developers obtain program options, that is
+ This package forms part of the Boost C++ Libraries collection.
+ This package is a dependency package, which depends on Debian's default
+ command line and config file.
+Architecture: i386
+Depends: libboost-program-options1.49-dev
+Description: program options library for C++ (default version)
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Homepage: http://www.boost.org/libs/program_options/
+Installed-Size: 26
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Package: libboost-program-options-dev
+Priority: optional
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+SHA512: d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c
+Section: libdevel
+Size: 2738
+Source: boost-defaults
+Version: 1.49.0.1

system/t06_publish/PublishUpdate12Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate12Test_release

@@ -0,0 +1,11 @@
+Origin: . maverick
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/drop.py

@@ -143,3 +143,66 @@ class PublishDrop7Test(BaseTest):
         self.check_not_exists('public/ppa/smira/dists/')
         self.check_not_exists('public/ppa/smira/pool/')
         self.check_exists('public/ppa/smira/')
+
+
+class PublishDrop8Test(BaseTest):
+    """
+    publish drop: skip component cleanup
+    """
+    fixtureCmds = [
+        "aptly repo create local1",
+        "aptly repo create local2",
+        "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly repo add local2 ${files}",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2",
+    ]
+    runCmd = "aptly publish drop -skip-cleanup sq2"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishDrop8Test, self).check()
+
+        self.check_exists('public/dists/sq1')
+        self.check_not_exists('public/dists/sq2')
+        self.check_exists('public/pool/main/')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class PublishDrop9Test(BaseTest):
+    """
+    publish drop: component cleanup after first cleanup skipped
+    """
+    fixtureCmds = [
+        "aptly repo create local1",
+        "aptly repo create local2",
+        "aptly repo create local3",
+        "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly repo add local2 ${files}",
+        "aptly repo add local3 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq3 local3",
+        "aptly publish drop -skip-cleanup sq2"
+    ]
+    runCmd = "aptly publish drop sq1"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishDrop9Test, self).check()
+
+        self.check_not_exists('public/dists/sq1')
+        self.check_not_exists('public/dists/sq2')
+        self.check_exists('public/dists/sq3')
+        self.check_exists('public/pool/main/')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')

system/t06_publish/switch.py

@@ -451,3 +451,85 @@ class PublishSwitch13Test(BaseTest):
         self.check_not_exists('public/dists/maverick/main/Contents-i386.gz')
         self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
         self.check_not_exists('public/dists/maverick/main/Contents-amd64.gz')
+
+
+class PublishSwitch14Test(BaseTest):
+    """
+    publish switch: removed some packages skipping cleanup
+    """
+    fixtureDB = True
+    fixturePool = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror gnuplot-maverick",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1",
+    ]
+    runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick snap3"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishSwitch14Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-i386.gz')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-amd64.gz')
+
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_amd64.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_i386.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_amd64.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+        # verify signatures
+        self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')])
+        self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
+                      os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+        # verify sums
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            elif len(fileHash) == 64:
+                h = hashlib.sha256()
+            else:
+                h = hashlib.sha512()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz',
+                             'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2',
+                             'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
+                             'main/Contents-i386.gz']):
+            raise Exception("path seen wrong: %r" % (pathsSeen, ))

system/t06_publish/update.py

@@ -339,3 +339,83 @@ class PublishUpdate11Test(BaseTest):
 
         self.check_exists('public/dists/maverick/main/binary-i386/Packages')
         self.check_not_exists('public/dists/maverick/main/Contents-i386.gz')
+
+
+class PublishUpdate12Test(BaseTest):
+    """
+    publish update: removed some packages skipping cleanup
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate12Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-i386.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+        # verify signatures
+        self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')])
+        self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
+                      os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+        # verify sums
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            elif len(fileHash) == 64:
+                h = hashlib.sha256()
+            else:
+                h = hashlib.sha512()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
+                             'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2',
+                             'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']):
+            raise Exception("path seen wrong: %r" % (pathsSeen, ))

system/t12_api/publish.py

@@ -214,6 +214,88 @@ class PublishUpdateAPITestRepo(APITest):
         self.check_not_exists("public/" + prefix + "dists/")
 
 
+class PublishUpdateSkipCleanupAPITestRepo(APITest):
+    """
+    PUT /publish/:prefix/:distribution (local repos), DELETE /publish/:prefix/:distribution
+    """
+    fixtureGpg = True
+
+    def check(self):
+        repo_name = self.random_name()
+        self.check_equal(self.post("/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(
+            self.upload("/api/files/" + d,
+                        "pyspi_0.6.1-1.3.dsc",
+                        "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
+                        "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
+        self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200)
+
+        prefix = self.random_name()
+        resp = self.post("/api/publish/" + prefix,
+                         json={
+                             "Architectures": ["i386", "source"],
+                             "SourceKind": "local",
+                             "Sources": [{"Name": repo_name}],
+                             "Signing": DefaultSigningOptions,
+                         })
+
+        self.check_equal(resp.status_code, 201)
+
+        self.check_not_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        # Publish two repos, so that deleting one while skipping cleanup will
+        # not delete the whole prefix.
+        resp = self.post("/api/publish/" + prefix,
+                         json={
+                             "Architectures": ["i386", "source"],
+                             "Distribution": "otherdist",
+                             "SourceKind": "local",
+                             "Sources": [{"Name": repo_name}],
+                             "Signing": DefaultSigningOptions,
+                         })
+
+        self.check_equal(resp.status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(self.upload("/api/files/" + d,
+                         "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200)
+        self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200)
+
+        self.check_equal(self.delete("/api/repos/" + repo_name + "/packages/",
+                         json={"PackageRefs": ['Psource pyspi 0.6.1-1.4 f8f1daa806004e89']}).status_code, 200)
+
+        resp = self.put("/api/publish/" + prefix + "/wheezy",
+                        json={
+                            "Signing": DefaultSigningOptions,
+                            "SkipCleanup": True,
+                        })
+        repo_expected = {
+            'Architectures': ['i386', 'source'],
+            'Distribution': 'wheezy',
+            'Label': '',
+            'Origin': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'SourceKind': 'local',
+            'Sources': [{'Component': 'main', 'Name': repo_name}],
+            'Storage': ''}
+
+        self.check_equal(resp.status_code, 200)
+        self.check_equal(resp.json(), repo_expected)
+
+        self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy", params={"SkipCleanup": "1"}).status_code, 200)
+        self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+
 class PublishSwitchAPITestRepo(APITest):
     """
     PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution
@@ -300,3 +382,119 @@ class PublishSwitchAPITestRepo(APITest):
 
         self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy").status_code, 200)
         self.check_not_exists("public/" + prefix + "dists/")
+
+
+class PublishSwitchAPISkipCleanupTestRepo(APITest):
+    """
+    PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution
+    """
+    fixtureGpg = True
+
+    def check(self):
+        repo_name = self.random_name()
+        self.check_equal(self.post("/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(
+            self.upload("/api/files/" + d,
+                        "pyspi_0.6.1-1.3.dsc",
+                        "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
+                        "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
+        self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200)
+
+        snapshot1_name = self.random_name()
+        self.check_equal(self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot1_name}).status_code, 201)
+
+        prefix = self.random_name()
+        resp = self.post("/api/publish/" + prefix,
+                         json={
+                             "Architectures": ["i386", "source"],
+                             "SourceKind": "snapshot",
+                             "Sources": [{"Name": snapshot1_name}],
+                             "Signing": DefaultSigningOptions,
+                         })
+
+        self.check_equal(resp.status_code, 201)
+        repo_expected = {
+            'Architectures': ['i386', 'source'],
+            'Distribution': 'wheezy',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': '',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot1_name}],
+            'Storage': ''}
+        self.check_equal(resp.json(), repo_expected)
+
+        self.check_not_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        # Publish two snapshots, so that deleting one while skipping cleanup will
+        # not delete the whole prefix.
+        resp = self.post("/api/publish/" + prefix,
+                         json={
+                             "Architectures": ["i386", "source"],
+                             "Distribution": "otherdist",
+                             "SourceKind": "snapshot",
+                             "Sources": [{"Name": snapshot1_name}],
+                             "Signing": DefaultSigningOptions,
+                         })
+
+        self.check_equal(resp.status_code, 201)
+        repo_expected = {
+            'Architectures': ['i386', 'source'],
+            'Distribution': 'otherdist',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': '',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot1_name}],
+            'Storage': ''}
+        self.check_equal(resp.json(), repo_expected)
+
+        d = self.random_name()
+        self.check_equal(self.upload("/api/files/" + d,
+                         "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200)
+        self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200)
+
+        self.check_equal(self.delete("/api/repos/" + repo_name + "/packages/",
+                         json={"PackageRefs": ['Psource pyspi 0.6.1-1.4 f8f1daa806004e89']}).status_code, 200)
+
+        snapshot2_name = self.random_name()
+        self.check_equal(self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot2_name}).status_code, 201)
+
+        resp = self.put("/api/publish/" + prefix + "/wheezy",
+                        json={
+                            "Snapshots": [{"Component": "main", "Name": snapshot2_name}],
+                            "Signing": DefaultSigningOptions,
+                            "SkipCleanup": True,
+                            "SkipContents": True,
+                        })
+        repo_expected = {
+            'Architectures': ['i386', 'source'],
+            'Distribution': 'wheezy',
+            'Label': '',
+            'Origin': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Prefix': prefix,
+            'SkipContents': True,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot2_name}],
+            'Storage': ''}
+
+        self.check_equal(resp.status_code, 200)
+        self.check_equal(resp.json(), repo_expected)
+
+        self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy", params={"SkipCleanup": "1"}).status_code, 200)
+        self.check_exists("public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")