Command aptly publish update: update local repo published in-place. #8

cmd/publish.go

@@ -31,6 +31,7 @@ func makeCmdPublish() *commander.Command {
 		Subcommands: []*commander.Command{
 			makeCmdPublishRepo(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishUpdate(),
 			makeCmdPublishList(),
 			makeCmdPublishDrop(),
 		},

cmd/publish_update.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return err
+	}
+
+	distribution := args[0]
+	prefix := "."
+
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "local" {
+		return fmt.Errorf("unable to update: not a local repository publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	published.UpdateLocalRepo()
+
+	signer, err := getSigner(context.flags)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, published.Component,
+		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishUpdate,
+		UsageLine: "update <distribution> [<prefix>]",
+		Short:     "update published local repository",
+		Long: `
+Command re-publishes (updates) published local repository. <distribution>
+and <prefix> should be occupied with local repository published
+using command aptly publish repo. Update happens in-place with
+minimum possible downtime for published repository.
+
+Example:
+
+    $ aptly publish update wheezy ppa
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+
+	return cmd
+}

system/t06_publish/PublishUpdate1Test_binary

@@ -0,0 +1,26 @@
+
+
+ (name, value) pairs from the user, via conventional methods such as
+ .
+ .
+ Boost version (currently 1.49).
+ Library to let program developers obtain program options, that is
+ This package forms part of the Boost C++ Libraries collection.
+ This package is a dependency package, which depends on Debian's default
+ command line and config file.
+Architecture: i386
+Depends: libboost-program-options1.49-dev
+Description: program options library for C++ (default version)
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Homepage: http://www.boost.org/libs/program_options/
+Installed-Size: 26
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Package: libboost-program-options-dev
+Priority: optional
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+Section: libdevel
+Size: 2738
+Source: boost-defaults
+Version: 1.49.0.1

system/t06_publish/PublishUpdate1Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Signing file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Clearsigning file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." component "main"...
+
+Publish for local repo ./maverick (main) [i386, source] publishes [local-repo] has been successfully updated.

system/t06_publish/PublishUpdate1Test_release

@@ -0,0 +1,9 @@
+Origin: . maverick
+Label: . maverick
+Codename: maverick
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:

system/t06_publish/PublishUpdate2Test_binary

@@ -0,0 +1,25 @@
+
+ (name, value) pairs from the user, via conventional methods such as
+ .
+ .
+ Boost version (currently 1.49).
+ Library to let program developers obtain program options, that is
+ This package forms part of the Boost C++ Libraries collection.
+ This package is a dependency package, which depends on Debian's default
+ command line and config file.
+Architecture: i386
+Depends: libboost-program-options1.49-dev
+Description: program options library for C++ (default version)
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Homepage: http://www.boost.org/libs/program_options/
+Installed-Size: 26
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Package: libboost-program-options-dev
+Priority: optional
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+Section: libdevel
+Size: 2738
+Source: boost-defaults
+Version: 1.49.0.1

system/t06_publish/PublishUpdate2Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Signing file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Clearsigning file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." component "main"...
+
+Publish for local repo ./maverick (main) [i386, source] publishes [local-repo] has been successfully updated.

system/t06_publish/PublishUpdate2Test_sources

@@ -0,0 +1,42 @@
+
+
+ 22ff26db69b73d3438fdde21ab5ba2f1 3456 pyspi_0.6.1-1.3.diff.gz
+ 2e770b28df948f3197ed0b679bdea99f3f2bf745e9ddb440c677df9c3aeaee3c 3456 pyspi_0.6.1-1.3.diff.gz
+ 56c8a9b1f4ab636052be8966690998cbe865cd6c 1782 pyspi_0.6.1-1.3.dsc
+ 64069ee828c50b1c597d10a3fefbba279f093a4723965388cdd0ac02f029bfb9 29063 pyspi_0.6.1.orig.tar.gz
+ 64069ee828c50b1c597d10a3fefbba279f093a4723965388cdd0ac02f029bfb9 29063 pyspi_0.6.1.orig.tar.gz
+ 95a2468e4bbce730ba286f2211fa41861b9f1d90 3456 pyspi_0.6.1-1.3.diff.gz
+ 9694b80acc171c0a5bc99f707933864edfce555e 29063 pyspi_0.6.1.orig.tar.gz
+ 9694b80acc171c0a5bc99f707933864edfce555e 29063 pyspi_0.6.1.orig.tar.gz
+ b72cb94699298a117b7c82641c68b6fd 1782 pyspi_0.6.1-1.3.dsc
+ d494aaf526f1ec6b02f14c2f81e060a5722d6532ddc760ec16972e45c2625989 1782 pyspi_0.6.1-1.3.dsc
+ def336bd566ea688a06ec03db7ccf1f4 29063 pyspi_0.6.1.orig.tar.gz
+ def336bd566ea688a06ec03db7ccf1f4 29063 pyspi_0.6.1.orig.tar.gz
+Architecture: any
+Architecture: any
+Binary: python-at-spi
+Binary: python-at-spi
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Checksums-Sha1: 5005fbd1f30637edc1d380b30f45db9b79100d07 893 pyspi-0.6.1-1.3.stripped.dsc
+Checksums-Sha1: 95a2468e4bbce730ba286f2211fa41861b9f1d90 3456 pyspi_0.6.1-1.3.diff.gz
+Checksums-Sha256: 289d3aefa970876e9c43686ce2b02f478d7f3ed35a713928464a98d54ae4fca3 893 pyspi-0.6.1-1.3.stripped.dsc
+Checksums-Sha256: 2e770b28df948f3197ed0b679bdea99f3f2bf745e9ddb440c677df9c3aeaee3c 3456 pyspi_0.6.1-1.3.diff.gz
+Directory: pool/main/p/pyspi
+Directory: pool/main/p/pyspi
+Files: 22ff26db69b73d3438fdde21ab5ba2f1 3456 pyspi_0.6.1-1.3.diff.gz
+Files: 2f5bd47cf38852b6fc927a50f98c1448 893 pyspi-0.6.1-1.3.stripped.dsc
+Format: 1.0
+Format: 1.0
+Homepage: http://people.redhat.com/zcerza/dogtail
+Homepage: http://people.redhat.com/zcerza/dogtail
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Package: pyspi
+Package: pyspi
+Standards-Version: 3.7.3
+Standards-Version: 3.7.3
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Version: 0.6.1-1.3
+Version: 0.6.1-1.4

system/t06_publish/PublishUpdate3Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Signing file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Clearsigning file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." component "main"...
+
+Publish for local repo ./maverick (main) [i386, source] publishes [local-repo] has been successfully updated.

system/t06_publish/PublishUpdate4Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Signing file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Clearsigning file '${HOME}/.aptly/public/dists/maverick/Release.tmp' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." component "main"...
+
+Publish for local repo ./maverick (main) [source] publishes [local-repo] has been successfully updated.

system/t06_publish/PublishUpdate5Test_gold

@@ -0,0 +1 @@
+ERROR: unable to update: published repo with prefix/distribution ppa/maverick not found

system/t06_publish/PublishUpdate6Test_gold

@@ -0,0 +1 @@
+ERROR: unable to update: not a local repository publish

system/t06_publish/__init__.py

@@ -6,3 +6,4 @@ from .drop import *
 from .list import *
 from .repo import *
 from .snapshot import *
+from .update import *

system/t06_publish/update.py

@@ -0,0 +1,212 @@
+import os
+import hashlib
+import inspect
+from lib import BaseTest
+
+
+def strip_processor(output):
+    return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:')])
+
+
+class PublishUpdate1Test(BaseTest):
+    """
+    publish update: removed some packages
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate1Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+        # verify signatures
+        self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')])
+        self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
+                      os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+        # verify sums
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            else:
+                h = hashlib.sha256()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
+                             'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2']):
+            raise Exception("path seen wrong: %r" % (pathsSeen, ))
+
+
+class PublishUpdate2Test(BaseTest):
+    """
+    publish update: added some packages
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb ${files}/pyspi_0.6.1-1.3.dsc",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly repo add local-repo ${files}/pyspi-0.6.1-1.3.stripped.dsc"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate2Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+
+class PublishUpdate3Test(BaseTest):
+    """
+    publish update: removed some packages, files occupied by another package
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick2 local-repo",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate3Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class PublishUpdate4Test(BaseTest):
+    """
+    publish update: added some packages, but list of published archs doesn't change
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/pyspi_0.6.1-1.3.dsc",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate4Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_not_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class PublishUpdate5Test(BaseTest):
+    """
+    publish update: no such publish
+    """
+    runCmd = "aptly publish update maverick ppa"
+    expectedCode = 1
+
+
+class PublishUpdate6Test(BaseTest):
+    """
+    publish update: not a local repo
+    """
+    fixtureDB = True
+    fixturePool = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror gnuplot-maverick",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1",
+    ]
+    runCmd = "aptly publish update maverick"
+    expectedCode = 1