From fbac926044896d1e96ae5ff66ddb79b4b4123739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Delafond?= Date: Sat, 2 May 2015 12:24:38 +0200 Subject: [PATCH 1/2] Imported Upstream version 0.9.1 From 9141f1b3439d09823cc2ebac2e6b81c3cdb616b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Delafond?= Date: Fri, 15 May 2015 10:45:09 +0200 Subject: [PATCH 2/2] Imported Upstream version 0.9.5 --- bash_completion.d/aptly | 12 ++ src/github.com/smira/aptly/api/publish.go | 4 +- src/github.com/smira/aptly/aptly/version.go | 2 +- src/github.com/smira/aptly/cmd/db_cleanup.go | 177 +++++++++++++++--- .../smira/aptly/cmd/publish_drop.go | 4 +- .../smira/aptly/cmd/snapshot_search.go | 4 + .../smira/aptly/console/progress.go | 27 ++- src/github.com/smira/aptly/deb/format.go | 32 +++- src/github.com/smira/aptly/deb/format_test.go | 12 ++ src/github.com/smira/aptly/deb/import.go | 18 ++ src/github.com/smira/aptly/deb/publish.go | 7 +- .../smira/aptly/deb/publish_test.go | 10 +- src/github.com/smira/aptly/man/aptly.1 | 24 ++- .../aptly/system/t01_version/VersionTest_gold | 2 +- .../t05_snapshot/SearchSnapshot5Test_gold | 1 + .../smira/aptly/system/t05_snapshot/search.py | 10 + .../aptly/system/t08_db/CleanupDB11Test_gold | 41 ++++ .../aptly/system/t08_db/CleanupDB12Test_gold | 42 +++++ .../smira/aptly/system/t08_db/cleanup.py | 26 +++ .../AddRepo15Test/pyspi_0.6-broken.dsc | 11 ++ .../pyspi_0.6.1-1.3.conflict.dsc | 12 ++ .../AddRepo15Test/pyspi_0.6.1.orig.tar.gz | 0 .../aptly/system/t09_repo/AddRepo15Test_gold | 6 + .../smira/aptly/system/t09_repo/add.py | 12 ++ .../aptly/system/t10_task/RunTask1Test_gold | 2 +- .../smira/aptly/system/t12_api/version.py | 2 +- 26 files changed, 446 insertions(+), 54 deletions(-) create mode 100644 src/github.com/smira/aptly/system/t05_snapshot/SearchSnapshot5Test_gold create mode 100644 src/github.com/smira/aptly/system/t08_db/CleanupDB11Test_gold create mode 100644 src/github.com/smira/aptly/system/t08_db/CleanupDB12Test_gold create mode 100644 src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6-broken.dsc create mode 100644 src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1-1.3.conflict.dsc create mode 100644 src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1.orig.tar.gz create mode 100644 src/github.com/smira/aptly/system/t09_repo/AddRepo15Test_gold diff --git a/bash_completion.d/aptly b/bash_completion.d/aptly index 9cfd752f..4ed8e2c0 100644 --- a/bash_completion.d/aptly +++ b/bash_completion.d/aptly @@ -572,5 +572,17 @@ _aptly() ;; esac ;; + "db") + case "$subcmd" in + "cleanup") + if [[ $numargs -eq 0 ]]; then + if [[ "$cur" == -* ]]; then + COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur})) + fi + return 0 + fi + ;; + esac + ;; esac } && complete -F _aptly aptly diff --git a/src/github.com/smira/aptly/api/publish.go b/src/github.com/smira/aptly/api/publish.go index a38b80c2..538234a3 100644 --- a/src/github.com/smira/aptly/api/publish.go +++ b/src/github.com/smira/aptly/api/publish.go @@ -312,6 +312,8 @@ func apiPublishUpdateSwitch(c *gin.Context) { // DELETE /publish/:prefix/:distribution func apiPublishDrop(c *gin.Context) { + force := c.Request.URL.Query().Get("force") == "1" + param := parseEscapedPath(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := c.Params.ByName("distribution") @@ -326,7 +328,7 @@ func apiPublishDrop(c *gin.Context) { defer collection.Unlock() err := collection.Remove(context, storage, prefix, distribution, - context.CollectionFactory(), context.Progress()) + context.CollectionFactory(), context.Progress(), force) if err != nil { c.Fail(500, fmt.Errorf("unable to drop: %s", err)) return diff --git a/src/github.com/smira/aptly/aptly/version.go b/src/github.com/smira/aptly/aptly/version.go index d6e947dc..76cf1d57 100644 --- a/src/github.com/smira/aptly/aptly/version.go +++ b/src/github.com/smira/aptly/aptly/version.go @@ -1,7 +1,7 @@ package aptly // Version of aptly -const Version = "0.9.1" +const Version = "0.9.5" // Enable debugging features? const EnableDebug = false diff --git a/src/github.com/smira/aptly/cmd/db_cleanup.go b/src/github.com/smira/aptly/cmd/db_cleanup.go index 40d5c5e4..8a5e1825 100644 --- a/src/github.com/smira/aptly/cmd/db_cleanup.go +++ b/src/github.com/smira/aptly/cmd/db_cleanup.go @@ -6,6 +6,7 @@ import ( "github.com/smira/aptly/utils" "github.com/smira/commander" "sort" + "strings" ) // aptly db cleanup @@ -17,51 +18,112 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { return commander.ErrCommandError } + verbose := context.Flags().Lookup("verbose").Value.Get().(bool) + dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool) + // collect information about references packages... existingPackageRefs := deb.NewPackageRefList() - context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n") + // used only in verbose mode to report package use source + packageRefSources := map[string][]string{} + + context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|") + if verbose { + context.Progress().ColoredPrintf("@{y}Loading mirrors:@|") + } err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { + if verbose { + context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name) + } + err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) if err != nil { return err } if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true) + + if verbose { + description := fmt.Sprintf("mirror %s", repo.Name) + repo.RefList().ForEach(func(key []byte) error { + packageRefSources[string(key)] = append(packageRefSources[string(key)], description) + return nil + }) + } } + return nil }) if err != nil { return err } + if verbose { + context.Progress().ColoredPrintf("@{y}Loading local repos:@|") + } err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { + if verbose { + context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name) + } + err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) if err != nil { return err } + if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true) + + if verbose { + description := fmt.Sprintf("local repo %s", repo.Name) + repo.RefList().ForEach(func(key []byte) error { + packageRefSources[string(key)] = append(packageRefSources[string(key)], description) + return nil + }) + } } + return nil }) if err != nil { return err } + if verbose { + context.Progress().ColoredPrintf("@{y}Loading snapshots:@|") + } err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { + if verbose { + context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name) + } + err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) if err != nil { return err } + existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true) + + if verbose { + description := fmt.Sprintf("snapshot %s", snapshot.Name) + snapshot.RefList().ForEach(func(key []byte) error { + packageRefSources[string(key)] = append(packageRefSources[string(key)], description) + return nil + }) + } return nil }) if err != nil { return err } + if verbose { + context.Progress().ColoredPrintf("@{y}Loading published repositories:@|") + } err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error { + if verbose { + context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution) + } if published.SourceKind != "local" { return nil } @@ -72,6 +134,14 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { for _, component := range published.Components() { existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true) + if verbose { + description := fmt.Sprintf("published repository %s:%s/%s component %s", + published.Storage, published.Prefix, published.Distribution, component) + published.RefList(component).ForEach(func(key []byte) error { + packageRefSources[string(key)] = append(packageRefSources[string(key)], description) + return nil + }) + } } return nil }) @@ -80,38 +150,65 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { } // ... and compare it to the list of all packages - context.Progress().Printf("Loading list of all packages...\n") + context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|") allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs() toDelete := allPackageRefs.Substract(existingPackageRefs) // delete packages that are no longer referenced - context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len()) + context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len()) // database can't err as collection factory already constructed db, _ := context.Database() - db.StartBatch() - err = toDelete.ForEach(func(ref []byte) error { - return context.CollectionFactory().PackageCollection().DeleteByKey(ref) - }) - if err != nil { - return err - } - err = db.FinishBatch() - if err != nil { - return fmt.Errorf("unable to write to DB: %s", err) + if toDelete.Len() > 0 { + if verbose { + context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|") + err = toDelete.ForEach(func(ref []byte) error { + context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref)) + return nil + }) + if err != nil { + return err + } + } + + if !dryRun { + db.StartBatch() + err = toDelete.ForEach(func(ref []byte) error { + return context.CollectionFactory().PackageCollection().DeleteByKey(ref) + }) + if err != nil { + return err + } + + err = db.FinishBatch() + if err != nil { + return fmt.Errorf("unable to write to DB: %s", err) + } + } else { + context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|") + } } // now, build a list of files that should be present in Repository (package pool) - context.Progress().Printf("Building list of files referenced by packages...\n") + context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|") referencedFiles := make([]string, 0, existingPackageRefs.Len()) context.Progress().InitBar(int64(existingPackageRefs.Len()), false) err = existingPackageRefs.ForEach(func(key []byte) error { pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key) if err2 != nil { - return err2 + tail := "" + if verbose { + tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", ")) + } + if dryRun { + context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s", + string(key), err2, tail) + return nil + } + return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail) } paths, err2 := pkg.FilepathList(context.PackagePool()) if err2 != nil { @@ -130,7 +227,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { context.Progress().ShutdownBar() // build a list of files in the package pool - context.Progress().Printf("Building list of files in package pool...\n") + context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|") existingFiles, err := context.PackagePool().FilepathList(context.Progress()) if err != nil { return fmt.Errorf("unable to collect file paths: %s", err) @@ -140,28 +237,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles) // delete files that are no longer referenced - context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete)) + context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete)) if len(filesToDelete) > 0 { - context.Progress().InitBar(int64(len(filesToDelete)), false) - - var size, totalSize int64 - for _, file := range filesToDelete { - size, err = context.PackagePool().Remove(file) - if err != nil { - return err + if verbose { + context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|") + for _, file := range filesToDelete { + context.Progress().ColoredPrintf(" - @{r}%s@|", file) } - - context.Progress().AddBar(1) - totalSize += size } - context.Progress().ShutdownBar() - context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize)) + if !dryRun { + context.Progress().InitBar(int64(len(filesToDelete)), false) + + var size, totalSize int64 + for _, file := range filesToDelete { + size, err = context.PackagePool().Remove(file) + if err != nil { + return err + } + + context.Progress().AddBar(1) + totalSize += size + } + context.Progress().ShutdownBar() + + context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize)) + } else { + context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|") + } } - context.Progress().Printf("Compacting database...\n") - err = db.CompactDB() + if !dryRun { + context.Progress().ColoredPrintf("@{w!}Compacting database...@|") + err = db.CompactDB() + } else { + context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|") + } return err } @@ -181,5 +293,8 @@ Example: `, } + cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them") + cmd.Flag.Bool("dry-run", false, "don't delete anything") + return cmd } diff --git a/src/github.com/smira/aptly/cmd/publish_drop.go b/src/github.com/smira/aptly/cmd/publish_drop.go index 0bedc7d5..9c465828 100644 --- a/src/github.com/smira/aptly/cmd/publish_drop.go +++ b/src/github.com/smira/aptly/cmd/publish_drop.go @@ -23,7 +23,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error { storage, prefix := deb.ParsePrefix(param) err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution, - context.CollectionFactory(), context.Progress()) + context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool)) if err != nil { return fmt.Errorf("unable to remove: %s", err) } @@ -48,5 +48,7 @@ Example: `, } + cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up") + return cmd } diff --git a/src/github.com/smira/aptly/cmd/snapshot_search.go b/src/github.com/smira/aptly/cmd/snapshot_search.go index d1aee367..6b896a5d 100644 --- a/src/github.com/smira/aptly/cmd/snapshot_search.go +++ b/src/github.com/smira/aptly/cmd/snapshot_search.go @@ -96,6 +96,10 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error return fmt.Errorf("unable to search: %s", err) } + if result.Len() == 0 { + return fmt.Errorf("no results") + } + result.ForEach(func(p *deb.Package) error { context.Progress().Printf("%s\n", p) return nil diff --git a/src/github.com/smira/aptly/console/progress.go b/src/github.com/smira/aptly/console/progress.go index 398add43..a10980ea 100644 --- a/src/github.com/smira/aptly/console/progress.go +++ b/src/github.com/smira/aptly/console/progress.go @@ -133,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) { p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"} } else { // stip color marks - var prev rune + var inColorMark, inCurly bool msg = strings.Map(func(r rune) rune { - if prev == '@' { - prev = 0 - if r == '@' { - return r + if inColorMark { + if inCurly { + if r == '}' { + inCurly = false + inColorMark = false + return -1 + } + } else { + if r == '{' { + inCurly = true + } else if r == '@' { + return '@' + } else { + inColorMark = false + } } return -1 } - prev = r - if r == '@' { - return -1 + if r == '@' { + inColorMark = true + return -1 } return r diff --git a/src/github.com/smira/aptly/deb/format.go b/src/github.com/smira/aptly/deb/format.go index 5663a564..1348fe2c 100644 --- a/src/github.com/smira/aptly/deb/format.go +++ b/src/github.com/smira/aptly/deb/format.go @@ -5,6 +5,7 @@ import ( "errors" "io" "strings" + "unicode" ) // Stanza or paragraph of Debian control file @@ -157,6 +158,35 @@ func init() { multilineFields["MD5Sum"] = true } +func canonicalCase(field string) string { + upper := strings.ToUpper(field) + switch upper { + case "SHA1", "SHA256", "SHA512": + return upper + case "MD5SUM": + return "MD5Sum" + case "NOTAUTOMATIC": + return "NotAutomatic" + case "BUTAUTOMATICUPGRADES": + return "ButAutomaticUpgrades" + } + + startOfWord := true + + return strings.Map(func(r rune) rune { + if startOfWord { + startOfWord = false + return unicode.ToUpper(r) + } + + if r == '-' { + startOfWord = true + } + + return unicode.ToLower(r) + }, field) +} + // ControlFileReader implements reading of control files stanza by stanza type ControlFileReader struct { scanner *bufio.Scanner @@ -195,7 +225,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) { if len(parts) != 2 { return nil, ErrMalformedStanza } - lastField = parts[0] + lastField = canonicalCase(parts[0]) _, lastFieldMultiline = multilineFields[lastField] if lastFieldMultiline { stanza[lastField] = parts[1] diff --git a/src/github.com/smira/aptly/deb/format_test.go b/src/github.com/smira/aptly/deb/format_test.go index 41046def..d7f43acc 100644 --- a/src/github.com/smira/aptly/deb/format_test.go +++ b/src/github.com/smira/aptly/deb/format_test.go @@ -123,6 +123,18 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) { c.Assert(strings.HasPrefix(str, "Package: "), Equals, true) } +func (s *ControlFileSuite) TestCanonicalCase(c *C) { + c.Check(canonicalCase("Package"), Equals, "Package") + c.Check(canonicalCase("package"), Equals, "Package") + c.Check(canonicalCase("pAckaGe"), Equals, "Package") + c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum") + c.Check(canonicalCase("SHA1"), Equals, "SHA1") + c.Check(canonicalCase("SHA256"), Equals, "SHA256") + c.Check(canonicalCase("Package-List"), Equals, "Package-List") + c.Check(canonicalCase("package-list"), Equals, "Package-List") + c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List") +} + func (s *ControlFileSuite) BenchmarkReadStanza(c *C) { for i := 0; i < c.N; i++ { reader := bytes.NewBufferString(controlFile) diff --git a/src/github.com/smira/aptly/deb/import.go b/src/github.com/smira/aptly/deb/import.go index ceea8ff9..2aa9a037 100644 --- a/src/github.com/smira/aptly/deb/import.go +++ b/src/github.com/smira/aptly/deb/import.go @@ -91,6 +91,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b continue } + if p.Name == "" { + reporter.Warning("Empty package name on %s", file) + failedFiles = append(failedFiles, file) + continue + } + + if p.Version == "" { + reporter.Warning("Empty version on %s", file) + failedFiles = append(failedFiles, file) + continue + } + + if p.Architecture == "" { + reporter.Warning("Empty architecture on %s", file) + failedFiles = append(failedFiles, file) + continue + } + var checksums utils.ChecksumInfo checksums, err = utils.ChecksumsForFile(file) if err != nil { diff --git a/src/github.com/smira/aptly/deb/publish.go b/src/github.com/smira/aptly/deb/publish.go index cec60d44..82ea06ab 100644 --- a/src/github.com/smira/aptly/deb/publish.go +++ b/src/github.com/smira/aptly/deb/publish.go @@ -1000,7 +1000,8 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st // Remove removes published repository, cleaning up directories, files func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider, - storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error { + storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress, + force bool) error { repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { return err @@ -1041,7 +1042,9 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents, publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress) if err != nil { - return err + if !force { + return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err) + } } } diff --git a/src/github.com/smira/aptly/deb/publish_test.go b/src/github.com/smira/aptly/deb/publish_test.go index ab2d8a7d..05e2f730 100644 --- a/src/github.com/smira/aptly/deb/publish_test.go +++ b/src/github.com/smira/aptly/deb/publish_test.go @@ -740,7 +740,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { - err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil) + err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda") @@ -760,10 +760,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists) - err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil) + err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false) c.Check(err, ErrorMatches, ".*not found") - err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil) + err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false) c.Check(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) @@ -778,7 +778,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) { - err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil) + err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda") @@ -800,7 +800,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) { } func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) { - err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil) + err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false) c.Check(err, IsNil) _, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog") diff --git a/src/github.com/smira/aptly/man/aptly.1 b/src/github.com/smira/aptly/man/aptly.1 index 11af4701..f4ed4b38 100644 --- a/src/github.com/smira/aptly/man/aptly.1 +++ b/src/github.com/smira/aptly/man/aptly.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "APTLY" "1" "February 2015" "" "" +.TH "APTLY" "1" "March 2015" "" "" . .SH "NAME" \fBaptly\fR \- Debian repository management tool @@ -133,6 +133,10 @@ specifies paramaters for short PPA url expansion, if left blank they default to \fBS3PublishEndpoints\fR configuration of Amazon S3 publishing endpoints (see below) . +.TP +\fBSwiftPublishEndpoints\fR +configuration of OpenStack Swift publishing endpoints (see below) +. .SH "S3 PUBLISHING ENDPOINTS" aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings: . @@ -1082,6 +1086,13 @@ $ aptly publish drop wheezy . .IP "" 0 . +.P +Options: +. +.TP +\-\fBforce\-drop\fR=false +remove published repository even if some files could not be cleaned up +. .SH "LIST OF PUBLISHED REPOSITORIES" \fBaptly\fR \fBpublish\fR \fBlist\fR . @@ -1469,6 +1480,17 @@ Example: .P $ aptly db cleanup . +.P +Options: +. +.TP +\-\fBdry\-run\fR=false +don\(cqt delete anything +. +.TP +\-\fBverbose\fR=false +be verbose when loading objects/removing them +. .SH "RECOVER DB AFTER CRASH" \fBaptly\fR \fBdb\fR \fBrecover\fR . diff --git a/src/github.com/smira/aptly/system/t01_version/VersionTest_gold b/src/github.com/smira/aptly/system/t01_version/VersionTest_gold index 3fdf462a..1189d3c3 100644 --- a/src/github.com/smira/aptly/system/t01_version/VersionTest_gold +++ b/src/github.com/smira/aptly/system/t01_version/VersionTest_gold @@ -1 +1 @@ -aptly version: 0.9.1 +aptly version: 0.9.5 diff --git a/src/github.com/smira/aptly/system/t05_snapshot/SearchSnapshot5Test_gold b/src/github.com/smira/aptly/system/t05_snapshot/SearchSnapshot5Test_gold new file mode 100644 index 00000000..2d08b563 --- /dev/null +++ b/src/github.com/smira/aptly/system/t05_snapshot/SearchSnapshot5Test_gold @@ -0,0 +1 @@ +ERROR: no results diff --git a/src/github.com/smira/aptly/system/t05_snapshot/search.py b/src/github.com/smira/aptly/system/t05_snapshot/search.py index 3a065675..9133b333 100644 --- a/src/github.com/smira/aptly/system/t05_snapshot/search.py +++ b/src/github.com/smira/aptly/system/t05_snapshot/search.py @@ -37,3 +37,13 @@ class SearchSnapshot4Test(BaseTest): fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"] outputMatchPrepare = lambda _, s: "\n".join(sorted(s.split("\n"))) runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (nginx)'" + + +class SearchSnapshot5Test(BaseTest): + """ + search snapshot: no results + """ + fixtureDB = True + fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"] + runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (no-such-package)'" + expectedCode = 1 diff --git a/src/github.com/smira/aptly/system/t08_db/CleanupDB11Test_gold b/src/github.com/smira/aptly/system/t08_db/CleanupDB11Test_gold new file mode 100644 index 00000000..be3767e9 --- /dev/null +++ b/src/github.com/smira/aptly/system/t08_db/CleanupDB11Test_gold @@ -0,0 +1,41 @@ +Loading mirrors, local repos, snapshots and published repos... +Loading mirrors: +- wheezy-backports +- wheezy-updates-src +- wheezy-main +- wheezy-contrib +- sensu +- wheezy-main-src +- wheezy-backports-src +- wheezy-contrib-src +- wheezy-non-free +- wheezy-non-free-src +- wheezy-updates +Loading local repos: +Loading snapshots: +Loading published repositories: +Loading list of all packages... +Deleting unreferenced packages (7)... +List of package keys to delete: + - Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1 + - Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29 + - Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003 + - Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641 + - Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815 + - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740 + - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f +Building list of files referenced by packages... +Building list of files in package pool... +Deleting unreferenced files (9)... +List of files to be deleted: + - 02/1d/gnuplot_4.6.1-1~maverick2.dsc + - 10/32/gnuplot_4.6.1-1~maverick2.debian.tar.gz + - 17/ab/gnuplot-x11_4.6.1-1~maverick2_amd64.deb + - 25/a5/gnuplot-doc_4.6.1-1~maverick2_all.deb + - 49/12/gnuplot_4.6.1-1~maverick2_all.deb + - 4c/9a/gnuplot_4.6.1.orig.tar.gz + - a7/ef/gnuplot-nox_4.6.1-1~maverick2_i386.deb + - db/55/gnuplot-nox_4.6.1-1~maverick2_amd64.deb + - fc/ad/gnuplot-x11_4.6.1-1~maverick2_i386.deb +Disk space freed: 10.85 MiB... +Compacting database... diff --git a/src/github.com/smira/aptly/system/t08_db/CleanupDB12Test_gold b/src/github.com/smira/aptly/system/t08_db/CleanupDB12Test_gold new file mode 100644 index 00000000..31da9b23 --- /dev/null +++ b/src/github.com/smira/aptly/system/t08_db/CleanupDB12Test_gold @@ -0,0 +1,42 @@ +Loading mirrors, local repos, snapshots and published repos... +Loading mirrors: +- wheezy-backports +- wheezy-updates-src +- wheezy-main +- wheezy-contrib +- sensu +- wheezy-main-src +- wheezy-backports-src +- wheezy-contrib-src +- wheezy-non-free +- wheezy-non-free-src +- wheezy-updates +Loading local repos: +Loading snapshots: +Loading published repositories: +Loading list of all packages... +Deleting unreferenced packages (7)... +List of package keys to delete: + - Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1 + - Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29 + - Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003 + - Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641 + - Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815 + - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740 + - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f +Skipped deletion, as -dry-run has been requested. +Building list of files referenced by packages... +Building list of files in package pool... +Deleting unreferenced files (9)... +List of files to be deleted: + - 02/1d/gnuplot_4.6.1-1~maverick2.dsc + - 10/32/gnuplot_4.6.1-1~maverick2.debian.tar.gz + - 17/ab/gnuplot-x11_4.6.1-1~maverick2_amd64.deb + - 25/a5/gnuplot-doc_4.6.1-1~maverick2_all.deb + - 49/12/gnuplot_4.6.1-1~maverick2_all.deb + - 4c/9a/gnuplot_4.6.1.orig.tar.gz + - a7/ef/gnuplot-nox_4.6.1-1~maverick2_i386.deb + - db/55/gnuplot-nox_4.6.1-1~maverick2_amd64.deb + - fc/ad/gnuplot-x11_4.6.1-1~maverick2_i386.deb +Skipped file deletion, as -dry-run has been requested. +Skipped DB compaction, as -dry-run has been requested. diff --git a/src/github.com/smira/aptly/system/t08_db/cleanup.py b/src/github.com/smira/aptly/system/t08_db/cleanup.py index ae3f0bf0..1e27621a 100644 --- a/src/github.com/smira/aptly/system/t08_db/cleanup.py +++ b/src/github.com/smira/aptly/system/t08_db/cleanup.py @@ -124,3 +124,29 @@ class CleanupDB10Test(BaseTest): "aptly repo add b ${testfiles}" ] runCmd = "aptly db cleanup" + + +class CleanupDB11Test(BaseTest): + """ + cleanup db: deleting packages and files, -verbose + """ + fixtureDB = True + fixturePoolCopy = True + fixtureCmds = [ + "aptly mirror drop gnuplot-maverick-src", + "aptly mirror drop gnuplot-maverick", + ] + runCmd = "aptly db cleanup -verbose" + + +class CleanupDB12Test(BaseTest): + """ + cleanup db: deleting packages and files, -verbose & -dry-run + """ + fixtureDB = True + fixturePoolCopy = True + fixtureCmds = [ + "aptly mirror drop gnuplot-maverick-src", + "aptly mirror drop gnuplot-maverick", + ] + runCmd = "aptly db cleanup -verbose -dry-run" diff --git a/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6-broken.dsc b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6-broken.dsc new file mode 100644 index 00000000..cc8cc8c4 --- /dev/null +++ b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6-broken.dsc @@ -0,0 +1,11 @@ +Format: 1.0 +sourcE: pyspi +binary: python-at-spi +Architecture: any +Maintainer: Jose Carlos Garcia Sogo +Homepage: http://people.redhat.com/zcerza/dogtail +Standards-Version: 3.7.3 +Vcs-svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk +Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev +files: + d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz diff --git a/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1-1.3.conflict.dsc b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1-1.3.conflict.dsc new file mode 100644 index 00000000..51f76041 --- /dev/null +++ b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1-1.3.conflict.dsc @@ -0,0 +1,12 @@ +Format: 1.0 +sourcE: pyspi +binary: python-at-spi +Architecture: any +Version: 0.6.1-1.3 +Maintainer: Jose Carlos Garcia Sogo +Homepage: http://people.redhat.com/zcerza/dogtail +Standards-Version: 3.7.3 +Vcs-svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk +Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev +files: + d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz diff --git a/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1.orig.tar.gz b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test/pyspi_0.6.1.orig.tar.gz new file mode 100644 index 00000000..e69de29b diff --git a/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test_gold b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test_gold new file mode 100644 index 00000000..44d2a85c --- /dev/null +++ b/src/github.com/smira/aptly/system/t09_repo/AddRepo15Test_gold @@ -0,0 +1,6 @@ +Loading packages... +[!] Empty version on /pyspi_0.6-broken.dsc +[+] pyspi_0.6.1-1.3_source added +[!] Some files were skipped due to errors: + /pyspi_0.6-broken.dsc +ERROR: some files failed to be added diff --git a/src/github.com/smira/aptly/system/t09_repo/add.py b/src/github.com/smira/aptly/system/t09_repo/add.py index 3c26cea1..b038a74f 100644 --- a/src/github.com/smira/aptly/system/t09_repo/add.py +++ b/src/github.com/smira/aptly/system/t09_repo/add.py @@ -283,3 +283,15 @@ class AddRepo14Test(BaseTest): super(AddRepo14Test, self).check() # check pool self.check_file_not_empty('pool/00/35/libboost-program-options-dev_1.49.0.1_i386.deb') + + +class AddRepo15Test(BaseTest): + """ + add package with wrong case in stanza and missing fields + """ + fixtureCmds = [ + "aptly repo create -comment=Repo15 -distribution=squeeze repo15", + ] + runCmd = "aptly repo add repo15 ${testfiles}" + outputMatchPrepare = lambda self, s: s.replace(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__), "").replace(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), "") + expectedCode = 1 diff --git a/src/github.com/smira/aptly/system/t10_task/RunTask1Test_gold b/src/github.com/smira/aptly/system/t10_task/RunTask1Test_gold index 54d8db36..25b43e94 100644 --- a/src/github.com/smira/aptly/system/t10_task/RunTask1Test_gold +++ b/src/github.com/smira/aptly/system/t10_task/RunTask1Test_gold @@ -21,6 +21,6 @@ End command output: ------------------------------ 4) [Running]: version Begin command output: ---------------------------- -aptly version: 0.9.1 +aptly version: 0.9.5 End command output: ------------------------------ diff --git a/src/github.com/smira/aptly/system/t12_api/version.py b/src/github.com/smira/aptly/system/t12_api/version.py index e8885230..b0f27d75 100644 --- a/src/github.com/smira/aptly/system/t12_api/version.py +++ b/src/github.com/smira/aptly/system/t12_api/version.py @@ -7,4 +7,4 @@ class VersionAPITest(APITest): """ def check(self): - self.check_equal(self.get("/api/version").json(), {'Version': '0.9.1'}) + self.check_equal(self.get("/api/version").json(), {'Version': '0.9.5'})