Basis for repo re-publishing, cleaning up prefix + component published package pool. #8

cmd/publish_drop.go

@@ -19,7 +19,8 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 		prefix = args[1]
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution)
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
+		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}

deb/publish.go

@@ -678,8 +678,69 @@ func (collection *PublishedRepoCollection) Len() int {
 	return len(collection.list)
 }
 
+// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
+func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix, component string,
+	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+
+	var err error
+	referencedFiles := []string{}
+
+	if progress != nil {
+		progress.Printf("Cleaning up prefix %s component %s...\n", prefix, component)
+	}
+
+	for _, r := range collection.list {
+		if r.Prefix == prefix && r.Component == component {
+			err = collection.LoadComplete(r, collectionFactory)
+			if err != nil {
+				return err
+			}
+
+			packageList, err := NewPackageListFromRefList(r.RefList(), collectionFactory.PackageCollection(), progress)
+			if err != nil {
+				return err
+			}
+
+			packageList.ForEach(func(p *Package) error {
+				poolDir, err := p.PoolDirectory()
+				if err != nil {
+					return err
+				}
+
+				for _, f := range p.Files() {
+					referencedFiles = append(referencedFiles, filepath.Join(poolDir, f.Filename))
+				}
+
+				return nil
+			})
+		}
+	}
+
+	sort.Strings(referencedFiles)
+
+	rootPath := filepath.Join(prefix, "pool", component)
+	existingFiles, err := publishedStorage.Filelist(rootPath)
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(existingFiles)
+
+	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+	for _, file := range filesToDelete {
+		err = publishedStorage.Remove(filepath.Join(rootPath, file))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Remove removes published repository, cleaning up directories, files
-func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string) error {
+func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string,
+	collectionFactory *CollectionFactory, progress aptly.Progress) error {
 	repo, err := collection.ByPrefixDistribution(prefix, distribution)
 	if err != nil {
 		return err
@@ -710,6 +771,13 @@ func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.Publish
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
+	if !removePrefix && !removePoolComponent {
+		err = collection.CleanupPrefixComponentFiles(repo.Prefix, repo.Component, publishedStorage, collectionFactory, progress)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = collection.db.Delete(repo.Key())
 	if err != nil {
 		return err

deb/publish_test.go

@@ -546,7 +546,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
+	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
@@ -564,10 +564,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
+	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza")
+	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza", s.factory, nil)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -580,7 +580,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.publishedStorage, ".", "anaconda")
+	err := s.collection.Remove(s.publishedStorage, ".", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByPrefixDistribution(".", "anaconda")