publish: write skel and AppStream files once per publish

The skeleton and AppStream loops were nested inside the per-component
loop while iterating over all components themselves, so every file was
written once per component. Publishing main + contrib appended each
payload twice. Release checksums are computed after the fact, so the
duplicated content is signed and apt accepts it silently.

AUTHORS

@@ -84,3 +84,4 @@ List of contributors, in chronological order:
 * Zhang Xiao (https://github.com/xzhang1)
 * Tom Nguyen (https://github.com/lecafard)
 * Philip Cramer (https://github.com/PhilipCramer)
+* Kerem Gur (https://github.com/KereMath)

deb/publish.go

@@ -1043,62 +1043,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			}
 		}
 
-		for component := range p.sourceItems {
-			skelFiles, err := p.GetSkelFiles(skelDir, component)
-			if err != nil {
-				return fmt.Errorf("unable to get skeleton files: %v", err)
-			}
-
-			for relPath, absPath := range skelFiles {
-				bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter()
-				if err != nil {
-					return fmt.Errorf("unable to generate skeleton index: %v", err)
-				}
-
-				file, err := os.Open(absPath)
-				if err != nil {
-					return fmt.Errorf("unable to read skeleton file: %v", err)
-				}
-
-				_, err = bufio.NewReader(file).WriteTo(bufWriter)
-				if err != nil {
-					return fmt.Errorf("unable to write skeleton file: %v", err)
-				}
-			}
-		}
-
-		// Pass-through AppStream (DEP-11) files from snapshots
-		for component, item := range p.sourceItems {
-			if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 {
-				continue
-			}
-
-			prefix := component + "/"
-			for relPath, poolPath := range item.snapshot.AppStreamFiles {
-				if !strings.HasPrefix(relPath, prefix) {
-					continue
-				}
-				withinComponent := strings.TrimPrefix(relPath, prefix)
-
-				poolFile, err := packagePool.Open(poolPath)
-				if err != nil {
-					return fmt.Errorf("unable to open AppStream file from pool: %v", err)
-				}
-
-				bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter()
-				if err != nil {
-					_ = poolFile.Close()
-					return fmt.Errorf("unable to generate AppStream index: %v", err)
-				}
-
-				_, err = bufio.NewReader(poolFile).WriteTo(bufWriter)
-				_ = poolFile.Close()
-				if err != nil {
-					return fmt.Errorf("unable to write AppStream file: %v", err)
-				}
-			}
-		}
-
 		udebs := []bool{false}
 		if hadUdebs {
 			udebs = append(udebs, true)
@@ -1144,6 +1088,66 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		}
 	}
 
+	// Skeleton and AppStream (DEP-11) files are keyed by component and already
+	// iterate over every component, so they must be emitted once for the whole
+	// publish -- not once per component, which would append each payload again.
+	for component := range p.sourceItems {
+		skelFiles, err := p.GetSkelFiles(skelDir, component)
+		if err != nil {
+			return fmt.Errorf("unable to get skeleton files: %v", err)
+		}
+
+		for relPath, absPath := range skelFiles {
+			bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter()
+			if err != nil {
+				return fmt.Errorf("unable to generate skeleton index: %v", err)
+			}
+
+			file, err := os.Open(absPath)
+			if err != nil {
+				return fmt.Errorf("unable to read skeleton file: %v", err)
+			}
+
+			_, err = bufio.NewReader(file).WriteTo(bufWriter)
+			_ = file.Close()
+			if err != nil {
+				return fmt.Errorf("unable to write skeleton file: %v", err)
+			}
+		}
+	}
+
+	// Pass-through AppStream (DEP-11) files from snapshots
+	for component, item := range p.sourceItems {
+		if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 {
+			continue
+		}
+
+		prefix := component + "/"
+		for relPath, poolPath := range item.snapshot.AppStreamFiles {
+			if !strings.HasPrefix(relPath, prefix) {
+				continue
+			}
+			withinComponent := strings.TrimPrefix(relPath, prefix)
+
+			poolFile, err := packagePool.Open(poolPath)
+			if err != nil {
+				return fmt.Errorf("unable to open AppStream file from pool: %v", err)
+			}
+
+			bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter()
+			if err != nil {
+				_ = poolFile.Close()
+				return fmt.Errorf("unable to generate AppStream index: %v", err)
+			}
+
+			_, err = bufio.NewReader(poolFile).WriteTo(bufWriter)
+			_ = poolFile.Close()
+			if err != nil {
+				return fmt.Errorf("unable to write AppStream file: %v", err)
+			}
+		}
+	}
+
 	for _, arch := range p.Architectures {
 		for _, udeb := range []bool{true, false} {
 			index := legacyContentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
@@ -1208,8 +1212,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		release["Valid-Until"] = publishDate.AddDate(100, 0, 0).Format(datetimeformat)
 	}
 	if p.Version != "" {
-        release["Version"] = p.Version
-    }
+		release["Version"] = p.Version
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""

deb/publish_test.go

@@ -501,6 +501,42 @@ func (s *PublishedRepoSuite) TestPublishAppStream(c *C) {
 	c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*")
 }
 
+func (s *PublishedRepoSuite) importAppStreamFile(c *C, name string, content []byte) string {
+	tmpFile := filepath.Join(c.MkDir(), name)
+	c.Assert(os.WriteFile(tmpFile, content, 0644), IsNil)
+
+	checksums := utils.ChecksumInfo{Size: int64(len(content))}
+	poolPath, err := s.packagePool.Import(tmpFile, name, &checksums, false, s.cs)
+	c.Assert(err, IsNil)
+
+	return poolPath
+}
+
+func (s *PublishedRepoSuite) TestPublishAppStreamMultipleComponents(c *C) {
+	mainContent := []byte("DEP-11 content for main\n")
+	contribContent := []byte("DEP-11 content for contrib\n")
+
+	s.snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": s.importAppStreamFile(c, "Components-main.yml.gz", mainContent),
+	}
+	s.snapshot2.AppStreamFiles = map[string]string{
+		"contrib/dep11/Components-amd64.yml.gz": s.importAppStreamFile(c, "Components-contrib.yml.gz", contribContent),
+	}
+
+	err := s.repo3.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	base := filepath.Join(s.publishedStorage.PublicPath(), "linux/dists/natty")
+
+	published, err := os.ReadFile(filepath.Join(base, "main/dep11/Components-amd64.yml.gz"))
+	c.Assert(err, IsNil)
+	c.Check(published, DeepEquals, mainContent)
+
+	published, err = os.ReadFile(filepath.Join(base, "contrib/dep11/Components-amd64.yml.gz"))
+	c.Assert(err, IsNil)
+	c.Check(published, DeepEquals, contribContent)
+}
+
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)