From 23c5a319226a1dd4a74ba1ded149fd1ee121a9f8 Mon Sep 17 00:00:00 2001 From: KereMath Date: Thu, 20 Aug 2026 03:09:09 +0300 Subject: [PATCH] publish: write skel and AppStream files once per publish The skeleton and AppStream loops were nested inside the per-component loop while iterating over all components themselves, so every file was written once per component. Publishing main + contrib appended each payload twice. Release checksums are computed after the fact, so the duplicated content is signed and apt accepts it silently. --- AUTHORS | 1 + deb/publish.go | 120 +++++++++++++++++++++++--------------------- deb/publish_test.go | 36 +++++++++++++ 3 files changed, 99 insertions(+), 58 deletions(-) diff --git a/AUTHORS b/AUTHORS index ea69b298..0dd5f481 100644 --- a/AUTHORS +++ b/AUTHORS @@ -84,3 +84,4 @@ List of contributors, in chronological order: * Zhang Xiao (https://github.com/xzhang1) * Tom Nguyen (https://github.com/lecafard) * Philip Cramer (https://github.com/PhilipCramer) +* Kerem Gur (https://github.com/KereMath) diff --git a/deb/publish.go b/deb/publish.go index 8ae71df9..4945623f 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1043,62 +1043,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } - for component := range p.sourceItems { - skelFiles, err := p.GetSkelFiles(skelDir, component) - if err != nil { - return fmt.Errorf("unable to get skeleton files: %v", err) - } - - for relPath, absPath := range skelFiles { - bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter() - if err != nil { - return fmt.Errorf("unable to generate skeleton index: %v", err) - } - - file, err := os.Open(absPath) - if err != nil { - return fmt.Errorf("unable to read skeleton file: %v", err) - } - - _, err = bufio.NewReader(file).WriteTo(bufWriter) - if err != nil { - return fmt.Errorf("unable to write skeleton file: %v", err) - } - } - } - - // Pass-through AppStream (DEP-11) files from snapshots - for component, item := range p.sourceItems { - if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 { - continue - } - - prefix := component + "/" - for relPath, poolPath := range item.snapshot.AppStreamFiles { - if !strings.HasPrefix(relPath, prefix) { - continue - } - withinComponent := strings.TrimPrefix(relPath, prefix) - - poolFile, err := packagePool.Open(poolPath) - if err != nil { - return fmt.Errorf("unable to open AppStream file from pool: %v", err) - } - - bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter() - if err != nil { - _ = poolFile.Close() - return fmt.Errorf("unable to generate AppStream index: %v", err) - } - - _, err = bufio.NewReader(poolFile).WriteTo(bufWriter) - _ = poolFile.Close() - if err != nil { - return fmt.Errorf("unable to write AppStream file: %v", err) - } - } - } - udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) @@ -1144,6 +1088,66 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } + // Skeleton and AppStream (DEP-11) files are keyed by component and already + // iterate over every component, so they must be emitted once for the whole + // publish -- not once per component, which would append each payload again. + for component := range p.sourceItems { + skelFiles, err := p.GetSkelFiles(skelDir, component) + if err != nil { + return fmt.Errorf("unable to get skeleton files: %v", err) + } + + for relPath, absPath := range skelFiles { + bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter() + if err != nil { + return fmt.Errorf("unable to generate skeleton index: %v", err) + } + + file, err := os.Open(absPath) + if err != nil { + return fmt.Errorf("unable to read skeleton file: %v", err) + } + + _, err = bufio.NewReader(file).WriteTo(bufWriter) + _ = file.Close() + if err != nil { + return fmt.Errorf("unable to write skeleton file: %v", err) + } + } + } + + // Pass-through AppStream (DEP-11) files from snapshots + for component, item := range p.sourceItems { + if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 { + continue + } + + prefix := component + "/" + for relPath, poolPath := range item.snapshot.AppStreamFiles { + if !strings.HasPrefix(relPath, prefix) { + continue + } + withinComponent := strings.TrimPrefix(relPath, prefix) + + poolFile, err := packagePool.Open(poolPath) + if err != nil { + return fmt.Errorf("unable to open AppStream file from pool: %v", err) + } + + bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter() + if err != nil { + _ = poolFile.Close() + return fmt.Errorf("unable to generate AppStream index: %v", err) + } + + _, err = bufio.NewReader(poolFile).WriteTo(bufWriter) + _ = poolFile.Close() + if err != nil { + return fmt.Errorf("unable to write AppStream file: %v", err) + } + } + } + for _, arch := range p.Architectures { for _, udeb := range []bool{true, false} { index := legacyContentIndexes[fmt.Sprintf("%s-%v", arch, udeb)] @@ -1208,8 +1212,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Valid-Until"] = publishDate.AddDate(100, 0, 0).Format(datetimeformat) } if p.Version != "" { - release["Version"] = p.Version - } + release["Version"] = p.Version + } release["Description"] = " Generated by aptly\n" release["MD5Sum"] = "" release["SHA1"] = "" diff --git a/deb/publish_test.go b/deb/publish_test.go index 20f78c4f..0459c113 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -501,6 +501,42 @@ func (s *PublishedRepoSuite) TestPublishAppStream(c *C) { c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*") } +func (s *PublishedRepoSuite) importAppStreamFile(c *C, name string, content []byte) string { + tmpFile := filepath.Join(c.MkDir(), name) + c.Assert(os.WriteFile(tmpFile, content, 0644), IsNil) + + checksums := utils.ChecksumInfo{Size: int64(len(content))} + poolPath, err := s.packagePool.Import(tmpFile, name, &checksums, false, s.cs) + c.Assert(err, IsNil) + + return poolPath +} + +func (s *PublishedRepoSuite) TestPublishAppStreamMultipleComponents(c *C) { + mainContent := []byte("DEP-11 content for main\n") + contribContent := []byte("DEP-11 content for contrib\n") + + s.snapshot.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": s.importAppStreamFile(c, "Components-main.yml.gz", mainContent), + } + s.snapshot2.AppStreamFiles = map[string]string{ + "contrib/dep11/Components-amd64.yml.gz": s.importAppStreamFile(c, "Components-contrib.yml.gz", contribContent), + } + + err := s.repo3.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") + c.Assert(err, IsNil) + + base := filepath.Join(s.publishedStorage.PublicPath(), "linux/dists/natty") + + published, err := os.ReadFile(filepath.Join(base, "main/dep11/Components-amd64.yml.gz")) + c.Assert(err, IsNil) + c.Check(published, DeepEquals, mainContent) + + published, err = os.ReadFile(filepath.Join(base, "contrib/dep11/Components-amd64.yml.gz")) + c.Assert(err, IsNil) + c.Check(published, DeepEquals, contribContent) +} + func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil)