mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-05 05:20:34 +00:00
Harald Sitter
00bb0ca8f3
the logic here was wrong.
if we managed to find the link target (the physical index file) pointed to
by our old symlink we want to remove it (this is basically "cleaning up old
index" logic).
previously we'd try to only delete it when the ReadLink came back with
error. which had two serious issues with it:
a) linkTarget was empty, so we basically called Remove("") which would
delete the storage -> root <- directory if the root is a symlink!
b) we'd leak old indexes as the cleanup logic only ran if there was en
error which would ordinarily never be
new code correctly cleans up unless there was an error.
this relates to a previous bugfix of readLink which incorrectly returned
absolute paths ultimately rendering the Remove call also broken.
375 lines
10 KiB
Go
375 lines
10 KiB
Go
package deb
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/smira/aptly/aptly"
|
|
"github.com/smira/aptly/pgp"
|
|
"github.com/smira/aptly/utils"
|
|
)
|
|
|
|
type indexFiles struct {
|
|
publishedStorage aptly.PublishedStorage
|
|
basePath string
|
|
renameMap map[string]string
|
|
generatedFiles map[string]utils.ChecksumInfo
|
|
tempDir string
|
|
suffix string
|
|
indexes map[string]*indexFile
|
|
acquireByHash bool
|
|
}
|
|
|
|
type indexFile struct {
|
|
parent *indexFiles
|
|
discardable bool
|
|
compressable bool
|
|
onlyGzip bool
|
|
signable bool
|
|
acquireByHash bool
|
|
relativePath string
|
|
tempFilename string
|
|
tempFile *os.File
|
|
w *bufio.Writer
|
|
}
|
|
|
|
func (file *indexFile) BufWriter() (*bufio.Writer, error) {
|
|
if file.w == nil {
|
|
var err error
|
|
file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
|
|
file.tempFile, err = os.Create(file.tempFilename)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create temporary index file: %s", err)
|
|
}
|
|
|
|
file.w = bufio.NewWriter(file.tempFile)
|
|
}
|
|
|
|
return file.w, nil
|
|
}
|
|
|
|
func (file *indexFile) Finalize(signer pgp.Signer) error {
|
|
if file.w == nil {
|
|
if file.discardable {
|
|
return nil
|
|
}
|
|
file.BufWriter()
|
|
}
|
|
|
|
err := file.w.Flush()
|
|
if err != nil {
|
|
file.tempFile.Close()
|
|
return fmt.Errorf("unable to write to index file: %s", err)
|
|
}
|
|
|
|
if file.compressable {
|
|
err = utils.CompressFile(file.tempFile, file.onlyGzip)
|
|
if err != nil {
|
|
file.tempFile.Close()
|
|
return fmt.Errorf("unable to compress index file: %s", err)
|
|
}
|
|
}
|
|
|
|
file.tempFile.Close()
|
|
|
|
exts := []string{""}
|
|
if file.compressable {
|
|
exts = append(exts, ".gz", ".bz2")
|
|
if file.onlyGzip {
|
|
exts = []string{".gz"}
|
|
}
|
|
}
|
|
|
|
for _, ext := range exts {
|
|
var checksumInfo utils.ChecksumInfo
|
|
|
|
checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to collect checksums: %s", err)
|
|
}
|
|
file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
|
|
}
|
|
|
|
filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
|
|
|
|
err = file.parent.publishedStorage.MkDir(filedir)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create dir: %s", err)
|
|
}
|
|
|
|
if file.acquireByHash {
|
|
for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
|
|
err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create dir: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, ext := range exts {
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
|
|
file.tempFilename+ext)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
|
|
if file.parent.suffix != "" {
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
|
|
filepath.Join(file.parent.basePath, file.relativePath+ext)
|
|
}
|
|
|
|
if file.acquireByHash {
|
|
sums := file.parent.generatedFiles[file.relativePath+ext]
|
|
for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
|
|
err = packageIndexByHash(file, ext, hash, sum)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to build hash file: %s", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if file.signable && signer != nil {
|
|
err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
|
|
if err != nil {
|
|
return fmt.Errorf("unable to detached sign file: %s", err)
|
|
}
|
|
|
|
err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to clearsign file: %s", err)
|
|
}
|
|
|
|
if file.parent.suffix != "" {
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
|
|
filepath.Join(file.parent.basePath, file.relativePath+".gpg")
|
|
file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
|
|
filepath.Join(file.parent.basePath, "In"+file.relativePath)
|
|
}
|
|
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
|
|
file.tempFilename+".gpg")
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
|
|
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
|
|
filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
|
|
if err != nil {
|
|
return fmt.Errorf("unable to publish file: %s", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
|
|
src := filepath.Join(file.parent.basePath, file.relativePath)
|
|
indexfile := path.Base(src + ext)
|
|
src = src + file.parent.suffix + ext
|
|
filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
|
|
dst := filepath.Join(filedir, "by-hash", hash)
|
|
sumfilePath := filepath.Join(dst, sum)
|
|
|
|
// link already exists? do nothing
|
|
exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
|
|
}
|
|
if exists {
|
|
return nil
|
|
}
|
|
|
|
// create the link
|
|
err = file.parent.publishedStorage.HardLink(src, sumfilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
|
|
}
|
|
|
|
// if a previous index file already exists exists, backup symlink
|
|
indexPath := filepath.Join(dst, indexfile)
|
|
oldIndexPath := filepath.Join(dst, indexfile+".old")
|
|
if exists, _ = file.parent.publishedStorage.FileExists(indexPath); exists {
|
|
// if exists, remove old symlink
|
|
if exists, _ = file.parent.publishedStorage.FileExists(oldIndexPath); exists {
|
|
var linkTarget string
|
|
linkTarget, err = file.parent.publishedStorage.ReadLink(oldIndexPath)
|
|
if err == nil {
|
|
// If we managed to resolve the link target: delete it. This is the
|
|
// oldest physical index file we no longer need. Once we drop our
|
|
// old symlink we'll essentially forget about it existing at all.
|
|
file.parent.publishedStorage.Remove(linkTarget)
|
|
}
|
|
file.parent.publishedStorage.Remove(oldIndexPath)
|
|
}
|
|
file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
|
|
}
|
|
|
|
// create symlink
|
|
err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
|
|
if err != nil {
|
|
return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
|
|
return &indexFiles{
|
|
publishedStorage: publishedStorage,
|
|
basePath: basePath,
|
|
renameMap: make(map[string]string),
|
|
generatedFiles: make(map[string]utils.ChecksumInfo),
|
|
tempDir: tempDir,
|
|
suffix: suffix,
|
|
indexes: make(map[string]*indexFile),
|
|
acquireByHash: acquireByHash,
|
|
}
|
|
}
|
|
|
|
func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if arch == ArchitectureSource {
|
|
relativePath = filepath.Join(component, "source", "Sources")
|
|
} else {
|
|
if udeb {
|
|
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
|
|
}
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: false,
|
|
compressable: true,
|
|
signable: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if arch == ArchitectureSource {
|
|
relativePath = filepath.Join(component, "source", "Release")
|
|
} else {
|
|
if udeb {
|
|
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
|
|
}
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: udeb,
|
|
compressable: false,
|
|
signable: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
|
|
if arch == ArchitectureSource {
|
|
udeb = false
|
|
}
|
|
key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
|
|
file, ok := files.indexes[key]
|
|
if !ok {
|
|
var relativePath string
|
|
|
|
if udeb {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
|
|
} else {
|
|
relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
|
|
}
|
|
|
|
file = &indexFile{
|
|
parent: files,
|
|
discardable: true,
|
|
compressable: true,
|
|
onlyGzip: true,
|
|
signable: false,
|
|
acquireByHash: files.acquireByHash,
|
|
relativePath: relativePath,
|
|
}
|
|
|
|
files.indexes[key] = file
|
|
}
|
|
|
|
return file
|
|
}
|
|
|
|
func (files *indexFiles) ReleaseFile() *indexFile {
|
|
return &indexFile{
|
|
parent: files,
|
|
discardable: false,
|
|
compressable: false,
|
|
signable: true,
|
|
relativePath: "Release",
|
|
}
|
|
}
|
|
|
|
func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
|
|
if progress != nil {
|
|
progress.InitBar(int64(len(files.indexes)), false)
|
|
defer progress.ShutdownBar()
|
|
}
|
|
|
|
for _, file := range files.indexes {
|
|
err = file.Finalize(nil)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if progress != nil {
|
|
progress.AddBar(1)
|
|
}
|
|
}
|
|
|
|
files.indexes = make(map[string]*indexFile)
|
|
|
|
return
|
|
}
|
|
|
|
func (files *indexFiles) RenameFiles() error {
|
|
var err error
|
|
|
|
for oldName, newName := range files.renameMap {
|
|
err = files.publishedStorage.RenameFile(oldName, newName)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to rename: %s", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|