aptly/deb/index_files.go

package deb

import (
	"bufio"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/smira/aptly/aptly"
	"github.com/smira/aptly/pgp"
	"github.com/smira/aptly/utils"
)

type indexFiles struct {
	publishedStorage aptly.PublishedStorage
	basePath         string
	renameMap        map[string]string
	generatedFiles   map[string]utils.ChecksumInfo
	tempDir          string
	suffix           string
	indexes          map[string]*indexFile
	accessByHash     bool
}

type indexFile struct {
	parent       *indexFiles
	discardable  bool
	compressable bool
	onlyGzip     bool
	signable     bool
	accessByHash bool
	relativePath string
	tempFilename string
	tempFile     *os.File
	w            *bufio.Writer
}

func (file *indexFile) BufWriter() (*bufio.Writer, error) {
	if file.w == nil {
		var err error
		file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
		file.tempFile, err = os.Create(file.tempFilename)
		if err != nil {
			return nil, fmt.Errorf("unable to create temporary index file: %s", err)
		}

		file.w = bufio.NewWriter(file.tempFile)
	}

	return file.w, nil
}

func (file *indexFile) Finalize(signer pgp.Signer) error {
	if file.w == nil {
		if file.discardable {
			return nil
		}
		file.BufWriter()
	}

	err := file.w.Flush()
	if err != nil {
		file.tempFile.Close()
		return fmt.Errorf("unable to write to index file: %s", err)
	}

	if file.compressable {
		err = utils.CompressFile(file.tempFile, file.onlyGzip)
		if err != nil {
			file.tempFile.Close()
			return fmt.Errorf("unable to compress index file: %s", err)
		}
	}

	file.tempFile.Close()

	exts := []string{""}
	if file.compressable {
		exts = append(exts, ".gz", ".bz2")
		if file.onlyGzip {
			exts = []string{".gz"}
		}
	}

	for _, ext := range exts {
		var checksumInfo utils.ChecksumInfo

		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
		if err != nil {
			return fmt.Errorf("unable to collect checksums: %s", err)
		}
		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
	}

	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))

	err = file.parent.publishedStorage.MkDir(filedir)
	if err != nil {
		return fmt.Errorf("unable to create dir: %s", err)
	}

	hashs := []string{}
	if file.accessByHash {
		hashs = append(hashs, "MD5", "SHA1", "SHA256", "SHA512")
		for _, hash := range hashs {
			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
			if err != nil {
				return fmt.Errorf("unable to create dir: %s", err)
			}
		}
	}

	for _, ext := range exts {
		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
			file.tempFilename+ext)
		if err != nil {
			return fmt.Errorf("unable to publish file: %s", err)
		}

		if file.parent.suffix != "" {
			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
				filepath.Join(file.parent.basePath, file.relativePath+ext)
		}

		if file.accessByHash {
			sums := file.parent.generatedFiles[file.relativePath+ext]

			err = packageIndexByHash(file, ext, "SHA512", sums.SHA512)
			if err != nil {
				fmt.Printf("%s\n", err)
			}
			err = packageIndexByHash(file, ext, "SHA256", sums.SHA256)
			if err != nil {
				fmt.Printf("%s\n", err)
			}
			err = packageIndexByHash(file, ext, "SHA1", sums.SHA1)
			if err != nil {
				fmt.Printf("%s\n", err)
			}
			err = packageIndexByHash(file, ext, "MD5", sums.MD5)
			if err != nil {
				fmt.Printf("%s\n", err)
			}
		}
	}

	if file.signable && signer != nil {
		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
		if err != nil {
			return fmt.Errorf("unable to detached sign file: %s", err)
		}

		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
		if err != nil {
			return fmt.Errorf("unable to clearsign file: %s", err)
		}

		if file.parent.suffix != "" {
			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
				filepath.Join(file.parent.basePath, "In"+file.relativePath)
		}

		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
			file.tempFilename+".gpg")
		if err != nil {
			return fmt.Errorf("unable to publish file: %s", err)
		}

		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
		if err != nil {
			return fmt.Errorf("unable to publish file: %s", err)
		}
	}

	return nil
}

func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
	src := filepath.Join(file.parent.basePath, file.relativePath)
	indexfile := path.Base(src + ext)
	src = src + file.parent.suffix + ext
	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
	dst := filepath.Join(filedir, "by-hash", hash)
	sumfilePath := filepath.Join(dst, sum)

	// link already exists? do nothing
	if file.parent.publishedStorage.FileExists(sumfilePath) {
		return nil
	}

	// create the link
	err := file.parent.publishedStorage.HardLink(src, sumfilePath)
	if err != nil {
		return fmt.Errorf("Access-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
	}

	// if a previous index file already exists exists, backup symlink
	if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)) {
		// if exists, remove old symlink
		if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")) {
			link, err := file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
			if err != nil {
				file.parent.publishedStorage.Remove(link)
			}
			file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
		}
		file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
			filepath.Join(dst, indexfile+".old"))
	}

	// create symlink
	err = file.parent.publishedStorage.SymLink(sum, filepath.Join(dst, indexfile))
	if err != nil {
		return fmt.Errorf("Access-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile))
	}
	return nil
}

func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, accessByHash bool) *indexFiles {
	return &indexFiles{
		publishedStorage: publishedStorage,
		basePath:         basePath,
		renameMap:        make(map[string]string),
		generatedFiles:   make(map[string]utils.ChecksumInfo),
		tempDir:          tempDir,
		suffix:           suffix,
		indexes:          make(map[string]*indexFile),
		accessByHash:     accessByHash,
	}
}

func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
	if arch == ArchitectureSource {
		udeb = false
	}
	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
	file, ok := files.indexes[key]
	if !ok {
		var relativePath string

		if arch == ArchitectureSource {
			relativePath = filepath.Join(component, "source", "Sources")
		} else {
			if udeb {
				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
			} else {
				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
			}
		}

		file = &indexFile{
			parent:       files,
			discardable:  false,
			compressable: true,
			signable:     false,
			accessByHash: files.accessByHash,
			relativePath: relativePath,
		}

		files.indexes[key] = file
	}

	return file
}

func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
	if arch == ArchitectureSource {
		udeb = false
	}
	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
	file, ok := files.indexes[key]
	if !ok {
		var relativePath string

		if arch == ArchitectureSource {
			relativePath = filepath.Join(component, "source", "Release")
		} else {
			if udeb {
				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
			} else {
				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
			}
		}

		file = &indexFile{
			parent:       files,
			discardable:  udeb,
			compressable: false,
			signable:     false,
			accessByHash: files.accessByHash,
			relativePath: relativePath,
		}

		files.indexes[key] = file
	}

	return file
}

func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
	if arch == ArchitectureSource {
		udeb = false
	}
	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
	file, ok := files.indexes[key]
	if !ok {
		var relativePath string

		if udeb {
			relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
		} else {
			relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
		}

		file = &indexFile{
			parent:       files,
			discardable:  true,
			compressable: true,
			onlyGzip:     true,
			signable:     false,
			accessByHash: files.accessByHash,
			relativePath: relativePath,
		}

		files.indexes[key] = file
	}

	return file
}

func (files *indexFiles) ReleaseFile() *indexFile {
	return &indexFile{
		parent:       files,
		discardable:  false,
		compressable: false,
		signable:     true,
		relativePath: "Release",
	}
}

func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
	if progress != nil {
		progress.InitBar(int64(len(files.indexes)), false)
		defer progress.ShutdownBar()
	}

	for _, file := range files.indexes {
		err = file.Finalize(nil)
		if err != nil {
			return
		}
		if progress != nil {
			progress.AddBar(1)
		}
	}

	files.indexes = make(map[string]*indexFile)

	return
}

func (files *indexFiles) RenameFiles() error {
	var err error

	for oldName, newName := range files.renameMap {
		err = files.publishedStorage.RenameFile(oldName, newName)
		if err != nil {
			return fmt.Errorf("unable to rename: %s", err)
		}
	}

	return nil
}