aptly/deb/list.go

package deb

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/aptly-dev/aptly/aptly"
	"github.com/aptly-dev/aptly/utils"
)

// Dependency options
const (
	// DepFollowSource pulls source packages when required
	DepFollowSource = 1 << iota
	// DepFollowSuggests pulls from suggests
	DepFollowSuggests
	// DepFollowRecommends pulls from recommends
	DepFollowRecommends
	// DepFollowAllVariants follows all variants if depends on "a | b"
	DepFollowAllVariants
	// DepFollowBuild pulls build dependencies
	DepFollowBuild
	// DepVerboseResolve emits additional logs while dependencies are being resolved
	DepVerboseResolve
)

// PackageList is list of unique (by key) packages
//
// It could be seen as repo snapshot, repo contents, result of filtering,
// merge, etc.
//
// If indexed, PackageList starts supporting searching
type PackageList struct {
	// Straight list of packages as map
	packages map[string]*Package
	// Indexed list of packages, sorted by name internally
	packagesIndex []*Package
	// Map of packages for each virtual package (provides)
	providesIndex map[string][]*Package
	// Package key generation function
	keyFunc func(p *Package) string
	// Allow duplicates?
	duplicatesAllowed bool
	// Has index been prepared?
	indexed bool
}

// PackageConflictError means that package can't be added to the list due to error
type PackageConflictError struct {
	error
}

// Verify interface
var (
	_ sort.Interface = &PackageList{}
	_ PackageCatalog = &PackageList{}
)

func packageShortKey(p *Package) string {
	return string(p.ShortKey(""))
}

func packageFullKey(p *Package) string {
	return string(p.Key(""))
}

// NewPackageList creates empty package list without duplicate package
func NewPackageList() *PackageList {
	return NewPackageListWithDuplicates(false, 1000)
}

// NewPackageListWithDuplicates creates empty package list which might allow or block duplicate packages
func NewPackageListWithDuplicates(duplicates bool, capacity int) *PackageList {
	if capacity == 0 {
		capacity = 1000
	}

	result := &PackageList{
		packages:          make(map[string]*Package, capacity),
		duplicatesAllowed: duplicates,
		keyFunc:           packageShortKey,
	}

	if duplicates {
		result.keyFunc = packageFullKey
	}

	return result
}

// NewPackageListFromRefList loads packages list from PackageRefList
func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageCollection, progress aptly.Progress) (*PackageList, error) {
	// empty reflist
	if reflist == nil {
		return NewPackageList(), nil
	}

	result := NewPackageListWithDuplicates(false, reflist.Len())

	if progress != nil {
		progress.InitBar(int64(reflist.Len()), false, aptly.BarGeneralBuildPackageList)
	}

	err := reflist.ForEach(func(key []byte) error {
		p, err2 := collection.ByKey(key)
		if err2 != nil {
			return fmt.Errorf("unable to load package with key %s: %s", key, err2)
		}
		if progress != nil {
			progress.AddBar(1)
		}
		return result.Add(p)
	})

	if progress != nil {
		progress.ShutdownBar()
	}

	if err != nil {
		return nil, err
	}

	return result, nil
}

// Has checks whether package is already in the list
func (l *PackageList) Has(p *Package) bool {
	key := l.keyFunc(p)
	_, ok := l.packages[key]

	return ok
}

// Add appends package to package list, additionally checking for uniqueness
func (l *PackageList) Add(p *Package) error {
	key := l.keyFunc(p)
	existing, ok := l.packages[key]
	if ok {
		if !existing.Equals(p) {
			return &PackageConflictError{fmt.Errorf("package already exists and is different: %s", p)}
		}
		return nil
	}
	l.packages[key] = p

	if l.indexed {
		for _, provides := range p.ProvidedPackages() {
			l.providesIndex[provides] = append(l.providesIndex[provides], p)
		}

		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })

		// insert p into l.packagesIndex in position i
		l.packagesIndex = append(l.packagesIndex, nil)
		copy(l.packagesIndex[i+1:], l.packagesIndex[i:])
		l.packagesIndex[i] = p
	}
	return nil
}

// ForEach calls handler for each package in list
func (l *PackageList) ForEach(handler func(*Package) error) error {
	var err error
	for _, p := range l.packages {
		err = handler(p)
		if err != nil {
			return err
		}
	}
	return err
}

// FilterLatest creates a copy of the package list containing only the
// latest version for each package name/architecture pair.
func (l *PackageList) FilterLatest() (*PackageList, error) {
	if l == nil {
		return nil, fmt.Errorf("package list is nil")
	}

	filtered := make(map[string]*Package, l.Len())

	err := l.ForEach(func(p *Package) error {
		key := p.Architecture + "|" + p.Name

		existing, found := filtered[key]
		if !found {
			filtered[key] = p
			return nil
		}

		cmp, compareErr := CompareVersions(p.Version, existing.Version)
		switch compareErr {
		case nil:
			if cmp > 0 {
				filtered[key] = p
			}
		case RightInvalid:
			filtered[key] = p
		case LeftInvalid, BothInvalid:
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))

	for _, pkg := range filtered {
		if err = result.Add(pkg); err != nil {
			return nil, err
		}
	}

	return result, nil
}

// ForEachIndexed calls handler for each package in list in indexed order
func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
	if !l.indexed {
		panic("list not indexed, can't iterate")
	}

	var err error
	for _, p := range l.packagesIndex {
		err = handler(p)
		if err != nil {
			return err
		}
	}
	return err
}

// Len returns number of packages in the list
func (l *PackageList) Len() int {
	return len(l.packages)
}

// Append adds content from one package list to another
func (l *PackageList) Append(pl *PackageList) error {
	if l.indexed {
		panic("Append not supported when indexed")
	}
	for k, p := range pl.packages {
		existing, ok := l.packages[k]
		if ok {
			if !existing.Equals(p) {
				return fmt.Errorf("package already exists and is different: %s", p)
			}
		} else {
			l.packages[k] = p
		}
	}

	return nil
}

// Remove removes package from the list, and updates index when required
func (l *PackageList) Remove(p *Package) {
	delete(l.packages, l.keyFunc(p))
	if l.indexed {
		for _, provides := range p.ProvidedPackages() {
			for i, pkg := range l.providesIndex[provides] {
				if pkg.Equals(p) {
					// remove l.ProvidesIndex[provides][i] w/o preserving order
					l.providesIndex[provides][len(l.providesIndex[provides])-1], l.providesIndex[provides][i], l.providesIndex[provides] =
						nil, l.providesIndex[provides][len(l.providesIndex[provides])-1], l.providesIndex[provides][:len(l.providesIndex[provides])-1]
					break
				}
			}
		}

		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == p.Name {
			if l.packagesIndex[i].Equals(p) {
				// remove l.packagesIndex[i] preserving order
				copy(l.packagesIndex[i:], l.packagesIndex[i+1:])
				l.packagesIndex[len(l.packagesIndex)-1] = nil
				l.packagesIndex = l.packagesIndex[:len(l.packagesIndex)-1]
				break
			}
			i++
		}
	}
}

// Architectures returns list of architectures present in packages and flag if source packages are present.
//
// If includeSource is true, meta-architecture "source" would be present in the list
func (l *PackageList) Architectures(includeSource bool) (result []string) {
	result = make([]string, 0, 10)
	for _, pkg := range l.packages {
		if pkg.Architecture != ArchitectureAll && (pkg.Architecture != ArchitectureSource || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
			result = append(result, pkg.Architecture)
		}
	}
	return
}

// Strings builds list of strings with package keys
func (l *PackageList) Strings() []string {
	result := make([]string, l.Len())
	i := 0

	for _, p := range l.packages {
		result[i] = string(p.Key(""))
		i++
	}

	return result
}

// FullNames builds a list of package {name}_{version}_{arch}
func (l *PackageList) FullNames() []string {
	result := make([]string, l.Len())
	i := 0

	for _, p := range l.packages {
		result[i] = p.GetFullName()
		i++
	}

	return result
}

// depSliceDeduplicate removes dups in slice of Dependencies
func depSliceDeduplicate(s []Dependency) []Dependency {
	l := len(s)
	if l < 2 {
		return s
	}
	if l == 2 {
		if s[0] == s[1] {
			return s[0:1]
		}
		return s
	}

	found := make(map[string]bool, l)
	j := 0
	for i, x := range s {
		h := x.Hash()
		if !found[h] {
			found[h] = true
			s[j] = s[i]
			j++
		}
	}

	return s[:j]
}

// VerifyDependencies looks for missing dependencies in package list.
//
// Analysis would be performed for each architecture, in specified sources
func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
	l.PrepareIndex()
	missing := make([]Dependency, 0, 128)

	if progress != nil {
		progress.InitBar(int64(l.Len())*int64(len(architectures)), false, aptly.BarGeneralVerifyDependencies)
	}

	if len(architectures) == 0 {
		return nil, fmt.Errorf("no architectures defined, cannot verify dependencies")
	}
	for _, arch := range architectures {
		cache := make(map[string]bool, 2048)

		for _, p := range l.packagesIndex {
			if progress != nil {
				progress.AddBar(1)
			}

			if !p.MatchesArchitecture(arch) {
				continue
			}

			for _, dep := range p.GetDependencies(options) {
				variants, err := ParseDependencyVariants(dep)
				if err != nil {
					return nil, fmt.Errorf("unable to process package %s: %s", p, err)
				}

				variants = depSliceDeduplicate(variants)

				variantsMissing := make([]Dependency, 0, len(variants))

				for _, dep := range variants {
					if dep.Architecture == "" {
						dep.Architecture = arch
					}

					hash := dep.Hash()
					satisfied, ok := cache[hash]
					if !ok {
						satisfied = sources.Search(dep, false, true) != nil
						cache[hash] = satisfied
					}

					if !satisfied {
						variantsMissing = append(variantsMissing, dep)
					}

					if satisfied && options&DepFollowAllVariants == 0 {
						variantsMissing = nil
						break
					}
				}

				missing = append(missing, variantsMissing...)
			}
		}
	}

	missing = depSliceDeduplicate(missing)

	if progress != nil {
		progress.ShutdownBar()
	}

	if options&DepVerboseResolve == DepVerboseResolve && progress != nil {
		missingStr := make([]string, len(missing))
		for i := range missing {
			missingStr[i] = missing[i].String()
		}
		progress.ColoredPrintf("@{y}Missing dependencies:@| %s", strings.Join(missingStr, ", "))
	}

	return missing, nil
}

// Swap swaps two packages in index
func (l *PackageList) Swap(i, j int) {
	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
}

func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
	if iPkg.Name == jPkg.Name {
		cmp, err := CompareVersions(iPkg.Version, jPkg.Version)
		switch err {
		case LeftInvalid, BothInvalid:
			return false
		case RightInvalid:
			return true
		}
		if cmp == 0 {
			return iPkg.Architecture < jPkg.Architecture
		}
		return cmp == 1
	}

	return iPkg.Name < jPkg.Name
}

// Less compares two packages by name (lexographical) and version (latest to oldest)
func (l *PackageList) Less(i, j int) bool {
	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
}

// PrepareIndex prepares list for indexing
func (l *PackageList) PrepareIndex() {
	if l.indexed {
		return
	}

	l.packagesIndex = make([]*Package, l.Len())
	l.providesIndex = make(map[string][]*Package, 128)

	i := 0
	for _, p := range l.packages {
		l.packagesIndex[i] = p
		i++

		for _, provides := range p.ProvidedPackages() {
			l.providesIndex[provides] = append(l.providesIndex[provides], p)
		}
	}

	sort.Sort(l)

	l.indexed = true
}

// Scan searches package index using full scan
func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
	for _, pkg := range l.packages {
		if q.Matches(pkg) {
			_ = result.Add(pkg)
		}
	}

	return
}

// SearchSupported returns true for PackageList
func (l *PackageList) SearchSupported() bool {
	return true
}

// SearchByKey looks up package by exact key reference
func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)

	pkg := l.packages["P"+arch+" "+name+" "+version]
	if pkg != nil {
		_ = result.Add(pkg)
	}

	return
}

// Search searches package index for specified package(s) using optimized queries
func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided bool) (searchResults []*Package) {
	if !l.indexed {
		panic("list not indexed, can't search")
	}

	i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })

	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
		p := l.packagesIndex[i]
		if p.MatchesDependency(dep) {
			searchResults = append(searchResults, p)

			if !allMatches {
				return
			}
		}

		i++
	}

	if searchProvided {
		providers, ok := l.providesIndex[dep.Pkg]
		if !ok {
			return
		}
		for _, p := range providers {
			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
				if p.MatchesDependency(dep) {
					searchResults = append(searchResults, p)
				}

				if !allMatches {
					return
				}
			}
		}
	}

	return
}

// FilterOptions specifies options for Filter()
type FilterOptions struct {
	Queries           []PackageQuery
	WithDependencies  bool
	WithSources       bool // Source packages corresponding to binary packages are included
	Source            *PackageList
	DependencyOptions int
	Architectures     []string
	Progress          aptly.Progress // set to non-nil value to report progress
}

// SourceRegex is a regular expression to match source package names.
// > In a binary package control file [...], the source package name may be followed by a version number in
// > parentheses. This version number may be omitted [...] if it has the same value as the Version field of
// > the binary package in question.
// > [...]
// > Package names (both source and binary, see Package) must consist only of lower case letters (a-z),
// > digits (0-9), plus (+) and minus (-) signs, and periods (.).
// > They must be at least two characters long and must start with an alphanumeric character.
// -- https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-source
var SourceRegex = regexp.MustCompile(`^([a-z0-9][-+.a-z0-9]+)(?:\s+\(([^)]+)\))?$`)

// Filter filters package index by specified queries (ORed together), possibly pulling dependencies
func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) {
	if !l.indexed {
		panic("list not indexed, can't filter")
	}

	result := NewPackageList()

	for _, query := range options.Queries {
		_ = result.Append(query.Query(l))
	}
	// The above loop already finds source packages that are named equal to their binary package, but we still need
	// to account for those that are named differently.
	if options.WithSources {
		sourceQueries := make([]PackageQuery, 0)
		for _, pkg := range result.packages {
			if pkg.Source == "" {
				continue
			}
			matches := SourceRegex.FindStringSubmatch(pkg.Source)
			if matches == nil {
				return nil, fmt.Errorf("invalid Source field: %s", pkg.Source)
			}
			sourceName := matches[1]
			if sourceName == pkg.Name {
				continue
			}
			sourceVersion := pkg.Version
			if matches[2] != "" {
				sourceVersion = matches[2]
			}
			sourceQueries = append(sourceQueries, &DependencyQuery{Dependency{
				Pkg:          sourceName,
				Version:      sourceVersion,
				Relation:     VersionEqual,
				Architecture: ArchitectureSource,
			}})
		}
		for _, query := range sourceQueries {
			_ = result.Append(query.Query(l))
		}
	}

	if options.WithDependencies {
		added := result.Len()
		result.PrepareIndex()

		dependencySource := NewPackageList()
		if options.Source != nil {
			_ = dependencySource.Append(options.Source)
		}
		_ = dependencySource.Append(result)
		dependencySource.PrepareIndex()

		// while some new dependencies were discovered
		for added > 0 {
			added = 0

			// find missing dependencies
			missing, err := result.VerifyDependencies(options.DependencyOptions, options.Architectures, dependencySource, options.Progress)
			if err != nil {
				return nil, err
			}

			// try to satisfy dependencies
			for _, dep := range missing {
				if options.DependencyOptions&DepFollowAllVariants == 0 {
					// dependency might have already been satisfied
					// with packages already been added
					//
					// when follow-all-variants is enabled, we need to try to expand anyway,
					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
					if result.Search(dep, false, true) != nil {
						if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
							options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true))
						}
						continue
					}
				}

				searchResults := l.Search(dep, true, true)
				if len(searchResults) > 0 {
					for _, p := range searchResults {
						if result.Has(p) {
							continue
						}

						if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
							options.Progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
						}
						_ = result.Add(p)
						_ = dependencySource.Add(p)
						added++
						if options.DependencyOptions&DepFollowAllVariants == 0 {
							break
						}
					}
				} else {
					if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
						options.Progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
					}

				}
			}
		}
	}

	return result, nil
}