vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-05-06 22:18:28 +00:00
Code Issues Projects Releases Wiki Activity

Major refactoring to lower memory consumption.

Browse Source 
Package has been split into multiple structure loaded on demand: files, extra, depends.

They're saved and loaded completely separately.
This commit is contained in:
Andrey Smirnov
2014-03-04 16:49:26 +04:00
parent 32e517b4f2
commit 3fe8a09928
15 changed files with 1301 additions and 1007 deletions
Download Patch File Download Diff File Expand all files Collapse all files
debian/package.go
Vendored
+124 -253
View File
@@ -1,54 +1,16 @@
package debian
import (
"bytes"
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/ugorji/go/codec"
"os"
"path/filepath"
"strconv"
"strings"
)
// PackageFile is a single file entry in package
type PackageFile struct {
// Filename is name of file for the package (without directory)
Filename string
// Hashes for the file
Checksums utils.ChecksumInfo
// Temporary field used while downloading, stored relative path on the mirror
downloadPath string
}
// Verify that package file is present and correct
func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return false, err
}
st, err := os.Stat(poolPath)
if err != nil {
return false, nil
}
// verify size
// TODO: verify checksum if configured
return st.Size() == f.Checksums.Size, nil
}
// DownloadURL return relative URL to package download location
func (f *PackageFile) DownloadURL() string {
return filepath.Join(f.downloadPath, f.Filename)
}
// Package is single instance of Debian package
type Package struct {
// Is this source package
IsSource bool
// Basic package properties
Name string
Version string
@@ -58,33 +20,18 @@ type Package struct {
SourceArchitecture string
// For binary package, name of source package
Source string
// Various dependencies
Provides []string
Depends []string
BuildDepends []string
BuildDependsInDep []string
PreDepends []string
Suggests []string
Recommends []string
// Files in package
Files []PackageFile
// Extra information from stanza
Extra Stanza
}
func parseDependencies(input Stanza, key string) []string {
value, ok := input[key]
if !ok {
return nil
}
delete(input, key)
result := strings.Split(value, ",")
for i := range result {
result[i] = strings.TrimSpace(result[i])
}
return result
// List of virtual packages this package provides
Provides []string
// Is this source package
IsSource bool
// Hash of files section
FilesHash uint64
// Offload fields
deps *PackageDependencies
extra *Stanza
files *PackageFiles
// Mother collection
collection *PackageCollection
}
// NewPackageFromControlFile creates Package from parsed Debian control file
@@ -94,7 +41,6 @@ func NewPackageFromControlFile(input Stanza) *Package {
Version: input["Version"],
Architecture: input["Architecture"],
Source: input["Source"],
Files: make([]PackageFile, 0, 1),
}
delete(input, "Package")
@@ -104,7 +50,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
result.Files = append(result.Files, PackageFile{
result.UpdateFiles(PackageFiles{PackageFile{
Filename: filepath.Base(input["Filename"]),
downloadPath: filepath.Dir(input["Filename"]),
Checksums: utils.ChecksumInfo{
@@ -113,7 +59,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
SHA1: strings.TrimSpace(input["SHA1"]),
SHA256: strings.TrimSpace(input["SHA256"]),
},
})
}})
delete(input, "Filename")
delete(input, "MD5sum")
@@ -121,13 +67,16 @@ func NewPackageFromControlFile(input Stanza) *Package {
delete(input, "SHA256")
delete(input, "Size")
result.Depends = parseDependencies(input, "Depends")
result.PreDepends = parseDependencies(input, "Pre-Depends")
result.Suggests = parseDependencies(input, "Suggests")
result.Recommends = parseDependencies(input, "Recommends")
depends := &PackageDependencies{}
depends.Depends = parseDependencies(input, "Depends")
depends.PreDepends = parseDependencies(input, "Pre-Depends")
depends.Suggests = parseDependencies(input, "Suggests")
depends.Recommends = parseDependencies(input, "Recommends")
result.deps = depends
result.Provides = parseDependencies(input, "Provides")
result.Extra = input
result.extra = &input
return result
}
@@ -146,6 +95,8 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
delete(input, "Version")
delete(input, "Architecture")
files := make(PackageFiles, 0, 3)
parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
for _, line := range strings.Split(input[field], "\n") {
line = strings.TrimSpace(line)
@@ -167,7 +118,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
found := false
pos := 0
for i, file := range result.Files {
for i, file := range files {
if file.Filename == filename {
found = true
pos = i
@@ -176,12 +127,12 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
}
if !found {
result.Files = append(result.Files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
pos = len(result.Files) - 1
files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
pos = len(files) - 1
}
result.Files[pos].Checksums.Size = size
setter(&result.Files[pos].Checksums, parts[0])
files[pos].Checksums.Size = size
setter(&files[pos].Checksums, parts[0])
}
delete(input, field)
@@ -202,46 +153,21 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
return nil, err
}
result.BuildDepends = parseDependencies(input, "Build-Depends")
result.BuildDependsInDep = parseDependencies(input, "Build-Depends-Indep")
result.UpdateFiles(files)
result.Extra = input
depends := &PackageDependencies{}
depends.BuildDepends = parseDependencies(input, "Build-Depends")
depends.BuildDependsInDep = parseDependencies(input, "Build-Depends-Indep")
result.deps = depends
result.extra = &input
return result, nil
}
// Key returns unique key identifying package
func (p *Package) Key() []byte {
return []byte("P" + p.Architecture + " " + p.Name + " " + p.Version)
}
// Internal buffer reused by all Package.Encode operations
var encodeBuf bytes.Buffer
// Encode does msgpack encoding of Package, []byte should be copied, as buffer would
// be used for the next call to Encode
func (p *Package) Encode() []byte {
encodeBuf.Reset()
encoder := codec.NewEncoder(&encodeBuf, &codec.MsgpackHandle{})
encoder.Encode(p)
return encodeBuf.Bytes()
}
// Decode decodes msgpack representation into Package
func (p *Package) Decode(input []byte) error {
decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
err := decoder.Decode(p)
if err != nil {
return err
}
for i := range p.Files {
p.Files[i].Filename = filepath.Base(p.Files[i].Filename)
}
return nil
func (p *Package) Key(prefix string) []byte {
return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
}
// String creates readable representation
@@ -291,21 +217,23 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
// GetDependencies compiles list of dependenices by flags from options
func (p *Package) GetDependencies(options int) (dependencies []string) {
deps := p.Deps()
dependencies = make([]string, 0, 30)
dependencies = append(dependencies, p.Depends...)
dependencies = append(dependencies, p.PreDepends...)
dependencies = append(dependencies, deps.Depends...)
dependencies = append(dependencies, deps.PreDepends...)
if options&DepFollowRecommends == DepFollowRecommends {
dependencies = append(dependencies, p.Recommends...)
dependencies = append(dependencies, deps.Recommends...)
}
if options&DepFollowSuggests == DepFollowSuggests {
dependencies = append(dependencies, p.Suggests...)
dependencies = append(dependencies, deps.Suggests...)
}
if options&DepFollowBuild == DepFollowBuild {
dependencies = append(dependencies, p.BuildDepends...)
dependencies = append(dependencies, p.BuildDependsInDep...)
dependencies = append(dependencies, deps.BuildDepends...)
dependencies = append(dependencies, deps.BuildDependsInDep...)
}
if options&DepFollowSource == DepFollowSource {
@@ -323,9 +251,53 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
return
}
// Extra returns Stanza of extra fields (it may load it from collection)
func (p *Package) Extra() Stanza {
if p.extra == nil {
if p.collection == nil {
panic("extra == nil && collection == nil")
}
p.extra = p.collection.loadExtra(p)
}
return *p.extra
}
// Deps returns parsed package dependencies (it may load it from collection)
func (p *Package) Deps() *PackageDependencies {
if p.deps == nil {
if p.collection == nil {
panic("deps == nil && collection == nil")
}
p.deps = p.collection.loadDependencies(p)
}
return p.deps
}
// Files returns parsed files records (it may load it from collection)
func (p *Package) Files() PackageFiles {
if p.files == nil {
if p.collection == nil {
panic("files == nil && collection == nil")
}
p.files = p.collection.loadFiles(p)
}
return *p.files
}
// UpdateFiles saves new state of files
func (p *Package) UpdateFiles(files PackageFiles) {
p.files = &files
p.FilesHash = files.Hash()
}
// Stanza creates original stanza from package
func (p *Package) Stanza() (result Stanza) {
result = p.Extra.Copy()
result = p.Extra().Copy()
result["Package"] = p.Name
result["Version"] = p.Version
@@ -339,7 +311,7 @@ func (p *Package) Stanza() (result Stanza) {
if p.IsSource {
md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0)
for _, f := range p.Files {
for _, f := range p.Files() {
if f.Checksums.MD5 != "" {
md5 = append(md5, fmt.Sprintf(" %s %d %s\n", f.Checksums.MD5, f.Checksums.Size, f.Filename))
}
@@ -355,39 +327,42 @@ func (p *Package) Stanza() (result Stanza) {
result["Checksums-Sha1"] = strings.Join(sha1, "")
result["Checksums-Sha256"] = strings.Join(sha256, "")
} else {
result["Filename"] = p.Files[0].DownloadURL()
if p.Files[0].Checksums.MD5 != "" {
result["MD5sum"] = p.Files[0].Checksums.MD5
f := p.Files()[0]
result["Filename"] = f.DownloadURL()
if f.Checksums.MD5 != "" {
result["MD5sum"] = f.Checksums.MD5
}
if p.Files[0].Checksums.SHA1 != "" {
result["SHA1"] = " " + p.Files[0].Checksums.SHA1
if f.Checksums.SHA1 != "" {
result["SHA1"] = " " + f.Checksums.SHA1
}
if p.Files[0].Checksums.SHA256 != "" {
result["SHA256"] = " " + p.Files[0].Checksums.SHA256
if f.Checksums.SHA256 != "" {
result["SHA256"] = " " + f.Checksums.SHA256
}
result["Size"] = fmt.Sprintf("%d", p.Files[0].Checksums.Size)
result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
}
if p.Depends != nil {
result["Depends"] = strings.Join(p.Depends, ", ")
deps := p.Deps()
if deps.Depends != nil {
result["Depends"] = strings.Join(deps.Depends, ", ")
}
if p.PreDepends != nil {
result["Pre-Depends"] = strings.Join(p.PreDepends, ", ")
if deps.PreDepends != nil {
result["Pre-Depends"] = strings.Join(deps.PreDepends, ", ")
}
if p.Suggests != nil {
result["Suggests"] = strings.Join(p.Suggests, ", ")
if deps.Suggests != nil {
result["Suggests"] = strings.Join(deps.Suggests, ", ")
}
if p.Recommends != nil {
result["Recommends"] = strings.Join(p.Recommends, ", ")
if deps.Recommends != nil {
result["Recommends"] = strings.Join(deps.Recommends, ", ")
}
if p.Provides != nil {
result["Provides"] = strings.Join(p.Provides, ", ")
}
if p.BuildDepends != nil {
result["Build-Depends"] = strings.Join(p.BuildDepends, ", ")
if deps.BuildDepends != nil {
result["Build-Depends"] = strings.Join(deps.BuildDepends, ", ")
}
if p.BuildDependsInDep != nil {
result["Build-Depends-Indep"] = strings.Join(p.BuildDependsInDep, ", ")
if deps.BuildDependsInDep != nil {
result["Build-Depends-Indep"] = strings.Join(deps.BuildDependsInDep, ", ")
}
return
@@ -395,33 +370,9 @@ func (p *Package) Stanza() (result Stanza) {
// Equals compares two packages to be identical
func (p *Package) Equals(p2 *Package) bool {
if len(p.Files) != len(p2.Files) {
return false
}
for _, f := range p.Files {
found := false
for _, f2 := range p2.Files {
if f2.Filename == f.Filename {
found = true
if f.Checksums != f2.Checksums {
return false
}
break
}
}
if !found {
return false
}
}
return p.Name == p2.Name && p.Version == p2.Version && p.SourceArchitecture == p2.SourceArchitecture &&
p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) &&
utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) &&
utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) &&
p.Source == p2.Source && utils.StrSlicesEqual(p.Provides, p2.Provides) && utils.StrSlicesEqual(p.BuildDepends, p2.BuildDepends) &&
utils.StrSlicesEqual(p.BuildDependsInDep, p2.BuildDependsInDep) && p.IsSource == p2.IsSource
p.Architecture == p2.Architecture && p.Source == p2.Source && p.IsSource == p2.IsSource &&
p.FilesHash == p2.FilesHash
}
// LinkFromPool links package file from pool to dist's pool location
@@ -431,7 +382,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
return err
}
for i, f := range p.Files {
for i, f := range p.Files() {
sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return err
@@ -444,9 +395,9 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
dir := filepath.Dir(relPath)
if p.IsSource {
p.Extra["Directory"] = dir
p.Extra()["Directory"] = dir
} else {
p.Files[i].downloadPath = dir
p.Files()[i].downloadPath = dir
}
}
@@ -487,7 +438,7 @@ type PackageDownloadTask struct {
func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
result = make([]PackageDownloadTask, 0, 1)
for _, f := range p.Files {
for _, f := range p.Files() {
poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return nil, err
@@ -510,7 +461,7 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD
func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
result = true
for _, f := range p.Files {
for _, f := range p.Files() {
result, err = f.Verify(packagePool)
if err != nil || !result {
return
@@ -523,9 +474,9 @@ func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err e
// FilepathList returns list of paths to files in package repository
func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) {
var err error
result := make([]string, len(p.Files))
result := make([]string, len(p.Files()))
for i, f := range p.Files {
for i, f := range p.Files() {
result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
if err != nil {
return nil, err
@@ -534,83 +485,3 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error)
return result, nil
}
// PackageCollection does management of packages in DB
type PackageCollection struct {
db database.Storage
}
// NewPackageCollection creates new PackageCollection and binds it to database
func NewPackageCollection(db database.Storage) *PackageCollection {
return &PackageCollection{
db: db,
}
}
// ByKey find package in DB by its key
func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
encoded, err := collection.db.Get(key)
if err != nil {
return nil, err
}
p := &Package{}
err = p.Decode(encoded)
if err != nil {
return nil, err
}
return p, nil
}
// Update adds or updates information about package in DB
func (collection *PackageCollection) Update(p *Package) error {
existing, err := collection.ByKey(p.Key())
if err == nil {
// check for conflict
if existing.Equals(p) {
// packages are the same, no need to update
return nil
}
// if .Files is different, consider to be conflict
if len(p.Files) != len(existing.Files) {
return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
}
for _, f := range p.Files {
found := false
for _, f2 := range existing.Files {
if f2.Filename == f.Filename {
found = true
if f.Checksums != f2.Checksums {
return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
}
break
}
}
if !found {
return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
}
}
// ok, .Files are the same, but some meta-data is different, proceed to saving
} else {
if err != database.ErrNotFound {
return err
}
// ok, package doesn't exist yet
}
return collection.db.Put(p.Key(), p.Encode())
}
// AllPackageRefs returns list of all packages as PackageRefList
func (collection *PackageCollection) AllPackageRefs() *PackageRefList {
return &PackageRefList{Refs: collection.db.KeysByPrefix([]byte("P"))}
}
// DeleteByKey deletes package in DB by key
func (collection *PackageCollection) DeleteByKey(key []byte) error {
return collection.db.Delete(key)
}