mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-02 04:50:49 +00:00
0f4bbc4752
See #761 aptly had a concept of loading small amount of info per each object into memory once collection is accessed for the first time. This might have simplified some operations, but it doesn't scale well with huge aptly databases. This is just intermediate step towards better memory management - list of objects is not loaded unless some method is called. `ForEach` method (mainly used in cleanup) is reimplemented to iterate over database without ever loading all the objects into memory. Memory was even worse with previous approach, as for each item usually `LoadComplete()` is called, which pulls even more data into memory and item stays in memory till the end of the iteration as it is referenced from `collection.list`. For the subsequent PR: reimplement `ByUUID()` and probably other methods to avoid loading all the items into memory, at least for all the collecitons except for published repos. When published repository is being loaded, it might pull source local repo which in turn would trigger loading for all the local repos which is not acceptable.
247 lines
5.7 KiB
Go
247 lines
5.7 KiB
Go
package deb
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
|
|
"github.com/aptly-dev/aptly/database"
|
|
"github.com/pborman/uuid"
|
|
"github.com/ugorji/go/codec"
|
|
)
|
|
|
|
// LocalRepo is a collection of packages created locally
|
|
type LocalRepo struct {
|
|
// Permanent internal ID
|
|
UUID string `json:"-"`
|
|
// User-assigned name
|
|
Name string
|
|
// Comment
|
|
Comment string
|
|
// DefaultDistribution
|
|
DefaultDistribution string `codec:",omitempty"`
|
|
// DefaultComponent
|
|
DefaultComponent string `codec:",omitempty"`
|
|
// Uploaders configuration
|
|
Uploaders *Uploaders `code:",omitempty" json:"-"`
|
|
// "Snapshot" of current list of packages
|
|
packageRefs *PackageRefList
|
|
}
|
|
|
|
// NewLocalRepo creates new instance of Debian local repository
|
|
func NewLocalRepo(name string, comment string) *LocalRepo {
|
|
return &LocalRepo{
|
|
UUID: uuid.New(),
|
|
Name: name,
|
|
Comment: comment,
|
|
}
|
|
}
|
|
|
|
// String interface
|
|
func (repo *LocalRepo) String() string {
|
|
if repo.Comment != "" {
|
|
return fmt.Sprintf("[%s]: %s", repo.Name, repo.Comment)
|
|
}
|
|
return fmt.Sprintf("[%s]", repo.Name)
|
|
}
|
|
|
|
// NumPackages return number of packages in local repo
|
|
func (repo *LocalRepo) NumPackages() int {
|
|
if repo.packageRefs == nil {
|
|
return 0
|
|
}
|
|
return repo.packageRefs.Len()
|
|
}
|
|
|
|
// RefList returns package list for repo
|
|
func (repo *LocalRepo) RefList() *PackageRefList {
|
|
return repo.packageRefs
|
|
}
|
|
|
|
// UpdateRefList changes package list for local repo
|
|
func (repo *LocalRepo) UpdateRefList(reflist *PackageRefList) {
|
|
repo.packageRefs = reflist
|
|
}
|
|
|
|
// Encode does msgpack encoding of LocalRepo
|
|
func (repo *LocalRepo) Encode() []byte {
|
|
var buf bytes.Buffer
|
|
|
|
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
|
|
encoder.Encode(repo)
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
// Decode decodes msgpack representation into LocalRepo
|
|
func (repo *LocalRepo) Decode(input []byte) error {
|
|
decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
|
|
return decoder.Decode(repo)
|
|
}
|
|
|
|
// Key is a unique id in DB
|
|
func (repo *LocalRepo) Key() []byte {
|
|
return []byte("L" + repo.UUID)
|
|
}
|
|
|
|
// RefKey is a unique id for package reference list
|
|
func (repo *LocalRepo) RefKey() []byte {
|
|
return []byte("E" + repo.UUID)
|
|
}
|
|
|
|
// LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
|
|
type LocalRepoCollection struct {
|
|
*sync.RWMutex
|
|
db database.Storage
|
|
list []*LocalRepo
|
|
}
|
|
|
|
// NewLocalRepoCollection loads LocalRepos from DB and makes up collection
|
|
func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
|
|
return &LocalRepoCollection{
|
|
RWMutex: &sync.RWMutex{},
|
|
db: db,
|
|
}
|
|
}
|
|
|
|
func (collection *LocalRepoCollection) loadList() {
|
|
if collection.list != nil {
|
|
return
|
|
}
|
|
|
|
blobs := collection.db.FetchByPrefix([]byte("L"))
|
|
collection.list = make([]*LocalRepo, 0, len(blobs))
|
|
|
|
for _, blob := range blobs {
|
|
r := &LocalRepo{}
|
|
if err := r.Decode(blob); err != nil {
|
|
log.Printf("Error decoding repo: %s\n", err)
|
|
} else {
|
|
collection.list = append(collection.list, r)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add appends new repo to collection and saves it
|
|
func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
|
|
collection.loadList()
|
|
|
|
for _, r := range collection.list {
|
|
if r.Name == repo.Name {
|
|
return fmt.Errorf("local repo with name %s already exists", repo.Name)
|
|
}
|
|
}
|
|
|
|
err := collection.Update(repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
collection.list = append(collection.list, repo)
|
|
return nil
|
|
}
|
|
|
|
// Update stores updated information about repo in DB
|
|
func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
|
|
err := collection.db.Put(repo.Key(), repo.Encode())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if repo.packageRefs != nil {
|
|
err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// LoadComplete loads additional information for local repo
|
|
func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
|
|
collection.loadList()
|
|
|
|
encoded, err := collection.db.Get(repo.RefKey())
|
|
if err == database.ErrNotFound {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
repo.packageRefs = &PackageRefList{}
|
|
return repo.packageRefs.Decode(encoded)
|
|
}
|
|
|
|
// ByName looks up repository by name
|
|
func (collection *LocalRepoCollection) ByName(name string) (*LocalRepo, error) {
|
|
collection.loadList()
|
|
|
|
for _, r := range collection.list {
|
|
if r.Name == name {
|
|
return r, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("local repo with name %s not found", name)
|
|
}
|
|
|
|
// ByUUID looks up repository by uuid
|
|
func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
|
|
collection.loadList()
|
|
|
|
for _, r := range collection.list {
|
|
if r.UUID == uuid {
|
|
return r, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
|
|
}
|
|
|
|
// ForEach runs method for each repository
|
|
func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
|
|
return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
|
|
r := &LocalRepo{}
|
|
if err := r.Decode(blob); err != nil {
|
|
log.Printf("Error decoding repo: %s\n", err)
|
|
return nil
|
|
}
|
|
|
|
return handler(r)
|
|
})
|
|
}
|
|
|
|
// Len returns number of remote repos
|
|
func (collection *LocalRepoCollection) Len() int {
|
|
collection.loadList()
|
|
|
|
return len(collection.list)
|
|
}
|
|
|
|
// Drop removes remote repo from collection
|
|
func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
|
|
collection.loadList()
|
|
|
|
repoPosition := -1
|
|
|
|
for i, r := range collection.list {
|
|
if r == repo {
|
|
repoPosition = i
|
|
break
|
|
}
|
|
}
|
|
|
|
if repoPosition == -1 {
|
|
panic("local repo not found!")
|
|
}
|
|
|
|
collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
|
|
nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
|
|
|
|
err := collection.db.Delete(repo.Key())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return collection.db.Delete(repo.RefKey())
|
|
}
|