First round of support for localRepos as source for publishing. Also more intelligent algo to get publishing defaults. #10 #12

This commit is contained in:
Andrey Smirnov
2014-03-19 16:43:42 +04:00
parent c2fc2f9988
commit 32717e92ba
11 changed files with 348 additions and 124 deletions

195
debian/publish.go vendored
View File

@@ -26,14 +26,44 @@ type PublishedRepo struct {
Component string
// Architectures is a list of all architectures published
Architectures []string
// Snapshot as a source of publishing
SnapshotUUID string
// SourceKind is "local"/"repo"
SourceKind string
// SourceUUID is UUID of either snapshot or local repo
SourceUUID string `codec:"SnapshotUUID"`
snapshot *Snapshot
snapshot *Snapshot
localRepo *LocalRepo
}
// NewPublishedRepo creates new published repository
func NewPublishedRepo(prefix string, distribution string, component string, architectures []string, snapshot *Snapshot) (*PublishedRepo, error) {
//
// prefix specifies publishing prefix
// distribution, component and architectures are user-defined properties
// source could either be *Snapshot or *LocalRepo
func NewPublishedRepo(prefix string, distribution string, component string, architectures []string, source interface{}, collectionFactory *CollectionFactory) (*PublishedRepo, error) {
var ok bool
result := &PublishedRepo{
UUID: uuid.New(),
Architectures: architectures,
}
// figure out source
result.snapshot, ok = source.(*Snapshot)
if ok {
result.SourceKind = "snapshot"
result.SourceUUID = result.snapshot.UUID
} else {
result.localRepo, ok = source.(*LocalRepo)
if ok {
result.SourceKind = "local"
result.SourceUUID = result.localRepo.UUID
} else {
panic("unknown source kind")
}
}
// clean & verify prefix
prefix = filepath.Clean(prefix)
if strings.HasPrefix(prefix, "/") {
prefix = prefix[1:]
@@ -43,32 +73,104 @@ func NewPublishedRepo(prefix string, distribution string, component string, arch
}
prefix = filepath.Clean(prefix)
for _, component := range strings.Split(prefix, "/") {
if component == ".." || component == "dists" || component == "pool" {
for _, part := range strings.Split(prefix, "/") {
if part == ".." || part == "dists" || part == "pool" {
return nil, fmt.Errorf("invalid prefix %s", prefix)
}
}
return &PublishedRepo{
UUID: uuid.New(),
Prefix: prefix,
Distribution: distribution,
Component: component,
Architectures: architectures,
SnapshotUUID: snapshot.UUID,
snapshot: snapshot,
}, nil
result.Prefix = prefix
// guessing distribution & component
if component == "" || distribution == "" {
var (
head interface{}
current = []interface{}{source}
rootComponents = []string{}
rootDistributions = []string{}
)
// walk up the tree from current source up to roots (local or remote repos)
// and collect information about distribution and components
for len(current) > 0 {
head, current = current[0], current[1:]
if snapshot, ok := head.(*Snapshot); ok {
for _, uuid := range snapshot.SourceIDs {
if snapshot.SourceKind == "repo" {
remoteRepo, err := collectionFactory.RemoteRepoCollection().ByUUID(uuid)
if err != nil {
continue
}
current = append(current, remoteRepo)
} else if snapshot.SourceKind == "local" {
localRepo, err := collectionFactory.LocalRepoCollection().ByUUID(uuid)
if err != nil {
continue
}
current = append(current, localRepo)
} else if snapshot.SourceKind == "snapshot" {
snap, err := collectionFactory.SnapshotCollection().ByUUID(uuid)
if err != nil {
continue
}
current = append(current, snap)
}
}
} else if localRepo, ok := head.(*LocalRepo); ok {
if localRepo.DefaultDistribution != "" {
rootDistributions = append(rootDistributions, localRepo.DefaultDistribution)
}
if localRepo.DefaultComponent != "" {
rootComponents = append(rootComponents, localRepo.DefaultComponent)
}
} else if remoteRepo, ok := head.(*RemoteRepo); ok {
if remoteRepo.Distribution != "" {
rootDistributions = append(rootDistributions, remoteRepo.Distribution)
}
rootComponents = append(rootComponents, remoteRepo.Components...)
} else {
panic("unknown type")
}
}
if distribution == "" {
sort.Strings(rootDistributions)
if len(rootDistributions) > 0 && rootDistributions[0] == rootDistributions[len(rootDistributions)-1] {
distribution = rootDistributions[0]
} else {
return nil, fmt.Errorf("unable to guess distribution name, please specify explicitly")
}
}
if component == "" {
sort.Strings(rootComponents)
if len(rootComponents) > 0 && rootComponents[0] == rootComponents[len(rootComponents)-1] {
component = rootComponents[0]
} else {
component = "main"
}
}
}
result.Distribution, result.Component = distribution, component
return result, nil
}
// String returns human-readable represenation of PublishedRepo
func (p *PublishedRepo) String() string {
var archs string
var source string
if len(p.Architectures) > 0 {
archs = fmt.Sprintf(" [%s]", strings.Join(p.Architectures, ", "))
if p.snapshot != nil {
source = p.snapshot.String()
} else if p.localRepo != nil {
source = p.localRepo.String()
} else {
panic("no snapshot/localRepo")
}
return fmt.Sprintf("%s/%s (%s)%s publishes %s", p.Prefix, p.Distribution, p.Component, archs, p.snapshot.String())
return fmt.Sprintf("%s/%s (%s) [%s] publishes %s", p.Prefix, p.Distribution, p.Component, strings.Join(p.Architectures, ", "), source)
}
// Key returns unique key identifying PublishedRepo
@@ -89,11 +191,21 @@ func (p *PublishedRepo) Encode() []byte {
// Decode decodes msgpack representation into PublishedRepo
func (p *PublishedRepo) Decode(input []byte) error {
decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
return decoder.Decode(p)
err := decoder.Decode(p)
if err != nil {
return err
}
// old PublishedRepo were publishing only snapshots
if p.SourceKind == "" {
p.SourceKind = "snapshot"
}
return nil
}
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer, progress aptly.Progress) error {
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress) error {
err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
if err != nil {
return err
@@ -108,8 +220,18 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage
progress.Printf("Loading packages...\n")
}
var refList *PackageRefList
if p.snapshot != nil {
refList = p.snapshot.RefList()
} else if p.localRepo != nil {
refList = p.localRepo.RefList()
} else {
panic("no source")
}
// Load all packages
list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection, progress)
list, err := NewPackageListFromRefList(refList, collectionFactory.PackageCollection(), progress)
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
@@ -373,14 +495,18 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
}
// LoadComplete loads additional information for remote repo
func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, snapshotCollection *SnapshotCollection) error {
snapshot, err := snapshotCollection.ByUUID(repo.SnapshotUUID)
if err != nil {
return err
func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) error {
var err error
if repo.SourceKind == "snapshot" {
repo.snapshot, err = collectionFactory.SnapshotCollection().ByUUID(repo.SourceUUID)
} else if repo.SourceKind == "local" {
repo.localRepo, err = collectionFactory.LocalRepoCollection().ByUUID(repo.SourceUUID)
} else {
panic("unknown SourceKind")
}
repo.snapshot = snapshot
return nil
return err
}
// ByPrefixDistribution looks up repository by prefix & distribution
@@ -407,7 +533,18 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
result := make([]*PublishedRepo, 0)
for _, r := range collection.list {
if r.SnapshotUUID == snapshot.UUID {
if r.SourceKind == "snapshot" && r.SourceUUID == snapshot.UUID {
result = append(result, r)
}
}
return result
}
// ByLocalRepo looks up repository by local repo source
func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
result := make([]*PublishedRepo, 0)
for _, r := range collection.list {
if r.SourceKind == "local" && r.SourceUUID == repo.UUID {
result = append(result, r)
}
}

141
debian/publish_test.go vendored
View File

@@ -45,12 +45,14 @@ func (n *NullSigner) ClearSign(source string, destination string) error {
type PublishedRepoSuite struct {
PackageListMixinSuite
repo *PublishedRepo
repo, repo2 *PublishedRepo
root string
publishedStorage aptly.PublishedStorage
packagePool aptly.PackagePool
localRepo *LocalRepo
snapshot *Snapshot
db database.Storage
factory *CollectionFactory
packageCollection *PackageCollection
}
@@ -60,6 +62,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.SetUpPackages()
s.db, _ = database.OpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.root = c.MkDir()
s.publishedStorage = files.NewPublishedStorage(s.root)
@@ -67,16 +70,24 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
repo.packageRefs = s.reflist
s.factory.RemoteRepoCollection().Add(repo)
s.localRepo = NewLocalRepo("local1", "comment1")
s.localRepo.packageRefs = s.reflist
s.factory.LocalRepoCollection().Add(s.localRepo)
s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
s.factory.SnapshotCollection().Add(s.snapshot)
s.repo, _ = NewPublishedRepo("ppa", "squeeze", "main", nil, s.snapshot)
s.packageCollection = NewPackageCollection(s.db)
s.packageCollection = s.factory.PackageCollection()
s.packageCollection.Update(s.p1)
s.packageCollection.Update(s.p2)
s.packageCollection.Update(s.p3)
s.repo, _ = NewPublishedRepo("ppa", "squeeze", "main", nil, s.snapshot, s.factory)
s.repo2, _ = NewPublishedRepo("ppa", "maverick", "main", nil, s.localRepo, s.factory)
poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
f, err := os.Create(poolPath)
@@ -144,7 +155,7 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
errorExpected: "invalid prefix .*",
},
} {
repo, err := NewPublishedRepo(t.prefix, "squeeze", "main", nil, s.snapshot)
repo, err := NewPublishedRepo(t.prefix, "squeeze", "main", nil, s.snapshot, s.factory)
if t.errorExpected != "" {
c.Check(err, ErrorMatches, t.errorExpected)
} else {
@@ -153,8 +164,42 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
}
}
func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
repo, err := NewPublishedRepo("ppa", "", "", nil, s.snapshot, s.factory)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Component, Equals, "main")
repo, err = NewPublishedRepo("ppa", "wheezy", "", nil, s.snapshot, s.factory)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "wheezy")
c.Check(repo.Component, Equals, "main")
repo, err = NewPublishedRepo("ppa", "", "non-free", nil, s.snapshot, s.factory)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Component, Equals, "non-free")
repo, err = NewPublishedRepo("ppa", "squeeze", "", nil, s.localRepo, s.factory)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Component, Equals, "main")
repo, err = NewPublishedRepo("ppa", "", "main", nil, s.localRepo, s.factory)
c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
s.localRepo.DefaultDistribution = "precise"
s.localRepo.DefaultComponent = "contrib"
s.factory.LocalRepoCollection().Update(s.localRepo)
repo, err = NewPublishedRepo("ppa", "", "", nil, s.localRepo, s.factory)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "precise")
c.Check(repo.Component, Equals, "contrib")
}
func (s *PublishedRepoSuite) TestPublish(c *C) {
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, &NullSigner{}, nil)
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, &NullSigner{}, nil)
c.Assert(err, IsNil)
c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
@@ -191,19 +236,28 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
}
func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, nil, nil)
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
c.Assert(err, IsNil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
}
func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
err := s.repo2.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
c.Assert(err, IsNil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
}
func (s *PublishedRepoSuite) TestString(c *C) {
c.Check(s.repo.String(), Equals,
"ppa/squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
repo, _ := NewPublishedRepo("", "squeeze", "main", nil, s.snapshot)
"ppa/squeeze (main) [] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
c.Check(s.repo2.String(), Equals,
"ppa/maverick (main) [] publishes [local1]: comment1")
repo, _ := NewPublishedRepo("", "squeeze", "main", []string{"s390"}, s.snapshot, s.factory)
c.Check(repo.String(), Equals,
"./squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
repo, _ = NewPublishedRepo("", "squeeze", "main", []string{"i386", "amd64"}, s.snapshot)
"./squeeze (main) [s390] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
repo, _ = NewPublishedRepo("", "squeeze", "main", []string{"i386", "amd64"}, s.snapshot, s.factory)
c.Check(repo.String(), Equals,
"./squeeze (main) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
}
@@ -220,23 +274,34 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
s.repo.snapshot = nil
c.Assert(err, IsNil)
c.Assert(repo, DeepEquals, s.repo)
encoded2 := s.repo2.Encode()
repo2 := &PublishedRepo{}
err = repo2.Decode(encoded2)
s.repo2.localRepo = nil
c.Assert(err, IsNil)
c.Assert(repo2, DeepEquals, s.repo2)
}
type PublishedRepoCollectionSuite struct {
PackageListMixinSuite
db database.Storage
snapshotCollection *SnapshotCollection
collection *PublishedRepoCollection
snap1, snap2 *Snapshot
repo1, repo2, repo3 *PublishedRepo
db database.Storage
factory *CollectionFactory
snapshotCollection *SnapshotCollection
collection *PublishedRepoCollection
snap1, snap2 *Snapshot
localRepo *LocalRepo
repo1, repo2, repo3, repo4 *PublishedRepo
}
var _ = Suite(&PublishedRepoCollectionSuite{})
func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.OpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = NewSnapshotCollection(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
@@ -244,11 +309,15 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.snapshotCollection.Add(s.snap1)
s.snapshotCollection.Add(s.snap2)
s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap2)
s.repo3, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap2)
s.localRepo = NewLocalRepo("local1", "comment1")
s.factory.LocalRepoCollection().Add(s.localRepo)
s.collection = NewPublishedRepoCollection(s.db)
s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1, s.factory)
s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap2, s.factory)
s.repo3, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap2, s.factory)
s.repo4, _ = NewPublishedRepo("ppa", "precise", "main", []string{}, s.localRepo, s.factory)
s.collection = s.factory.PublishedRepoCollection()
}
func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
@@ -265,11 +334,12 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
c.Assert(s.collection.Add(s.repo2), IsNil)
c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
c.Assert(s.collection.Add(s.repo4), IsNil)
r, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
c.Assert(err, IsNil)
err = s.collection.LoadComplete(r, s.snapshotCollection)
err = s.collection.LoadComplete(r, s.factory)
c.Assert(err, IsNil)
c.Assert(r.String(), Equals, s.repo1.String())
@@ -277,7 +347,7 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
r, err = collection.ByPrefixDistribution("ppa", "anaconda")
c.Assert(err, IsNil)
err = s.collection.LoadComplete(r, s.snapshotCollection)
err = s.collection.LoadComplete(r, s.factory)
c.Assert(err, IsNil)
c.Assert(r.String(), Equals, s.repo1.String())
}
@@ -291,20 +361,27 @@ func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
r, err = s.collection.ByUUID(s.repo1.UUID)
c.Assert(err, IsNil)
err = s.collection.LoadComplete(r, s.snapshotCollection)
err = s.collection.LoadComplete(r, s.factory)
c.Assert(err, IsNil)
c.Assert(r.String(), Equals, s.repo1.String())
}
func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
c.Assert(s.collection.Update(s.repo1), IsNil)
c.Assert(s.collection.Update(s.repo4), IsNil)
collection := NewPublishedRepoCollection(s.db)
r, err := collection.ByPrefixDistribution("ppa", "anaconda")
c.Assert(err, IsNil)
c.Assert(r.snapshot, IsNil)
c.Assert(s.collection.LoadComplete(r, s.snapshotCollection), IsNil)
c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
c.Assert(r.snapshot.UUID, Equals, s.repo1.snapshot.UUID)
r, err = collection.ByPrefixDistribution("ppa", "precise")
c.Assert(err, IsNil)
c.Assert(r.localRepo, IsNil)
c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
c.Assert(r.localRepo.UUID, Equals, s.repo4.localRepo.UUID)
}
func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
@@ -339,6 +416,7 @@ func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
type PublishedRepoRemoveSuite struct {
PackageListMixinSuite
db database.Storage
factory *CollectionFactory
snapshotCollection *SnapshotCollection
collection *PublishedRepoCollection
root string
@@ -351,19 +429,20 @@ var _ = Suite(&PublishedRepoRemoveSuite{})
func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.db, _ = database.OpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = NewSnapshotCollection(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
s.snapshotCollection.Add(s.snap1)
s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap1)
s.repo3, _ = NewPublishedRepo("ppa", "meduza", "main", []string{}, s.snap1)
s.repo4, _ = NewPublishedRepo("ppa", "osminog", "contrib", []string{}, s.snap1)
s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1, s.factory)
s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap1, s.factory)
s.repo3, _ = NewPublishedRepo("ppa", "meduza", "main", []string{}, s.snap1, s.factory)
s.repo4, _ = NewPublishedRepo("ppa", "osminog", "contrib", []string{}, s.snap1, s.factory)
s.collection = NewPublishedRepoCollection(s.db)
s.collection = s.factory.PublishedRepoCollection()
s.collection.Add(s.repo1)
s.collection.Add(s.repo2)
s.collection.Add(s.repo3)