vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-08 05:50:47 +00:00
Code Issues Projects Releases Wiki Activity

Refactor database code to support standalone batches, transactions.

Browse Source 
This is spin-off of changes from #459.

Transactions are not being used yet, but batches are updated to work
with the new API.

`database/` package was refactored to split abstract interfaces and
implementation via goleveldb. This should make it easier to implement
new database types.
This commit is contained in:
Andrey Smirnov
2019-08-02 00:10:36 +03:00
committed by Andrey Smirnov
parent 26098f6c8d
commit 67e38955ae
23 changed files with 539 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deb/changes_test.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
@@ -37,7 +38,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
c.Assert(err, IsNil)
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.localRepoCollection = NewLocalRepoCollection(s.db)
s.packageCollection = NewPackageCollection(s.db)
deb/checksum_collection_test.go
+2 -1
View File
@@ -2,6 +2,7 @@ package deb
import (
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -22,7 +23,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
}
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewChecksumCollection(s.db)
}
deb/contents.go
+2 -2
View File
@@ -25,11 +25,11 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
}
// Push adds package to contents index, calculating package contents as required
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string, dbw database.Writer) error {
for _, path := range contents {
// for performance reasons we only write to leveldb during push.
// merging of qualified names per path will be done in WriteTo
err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
err := dbw.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
if err != nil {
return err
}
deb/local_test.go
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"errors"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -18,7 +19,7 @@ type LocalRepoSuite struct {
var _ = Suite(&LocalRepoSuite{})
func (s *LocalRepoSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.list = NewPackageList()
s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +84,7 @@ type LocalRepoCollectionSuite struct {
var _ = Suite(&LocalRepoCollectionSuite{})
func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewLocalRepoCollection(s.db)
s.list = NewPackageList()
deb/package_collection.go
+2 -2
View File
@@ -275,9 +275,9 @@ func (collection *PackageCollection) AllPackageRefs() *PackageRefList {
}
// DeleteByKey deletes package in DB by key
func (collection *PackageCollection) DeleteByKey(key []byte) error {
func (collection *PackageCollection) DeleteByKey(key []byte, dbw database.Writer) error {
for _, key := range [][]byte{key, append([]byte("xF"), key...), append([]byte("xD"), key...), append([]byte("xE"), key...)} {
err := collection.db.Delete(key)
err := dbw.Delete(key)
if err != nil {
return err
}
deb/package_collection_test.go
+3 -2
View File
@@ -2,6 +2,7 @@ package deb
import (
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -17,7 +18,7 @@ var _ = Suite(&PackageCollectionSuite{})
func (s *PackageCollectionSuite) SetUpTest(c *C) {
s.p = NewPackageFromControlFile(packageStanza.Copy())
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewPackageCollection(s.db)
}
@@ -113,7 +114,7 @@ func (s *PackageCollectionSuite) TestDeleteByKey(c *C) {
_, err = s.db.Get(s.p.Key("xF"))
c.Check(err, IsNil)
err = s.collection.DeleteByKey(s.p.Key(""))
err = s.collection.DeleteByKey(s.p.Key(""), s.db)
c.Check(err, IsNil)
_, err = s.collection.ByKey(s.p.Key(""))
deb/publish.go
+3 -3
View File
@@ -612,8 +612,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
// to push each path of the package into the database.
// We'll want this batched so as to avoid an excessive
// amount of write() calls.
tempDB.StartBatch()
defer tempDB.FinishBatch()
tempBatch := tempDB.CreateBatch()
defer tempBatch.Write()
for _, arch := range p.Architectures {
if pkg.MatchesArchitecture(arch) {
@@ -632,7 +632,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
contentIndexesMap[key] = contentIndex
}
contentIndex.Push(qualifiedName, contents)
contentIndex.Push(qualifiedName, contents, tempBatch)
}
}
deb/publish_test.go
+4 -3
View File
@@ -10,6 +10,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/ugorji/go/codec"
@@ -87,7 +88,7 @@ var _ = Suite(&PublishedRepoSuite{})
func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.SetUpPackages()
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.root = c.MkDir()
@@ -449,7 +450,7 @@ type PublishedRepoCollectionSuite struct {
var _ = Suite(&PublishedRepoCollectionSuite{})
func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
@@ -640,7 +641,7 @@ type PublishedRepoRemoveSuite struct {
var _ = Suite(&PublishedRepoRemoveSuite{})
func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
deb/reflist_test.go
+4 -4
View File
@@ -3,7 +3,7 @@ package deb
import (
"errors"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
}
func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
coll.Update(s.p1)
coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
}
func (s *PackageRefListSuite) TestDiff(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
}
func (s *PackageRefListSuite) TestMerge(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
packages := []*Package{
deb/remote_test.go
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/http"
"github.com/aptly-dev/aptly/pgp"
@@ -92,7 +93,7 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
s.progress = console.NewProgress()
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collectionFactory = NewCollectionFactory(s.db)
s.packagePool = files.NewPackagePool(c.MkDir(), false)
s.cs = files.NewMockChecksumStorage()
@@ -663,7 +664,7 @@ type RemoteRepoCollectionSuite struct {
var _ = Suite(&RemoteRepoCollectionSuite{})
func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewRemoteRepoCollection(s.db)
s.SetUpPackages()
}
deb/snapshot_bench_test.go
+4 -4
View File
@@ -5,7 +5,7 @@ import (
"os"
"testing"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
)
func BenchmarkSnapshotCollectionForEach(b *testing.B) {
@@ -14,7 +14,7 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
@@ -43,7 +43,7 @@ func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
@@ -74,7 +74,7 @@ func BenchmarkSnapshotCollectionByName(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
deb/snapshot_test.go
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"sort"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -113,7 +114,7 @@ type SnapshotCollectionSuite struct {
var _ = Suite(&SnapshotCollectionSuite{})
func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewSnapshotCollection(s.db)
s.SetUpPackages()