vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-10 06:14:22 +00:00
Code Issues Projects Releases Wiki Activity

Refactor database code to support standalone batches, transactions.

Browse Source 
This is spin-off of changes from #459.

Transactions are not being used yet, but batches are updated to work
with the new API.

`database/` package was refactored to split abstract interfaces and
implementation via goleveldb. This should make it easier to implement
new database types.
This commit is contained in:
Andrey Smirnov
2019-08-02 00:10:36 +03:00
committed by Andrey Smirnov
parent 26098f6c8d
commit 67e38955ae
23 changed files with 539 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/db_cleanup.go
+3 -3
View File
@@ -183,15 +183,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
}
if !dryRun {
db.StartBatch()
batch := db.CreateBatch()
err = toDelete.ForEach(func(ref []byte) error {
return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
return context.CollectionFactory().PackageCollection().DeleteByKey(ref, batch)
})
if err != nil {
return err
}
err = db.FinishBatch()
err = batch.Write()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
}
cmd/db_recover.go
+3 -2
View File
@@ -1,8 +1,9 @@
package cmd
import (
"github.com/aptly-dev/aptly/database"
"github.com/smira/commander"
"github.com/aptly-dev/aptly/database/goleveldb"
)
// aptly db recover
@@ -15,7 +16,7 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
}
context.Progress().Printf("Recovering database...\n")
err = database.RecoverDB(context.DBPath())
err = goleveldb.RecoverDB(context.DBPath())
return err
}
context/context.go
+2 -1
View File
@@ -17,6 +17,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/http"
@@ -252,7 +253,7 @@ func (context *AptlyContext) _database() (database.Storage, error) {
if context.database == nil {
var err error
context.database, err = database.NewDB(context.dbPath())
context.database, err = goleveldb.NewDB(context.dbPath())
if err != nil {
return nil, fmt.Errorf("can't instantiate database: %s", err)
}
database/database.go
+69
View File
@@ -0,0 +1,69 @@
// Package database provides KV database for meta-information
package database
import "errors"
// Errors for Storage
var (
ErrNotFound = errors.New("key not found")
)
// StorageProcessor is a function to process one single storage entry
type StorageProcessor func(key []byte, value []byte) error
// Reader provides KV read calls
type Reader interface {
Get(key []byte) ([]byte, error)
}
// PrefixReader provides prefixed operations
type PrefixReader interface {
HasPrefix(prefix []byte) bool
ProcessByPrefix(prefix []byte, proc StorageProcessor) error
KeysByPrefix(prefix []byte) [][]byte
FetchByPrefix(prefix []byte) [][]byte
}
// Writer provides KV update/delete calls
type Writer interface {
Put(key []byte, value []byte) error
Delete(key []byte) error
}
// Storage is an interface to KV storage
type Storage interface {
Reader
Writer
PrefixReader
CreateBatch() Batch
OpenTransaction() (Transaction, error)
CreateTemporary() (Storage, error)
Open() error
Close() error
CompactDB() error
Drop() error
}
// Batch provides a way to pack many writes.
type Batch interface {
Writer
// Write closes batch and send accumulated writes to the database
Write() error
}
// Transaction provides isolated atomic way to perform updates.
//
// Transactions might be expensive.
// Transaction should always finish with either Discard() or Commit()
type Transaction interface {
Reader
Writer
Commit() error
Discard()
}
database/goleveldb/batch.go
+34
View File
@@ -0,0 +1,34 @@
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/aptly-dev/aptly/database"
)
type batch struct {
db *leveldb.DB
b *leveldb.Batch
}
func (b *batch) Put(key, value []byte) error {
b.b.Put(key, value)
return nil
}
func (b *batch) Delete(key []byte) error {
b.b.Delete(key)
return nil
}
func (b *batch) Write() error {
return b.db.Write(b.b, &opt.WriteOptions{})
}
// batch should implement database.Batch
var (
_ database.Batch = &batch{}
)
database/goleveldb/database.go
+58
View File
@@ -0,0 +1,58 @@
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
leveldbstorage "github.com/syndtr/goleveldb/leveldb/storage"
"github.com/aptly-dev/aptly/database"
)
func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
o := &opt.Options{
Filter: filter.NewBloomFilter(10),
OpenFilesCacheCapacity: 256,
}
if throttleCompaction {
o.CompactionL0Trigger = 32
o.WriteL0PauseTrigger = 96
o.WriteL0SlowdownTrigger = 64
}
return leveldb.OpenFile(path, o)
}
// NewDB creates new instance of DB, but doesn't open it (yet)
func NewDB(path string) (database.Storage, error) {
return &storage{path: path}, nil
}
// NewOpenDB creates new instance of DB and opens it
func NewOpenDB(path string) (database.Storage, error) {
db, err := NewDB(path)
if err != nil {
return nil, err
}
return db, db.Open()
}
// RecoverDB recovers LevelDB database from corruption
func RecoverDB(path string) error {
stor, err := leveldbstorage.OpenFile(path, false)
if err != nil {
return err
}
db, err := leveldb.Recover(stor, nil)
if err != nil {
return err
}
db.Close()
stor.Close()
return nil
}
database/leveldb_test.go → database/goleveldb/database_test.go
+94 -15
View File
@@ -1,9 +1,12 @@
package database
package goleveldb_test
import (
"testing"
. "gopkg.in/check.v1"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
)
// Launch gocheck tests
@@ -13,7 +16,7 @@ func Test(t *testing.T) {
type LevelDBSuite struct {
path string
db Storage
db database.Storage
}
var _ = Suite(&LevelDBSuite{})
@@ -22,7 +25,7 @@ func (s *LevelDBSuite) SetUpTest(c *C) {
var err error
s.path = c.MkDir()
s.db, err = NewOpenDB(s.path)
s.db, err = goleveldb.NewOpenDB(s.path)
c.Assert(err, IsNil)
}
@@ -43,10 +46,10 @@ func (s *LevelDBSuite) TestRecoverDB(c *C) {
err = s.db.Close()
c.Check(err, IsNil)
err = RecoverDB(s.path)
err = goleveldb.RecoverDB(s.path)
c.Check(err, IsNil)
s.db, err = NewOpenDB(s.path)
s.db, err = goleveldb.NewOpenDB(s.path)
c.Check(err, IsNil)
result, err := s.db.Get(key)
@@ -143,11 +146,11 @@ func (s *LevelDBSuite) TestByPrefix(c *C) {
c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
return ErrNotFound
}), Equals, ErrNotFound)
return database.ErrNotFound
}), Equals, database.ErrNotFound)
c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
return ErrNotFound
return database.ErrNotFound
}), IsNil)
c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
@@ -176,9 +179,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
err := s.db.Put(key, value)
c.Assert(err, IsNil)
s.db.StartBatch()
s.db.Put(key2, value2)
s.db.Delete(key)
batch := s.db.CreateBatch()
batch.Put(key2, value2)
batch.Delete(key)
v, err := s.db.Get(key)
c.Check(err, IsNil)
@@ -187,7 +190,7 @@ func (s *LevelDBSuite) TestBatch(c *C) {
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
err = s.db.FinishBatch()
err = batch.Write()
c.Check(err, IsNil)
v2, err := s.db.Get(key2)
@@ -196,11 +199,87 @@ func (s *LevelDBSuite) TestBatch(c *C) {
_, err = s.db.Get(key)
c.Check(err, ErrorMatches, "key not found")
}
c.Check(func() { s.db.FinishBatch() }, Panics, "no batch")
func (s *LevelDBSuite) TestTransactionCommit(c *C) {
var (
key = []byte("key")
key2 = []byte("key2")
value = []byte("value")
value2 = []byte("value2")
)
s.db.StartBatch()
c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
err := s.db.Put(key, value)
c.Assert(err, IsNil)
transaction, err := s.db.OpenTransaction()
c.Assert(err, IsNil)
transaction.Put(key2, value2)
transaction.Delete(key)
v, err := s.db.Get(key)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value)
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
v2, err := transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Check(err, ErrorMatches, "key not found")
err = transaction.Commit()
c.Check(err, IsNil)
v2, err = s.db.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = s.db.Get(key)
c.Check(err, ErrorMatches, "key not found")
}
func (s *LevelDBSuite) TestTransactionDiscard(c *C) {
var (
key = []byte("key")
key2 = []byte("key2")
value = []byte("value")
value2 = []byte("value2")
)
err := s.db.Put(key, value)
c.Assert(err, IsNil)
transaction, err := s.db.OpenTransaction()
c.Assert(err, IsNil)
transaction.Put(key2, value2)
transaction.Delete(key)
v, err := s.db.Get(key)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value)
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
v2, err := transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Check(err, ErrorMatches, "key not found")
transaction.Discard()
v, err = s.db.Get(key)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value)
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
}
func (s *LevelDBSuite) TestCompactDB(c *C) {
database/goleveldb/leveldb.go
+2
View File
@@ -0,0 +1,2 @@
// Package goleveldb implements database interface via goleveldb
package goleveldb
database/goleveldb/storage.go
+180
View File
@@ -0,0 +1,180 @@
package goleveldb
import (
"bytes"
"errors"
"io/ioutil"
"os"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
"github.com/aptly-dev/aptly/database"
)
type storage struct {
path string
db *leveldb.DB
}
// CreateTemporary creates new DB of the same type in temp dir
func (s *storage) CreateTemporary() (database.Storage, error) {
tempdir, err := ioutil.TempDir("", "aptly")
if err != nil {
return nil, err
}
db, err := internalOpen(tempdir, true)
if err != nil {
return nil, err
}
return &storage{db: db, path: tempdir}, nil
}
// Get key value from database
func (s *storage) Get(key []byte) ([]byte, error) {
value, err := s.db.Get(key, nil)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, database.ErrNotFound
}
return nil, err
}
return value, nil
}
// Put saves key to database, if key has the same value in DB already, it is not saved
func (s *storage) Put(key []byte, value []byte) error {
old, err := s.db.Get(key, nil)
if err != nil {
if err != leveldb.ErrNotFound {
return err
}
} else {
if bytes.Equal(old, value) {
return nil
}
}
return s.db.Put(key, value, nil)
}
// Delete removes key from DB
func (s *storage) Delete(key []byte) error {
return s.db.Delete(key, nil)
}
// KeysByPrefix returns all keys that start with prefix
func (s *storage) KeysByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0, 20)
iterator := s.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
key := iterator.Key()
keyc := make([]byte, len(key))
copy(keyc, key)
result = append(result, keyc)
}
return result
}
// FetchByPrefix returns all values with keys that start with prefix
func (s *storage) FetchByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0, 20)
iterator := s.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
val := iterator.Value()
valc := make([]byte, len(val))
copy(valc, val)
result = append(result, valc)
}
return result
}
// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
func (s *storage) HasPrefix(prefix []byte) bool {
iterator := s.db.NewIterator(nil, nil)
defer iterator.Release()
return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
}
// ProcessByPrefix iterates through all entries where key starts with prefix and calls
// StorageProcessor on key value pair
func (s *storage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
iterator := s.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
err := proc(iterator.Key(), iterator.Value())
if err != nil {
return err
}
}
return nil
}
// Close finishes DB work
func (s *storage) Close() error {
if s.db == nil {
return nil
}
err := s.db.Close()
s.db = nil
return err
}
// Reopen tries to open (re-open) the database
func (s *storage) Open() error {
if s.db != nil {
return nil
}
var err error
s.db, err = internalOpen(s.path, false)
return err
}
// CreateBatch creates a Batch object
func (s *storage) CreateBatch() database.Batch {
return &batch{
db: s.db,
b: &leveldb.Batch{},
}
}
// OpenTransaction creates new transaction.
func (s *storage) OpenTransaction() (database.Transaction, error) {
t, err := s.db.OpenTransaction()
if err != nil {
return nil, err
}
return &transaction{t: t}, nil
}
// CompactDB compacts database by merging layers
func (s *storage) CompactDB() error {
return s.db.CompactRange(util.Range{})
}
// Drop removes all the DB files (DANGEROUS!)
func (s *storage) Drop() error {
if s.db != nil {
return errors.New("DB is still open")
}
return os.RemoveAll(s.path)
}
// Check interface
var (
_ database.Storage = &storage{}
)
database/goleveldb/transaction.go
+60
View File
@@ -0,0 +1,60 @@
package goleveldb
import (
"bytes"
"github.com/aptly-dev/aptly/database"
"github.com/syndtr/goleveldb/leveldb"
)
type transaction struct {
t *leveldb.Transaction
}
// Get implements database.Reader interface.
func (t *transaction) Get(key []byte) ([]byte, error) {
value, err := t.t.Get(key, nil)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, database.ErrNotFound
}
return nil, err
}
return value, nil
}
// Put implements database.Writer interface.
func (t *transaction) Put(key, value []byte) error {
old, err := t.t.Get(key, nil)
if err != nil {
if err != leveldb.ErrNotFound {
return err
}
} else {
if bytes.Equal(old, value) {
return nil
}
}
return t.t.Put(key, value, nil)
}
// Delete implements database.Writer interface.
func (t *transaction) Delete(key []byte) error {
return t.t.Delete(key, nil)
}
// Commit finalizes transaction and commits changes to the stable storage.
func (t *transaction) Commit() error {
return t.t.Commit()
}
// Discard any transaction changes.
//
// Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() {
t.t.Discard()
}
// transaction should implement database.Transaction
var _ database.Transaction = &transaction{}
database/leveldb.go
-267
View File
@@ -1,267 +0,0 @@
// Package database provides KV database for meta-information
package database
import (
"bytes"
"errors"
"io/ioutil"
"os"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/util"
)
// Errors for Storage
var (
ErrNotFound = errors.New("key not found")
)
// StorageProcessor is a function to process one single storage entry
type StorageProcessor func(key []byte, value []byte) error
// Storage is an interface to KV storage
type Storage interface {
CreateTemporary() (Storage, error)
Get(key []byte) ([]byte, error)
Put(key []byte, value []byte) error
Delete(key []byte) error
HasPrefix(prefix []byte) bool
ProcessByPrefix(prefix []byte, proc StorageProcessor) error
KeysByPrefix(prefix []byte) [][]byte
FetchByPrefix(prefix []byte) [][]byte
Open() error
Close() error
StartBatch()
FinishBatch() error
CompactDB() error
Drop() error
}
type levelDB struct {
path string
db *leveldb.DB
batch *leveldb.Batch
}
// Check interface
var (
_ Storage = &levelDB{}
)
func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
o := &opt.Options{
Filter: filter.NewBloomFilter(10),
OpenFilesCacheCapacity: 256,
}
if throttleCompaction {
o.CompactionL0Trigger = 32
o.WriteL0PauseTrigger = 96
o.WriteL0SlowdownTrigger = 64
}
return leveldb.OpenFile(path, o)
}
// NewDB creates new instance of DB, but doesn't open it (yet)
func NewDB(path string) (Storage, error) {
return &levelDB{path: path}, nil
}
// NewOpenDB creates new instance of DB and opens it
func NewOpenDB(path string) (Storage, error) {
db, err := NewDB(path)
if err != nil {
return nil, err
}
return db, db.Open()
}
// RecoverDB recovers LevelDB database from corruption
func RecoverDB(path string) error {
stor, err := storage.OpenFile(path, false)
if err != nil {
return err
}
db, err := leveldb.Recover(stor, nil)
if err != nil {
return err
}
db.Close()
stor.Close()
return nil
}
// CreateTemporary creates new DB of the same type in temp dir
func (l *levelDB) CreateTemporary() (Storage, error) {
tempdir, err := ioutil.TempDir("", "aptly")
if err != nil {
return nil, err
}
db, err := internalOpen(tempdir, true)
if err != nil {
return nil, err
}
return &levelDB{db: db, path: tempdir}, nil
}
// Get key value from database
func (l *levelDB) Get(key []byte) ([]byte, error) {
value, err := l.db.Get(key, nil)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}
return value, nil
}
// Put saves key to database, if key has the same value in DB already, it is not saved
func (l *levelDB) Put(key []byte, value []byte) error {
if l.batch != nil {
l.batch.Put(key, value)
return nil
}
old, err := l.db.Get(key, nil)
if err != nil {
if err != leveldb.ErrNotFound {
return err
}
} else {
if bytes.Equal(old, value) {
return nil
}
}
return l.db.Put(key, value, nil)
}
// Delete removes key from DB
func (l *levelDB) Delete(key []byte) error {
if l.batch != nil {
l.batch.Delete(key)
return nil
}
return l.db.Delete(key, nil)
}
// KeysByPrefix returns all keys that start with prefix
func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0, 20)
iterator := l.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
key := iterator.Key()
keyc := make([]byte, len(key))
copy(keyc, key)
result = append(result, keyc)
}
return result
}
// FetchByPrefix returns all values with keys that start with prefix
func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0, 20)
iterator := l.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
val := iterator.Value()
valc := make([]byte, len(val))
copy(valc, val)
result = append(result, valc)
}
return result
}
// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
func (l *levelDB) HasPrefix(prefix []byte) bool {
iterator := l.db.NewIterator(nil, nil)
defer iterator.Release()
return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
}
// ProcessByPrefix iterates through all entries where key starts with prefix and calls
// StorageProcessor on key value pair
func (l *levelDB) ProcessByPrefix(prefix []byte, proc StorageProcessor) error {
iterator := l.db.NewIterator(nil, nil)
defer iterator.Release()
for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
err := proc(iterator.Key(), iterator.Value())
if err != nil {
return err
}
}
return nil
}
// Close finishes DB work
func (l *levelDB) Close() error {
if l.db == nil {
return nil
}
err := l.db.Close()
l.db = nil
return err
}
// Reopen tries to open (re-open) the database
func (l *levelDB) Open() error {
if l.db != nil {
return nil
}
var err error
l.db, err = internalOpen(l.path, false)
return err
}
// StartBatch starts batch processing of keys
//
// All subsequent Get, Put and Delete would work on batch
func (l *levelDB) StartBatch() {
if l.batch != nil {
panic("batch already started")
}
l.batch = new(leveldb.Batch)
}
// FinishBatch finalizes the batch, saving operations
func (l *levelDB) FinishBatch() error {
if l.batch == nil {
panic("no batch")
}
err := l.db.Write(l.batch, nil)
l.batch = nil
return err
}
// CompactDB compacts database by merging layers
func (l *levelDB) CompactDB() error {
return l.db.CompactRange(util.Range{})
}
// Drop removes all the DB files (DANGEROUS!)
func (l *levelDB) Drop() error {
if l.db != nil {
return errors.New("DB is still open")
}
return os.RemoveAll(l.path)
}
deb/changes_test.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
@@ -37,7 +38,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
c.Assert(err, IsNil)
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.localRepoCollection = NewLocalRepoCollection(s.db)
s.packageCollection = NewPackageCollection(s.db)
deb/checksum_collection_test.go
+2 -1
View File
@@ -2,6 +2,7 @@ package deb
import (
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -22,7 +23,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
}
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewChecksumCollection(s.db)
}
deb/contents.go
+2 -2
View File
@@ -25,11 +25,11 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
}
// Push adds package to contents index, calculating package contents as required
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string, dbw database.Writer) error {
for _, path := range contents {
// for performance reasons we only write to leveldb during push.
// merging of qualified names per path will be done in WriteTo
err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
err := dbw.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
if err != nil {
return err
}
deb/local_test.go
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"errors"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -18,7 +19,7 @@ type LocalRepoSuite struct {
var _ = Suite(&LocalRepoSuite{})
func (s *LocalRepoSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.list = NewPackageList()
s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +84,7 @@ type LocalRepoCollectionSuite struct {
var _ = Suite(&LocalRepoCollectionSuite{})
func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewLocalRepoCollection(s.db)
s.list = NewPackageList()
deb/package_collection.go
+2 -2
View File
@@ -275,9 +275,9 @@ func (collection *PackageCollection) AllPackageRefs() *PackageRefList {
}
// DeleteByKey deletes package in DB by key
func (collection *PackageCollection) DeleteByKey(key []byte) error {
func (collection *PackageCollection) DeleteByKey(key []byte, dbw database.Writer) error {
for _, key := range [][]byte{key, append([]byte("xF"), key...), append([]byte("xD"), key...), append([]byte("xE"), key...)} {
err := collection.db.Delete(key)
err := dbw.Delete(key)
if err != nil {
return err
}
deb/package_collection_test.go
+3 -2
View File
@@ -2,6 +2,7 @@ package deb
import (
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -17,7 +18,7 @@ var _ = Suite(&PackageCollectionSuite{})
func (s *PackageCollectionSuite) SetUpTest(c *C) {
s.p = NewPackageFromControlFile(packageStanza.Copy())
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewPackageCollection(s.db)
}
@@ -113,7 +114,7 @@ func (s *PackageCollectionSuite) TestDeleteByKey(c *C) {
_, err = s.db.Get(s.p.Key("xF"))
c.Check(err, IsNil)
err = s.collection.DeleteByKey(s.p.Key(""))
err = s.collection.DeleteByKey(s.p.Key(""), s.db)
c.Check(err, IsNil)
_, err = s.collection.ByKey(s.p.Key(""))
deb/publish.go
+3 -3
View File
@@ -612,8 +612,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
// to push each path of the package into the database.
// We'll want this batched so as to avoid an excessive
// amount of write() calls.
tempDB.StartBatch()
defer tempDB.FinishBatch()
tempBatch := tempDB.CreateBatch()
defer tempBatch.Write()
for _, arch := range p.Architectures {
if pkg.MatchesArchitecture(arch) {
@@ -632,7 +632,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
contentIndexesMap[key] = contentIndex
}
contentIndex.Push(qualifiedName, contents)
contentIndex.Push(qualifiedName, contents, tempBatch)
}
}
deb/publish_test.go
+4 -3
View File
@@ -10,6 +10,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/ugorji/go/codec"
@@ -87,7 +88,7 @@ var _ = Suite(&PublishedRepoSuite{})
func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.SetUpPackages()
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.root = c.MkDir()
@@ -449,7 +450,7 @@ type PublishedRepoCollectionSuite struct {
var _ = Suite(&PublishedRepoCollectionSuite{})
func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
@@ -640,7 +641,7 @@ type PublishedRepoRemoveSuite struct {
var _ = Suite(&PublishedRepoRemoveSuite{})
func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
deb/reflist_test.go
+4 -4
View File
@@ -3,7 +3,7 @@ package deb
import (
"errors"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
}
func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
coll.Update(s.p1)
coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
}
func (s *PackageRefListSuite) TestDiff(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
}
func (s *PackageRefListSuite) TestMerge(c *C) {
db, _ := database.NewOpenDB(c.MkDir())
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
packages := []*Package{
deb/remote_test.go
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/http"
"github.com/aptly-dev/aptly/pgp"
@@ -92,7 +93,7 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
s.progress = console.NewProgress()
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collectionFactory = NewCollectionFactory(s.db)
s.packagePool = files.NewPackagePool(c.MkDir(), false)
s.cs = files.NewMockChecksumStorage()
@@ -663,7 +664,7 @@ type RemoteRepoCollectionSuite struct {
var _ = Suite(&RemoteRepoCollectionSuite{})
func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewRemoteRepoCollection(s.db)
s.SetUpPackages()
}
deb/snapshot_bench_test.go
+4 -4
View File
@@ -5,7 +5,7 @@ import (
"os"
"testing"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
)
func BenchmarkSnapshotCollectionForEach(b *testing.B) {
@@ -14,7 +14,7 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
@@ -43,7 +43,7 @@ func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
@@ -74,7 +74,7 @@ func BenchmarkSnapshotCollectionByName(b *testing.B) {
tmpDir := os.TempDir()
defer os.RemoveAll(tmpDir)
db, _ := database.NewOpenDB(tmpDir)
db, _ := goleveldb.NewOpenDB(tmpDir)
defer db.Close()
collection := NewSnapshotCollection(db)
deb/snapshot_test.go
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"sort"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
. "gopkg.in/check.v1"
)
@@ -113,7 +114,7 @@ type SnapshotCollectionSuite struct {
var _ = Suite(&SnapshotCollectionSuite{})
func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
s.db, _ = database.NewOpenDB(c.MkDir())
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.collection = NewSnapshotCollection(s.db)
s.SetUpPackages()