New upstream version 1.5.0+ds1

database/goleveldb/batch.go

@@ -0,0 +1,34 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+type batch struct {
+	db *leveldb.DB
+	b  *leveldb.Batch
+}
+
+func (b *batch) Put(key, value []byte) error {
+	b.b.Put(key, value)
+
+	return nil
+}
+
+func (b *batch) Delete(key []byte) error {
+	b.b.Delete(key)
+
+	return nil
+}
+
+func (b *batch) Write() error {
+	return b.db.Write(b.b, &opt.WriteOptions{})
+}
+
+// batch should implement database.Batch
+var (
+	_ database.Batch = &batch{}
+)

database/goleveldb/database.go

@@ -0,0 +1,58 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldbstorage "github.com/syndtr/goleveldb/leveldb/storage"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
+	o := &opt.Options{
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
+	}
+
+	if throttleCompaction {
+		o.CompactionL0Trigger = 32
+		o.WriteL0PauseTrigger = 96
+		o.WriteL0SlowdownTrigger = 64
+	}
+
+	return leveldb.OpenFile(path, o)
+}
+
+// NewDB creates new instance of DB, but doesn't open it (yet)
+func NewDB(path string) (database.Storage, error) {
+	return &storage{path: path}, nil
+}
+
+// NewOpenDB creates new instance of DB and opens it
+func NewOpenDB(path string) (database.Storage, error) {
+	db, err := NewDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, db.Open()
+}
+
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := leveldbstorage.OpenFile(path, false)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}

database/goleveldb/database_test.go

@@ -0,0 +1,230 @@
+package goleveldb_test
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type LevelDBSuite struct {
+	path string
+	db   database.Storage
+}
+
+var _ = Suite(&LevelDBSuite{})
+
+func (s *LevelDBSuite) SetUpTest(c *C) {
+	var err error
+
+	s.path = c.MkDir()
+	s.db, err = goleveldb.NewOpenDB(s.path)
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TearDownTest(c *C) {
+	err := s.db.Close()
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TestRecoverDB(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Check(err, IsNil)
+
+	err = s.db.Close()
+	c.Check(err, IsNil)
+
+	err = goleveldb.RecoverDB(s.path)
+	c.Check(err, IsNil)
+
+	s.db, err = goleveldb.NewOpenDB(s.path)
+	c.Check(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *LevelDBSuite) TestGetPut(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	_, err := s.db.Get(key)
+	c.Assert(err, ErrorMatches, "key not found")
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *LevelDBSuite) TestTemporaryDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	temp, err := s.db.CreateTemporary()
+	c.Assert(err, IsNil)
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, false)
+
+	err = temp.Put(key, value)
+	c.Assert(err, IsNil)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, true)
+
+	c.Assert(temp.Close(), IsNil)
+	c.Assert(temp.Drop(), IsNil)
+}
+
+func (s *LevelDBSuite) TestDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+
+	_, err = s.db.Get(key)
+	c.Assert(err, ErrorMatches, "key not found")
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TestByPrefix(c *C) {
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), Equals, database.ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), IsNil)
+
+	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+}
+
+func (s *LevelDBSuite) TestHasPrefix(c *C) {
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
+func (s *LevelDBSuite) TestBatch(c *C) {
+	var (
+		key    = []byte("key")
+		key2   = []byte("key2")
+		value  = []byte("value")
+		value2 = []byte("value2")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	batch := s.db.CreateBatch()
+	batch.Put(key2, value2)
+	batch.Delete(key)
+
+	v, err := s.db.Get(key)
+	c.Check(err, IsNil)
+	c.Check(v, DeepEquals, value)
+
+	_, err = s.db.Get(key2)
+	c.Check(err, ErrorMatches, "key not found")
+
+	err = batch.Write()
+	c.Check(err, IsNil)
+
+	v2, err := s.db.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = s.db.Get(key)
+	c.Check(err, ErrorMatches, "key not found")
+}
+
+func (s *LevelDBSuite) TestCompactDB(c *C) {
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+
+	c.Check(s.db.CompactDB(), IsNil)
+}
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.Open()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

database/goleveldb/leveldb.go

@@ -0,0 +1,2 @@
+// Package goleveldb implements database interface via goleveldb
+package goleveldb

database/goleveldb/storage.go

@@ -0,0 +1,180 @@
+package goleveldb
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+type storage struct {
+	path string
+	db   *leveldb.DB
+}
+
+// CreateTemporary creates new DB of the same type in temp dir
+func (s *storage) CreateTemporary() (database.Storage, error) {
+	tempdir, err := ioutil.TempDir("", "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := internalOpen(tempdir, true)
+	if err != nil {
+		return nil, err
+	}
+	return &storage{db: db, path: tempdir}, nil
+}
+
+// Get key value from database
+func (s *storage) Get(key []byte) ([]byte, error) {
+	value, err := s.db.Get(key, nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, database.ErrNotFound
+		}
+		return nil, err
+	}
+
+	return value, nil
+}
+
+// Put saves key to database, if key has the same value in DB already, it is not saved
+func (s *storage) Put(key []byte, value []byte) error {
+	old, err := s.db.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Equal(old, value) {
+			return nil
+		}
+	}
+	return s.db.Put(key, value, nil)
+}
+
+// Delete removes key from DB
+func (s *storage) Delete(key []byte) error {
+	return s.db.Delete(key, nil)
+}
+
+// KeysByPrefix returns all keys that start with prefix
+func (s *storage) KeysByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		key := iterator.Key()
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, keyc)
+	}
+
+	return result
+}
+
+// FetchByPrefix returns all values with keys that start with prefix
+func (s *storage) FetchByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		val := iterator.Value()
+		valc := make([]byte, len(val))
+		copy(valc, val)
+		result = append(result, valc)
+	}
+
+	return result
+}
+
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (s *storage) HasPrefix(prefix []byte) bool {
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (s *storage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		err := proc(iterator.Key(), iterator.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Close finishes DB work
+func (s *storage) Close() error {
+	if s.db == nil {
+		return nil
+	}
+	err := s.db.Close()
+	s.db = nil
+	return err
+}
+
+// Reopen tries to open (re-open) the database
+func (s *storage) Open() error {
+	if s.db != nil {
+		return nil
+	}
+
+	var err error
+	s.db, err = internalOpen(s.path, false)
+	return err
+}
+
+// CreateBatch creates a Batch object
+func (s *storage) CreateBatch() database.Batch {
+	return &batch{
+		db: s.db,
+		b:  &leveldb.Batch{},
+	}
+}
+
+// OpenTransaction creates new transaction.
+func (s *storage) OpenTransaction() (database.Transaction, error) {
+	t, err := s.db.OpenTransaction()
+	if err != nil {
+		return nil, err
+	}
+
+	return &transaction{t: t}, nil
+}
+
+// CompactDB compacts database by merging layers
+func (s *storage) CompactDB() error {
+	return s.db.CompactRange(util.Range{})
+}
+
+// Drop removes all the DB files (DANGEROUS!)
+func (s *storage) Drop() error {
+	if s.db != nil {
+		return errors.New("DB is still open")
+	}
+
+	return os.RemoveAll(s.path)
+}
+
+// Check interface
+var (
+	_ database.Storage = &storage{}
+)

database/goleveldb/transaction.go

@@ -0,0 +1,60 @@
+package goleveldb
+
+import (
+	"bytes"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type transaction struct {
+	t *leveldb.Transaction
+}
+
+// Get implements database.Reader interface.
+func (t *transaction) Get(key []byte) ([]byte, error) {
+	value, err := t.t.Get(key, nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, database.ErrNotFound
+		}
+		return nil, err
+	}
+
+	return value, nil
+}
+
+// Put implements database.Writer interface.
+func (t *transaction) Put(key, value []byte) error {
+	old, err := t.t.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Equal(old, value) {
+			return nil
+		}
+	}
+	return t.t.Put(key, value, nil)
+}
+
+// Delete implements database.Writer interface.
+func (t *transaction) Delete(key []byte) error {
+	return t.t.Delete(key, nil)
+}
+
+// Commit finalizes transaction and commits changes to the stable storage.
+func (t *transaction) Commit() error {
+	return t.t.Commit()
+}
+
+// Discard any transaction changes.
+//
+// Discard is safe to call after Commit(), it would be no-op
+func (t *transaction) Discard() {
+	t.t.Discard()
+}
+
+// transaction should implement database.Transaction
+var _ database.Transaction = &transaction{}