From 9a3922fe178ecdcdc73f3a59eac1b5598f128c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Delafond?= Date: Sat, 16 Nov 2024 20:13:59 +0100 Subject: [PATCH] Revert "debian: disable etcd" This reverts commit a0610292a7c435ee507bac971f44453901203c3d. --- debian/control | 1 + debian/patches/0002-debian-disable-etcd.patch | 589 ------------------ debian/patches/series | 1 - 3 files changed, 1 insertion(+), 590 deletions(-) delete mode 100644 debian/patches/0002-debian-disable-etcd.patch diff --git a/debian/control b/debian/control index 702a2900..09d0b824 100644 --- a/debian/control +++ b/debian/control @@ -76,6 +76,7 @@ Build-Depends: bash-completion, golang-gopkg-yaml.v2-dev, golang-go.uber-multierr-dev, golang-go.uber-zap-dev, + golang-etcd-server-dev (>= 3.5.15-7), git Standards-Version: 4.7.0 Homepage: https://www.aptly.info diff --git a/debian/patches/0002-debian-disable-etcd.patch b/debian/patches/0002-debian-disable-etcd.patch deleted file mode 100644 index 29c7206a..00000000 --- a/debian/patches/0002-debian-disable-etcd.patch +++ /dev/null @@ -1,589 +0,0 @@ -From: =?utf-8?q?Andr=C3=A9_Roth?= -Date: Thu, 26 Sep 2024 17:37:11 +0200 -Subject: debian: disable etcd - ---- - context/context.go | 3 - - database/etcddb/batch.go | 53 ---------- - database/etcddb/database.go | 32 ------- - database/etcddb/database_test.go | 158 ------------------------------ - database/etcddb/storage.go | 202 --------------------------------------- - database/etcddb/transaction.go | 75 --------------- - 6 files changed, 523 deletions(-) - delete mode 100644 database/etcddb/batch.go - delete mode 100644 database/etcddb/database.go - delete mode 100644 database/etcddb/database_test.go - delete mode 100644 database/etcddb/storage.go - delete mode 100644 database/etcddb/transaction.go - -diff --git a/context/context.go b/context/context.go -index 7e468b0..85c9637 100644 ---- a/context/context.go -+++ b/context/context.go -@@ -20,7 +20,6 @@ - "github.com/aptly-dev/aptly/azure" - "github.com/aptly-dev/aptly/console" - "github.com/aptly-dev/aptly/database" -- "github.com/aptly-dev/aptly/database/etcddb" - "github.com/aptly-dev/aptly/database/goleveldb" - "github.com/aptly-dev/aptly/deb" - "github.com/aptly-dev/aptly/files" -@@ -299,8 +298,6 @@ func (context *AptlyContext) _database() (database.Storage, error) { - } - dbPath := filepath.Join(context.config().RootDir, context.config().DatabaseBackend.DbPath) - context.database, err = goleveldb.NewDB(dbPath) -- case "etcd": -- context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL) - default: - context.database, err = goleveldb.NewDB(context.dbPath()) - } -diff --git a/database/etcddb/batch.go b/database/etcddb/batch.go -deleted file mode 100644 -index 24b83de..0000000 ---- a/database/etcddb/batch.go -+++ /dev/null -@@ -1,53 +0,0 @@ --package etcddb -- --import ( -- "github.com/aptly-dev/aptly/database" -- clientv3 "go.etcd.io/etcd/client/v3" --) -- --type EtcDBatch struct { -- s *EtcDStorage -- ops []clientv3.Op --} -- --type WriteOptions struct { -- NoWriteMerge bool -- Sync bool --} -- --func (b *EtcDBatch) Put(key []byte, value []byte) (err error) { -- b.ops = append(b.ops, clientv3.OpPut(string(key), string(value))) -- return --} -- --func (b *EtcDBatch) Delete(key []byte) (err error) { -- b.ops = append(b.ops, clientv3.OpDelete(string(key))) -- return --} -- --func (b *EtcDBatch) Write() (err error) { -- kv := clientv3.NewKV(b.s.db) -- -- batchSize := 128 -- for i := 0; i < len(b.ops); i += batchSize { -- txn := kv.Txn(Ctx) -- end := i + batchSize -- if end > len(b.ops) { -- end = len(b.ops) -- } -- -- batch := b.ops[i:end] -- txn.Then(batch...) -- _, err = txn.Commit() -- if err != nil { -- panic(err) -- } -- } -- -- return --} -- --// batch should implement database.Batch --var ( -- _ database.Batch = &EtcDBatch{} --) -diff --git a/database/etcddb/database.go b/database/etcddb/database.go -deleted file mode 100644 -index 37a222e..0000000 ---- a/database/etcddb/database.go -+++ /dev/null -@@ -1,32 +0,0 @@ --package etcddb -- --import ( -- "context" -- "time" -- -- "github.com/aptly-dev/aptly/database" -- clientv3 "go.etcd.io/etcd/client/v3" --) -- --var Ctx = context.TODO() -- --func internalOpen(url string) (cli *clientv3.Client, err error) { -- cfg := clientv3.Config{ -- Endpoints: []string{url}, -- DialTimeout: 30 * time.Second, -- MaxCallSendMsgSize: 2147483647, // (2048 * 1024 * 1024) - 1 -- MaxCallRecvMsgSize: 2147483647, -- DialKeepAliveTimeout: 7200 * time.Second, -- } -- -- cli, err = clientv3.New(cfg) -- return --} -- --func NewDB(url string) (database.Storage, error) { -- cli, err := internalOpen(url) -- if err != nil { -- return nil, err -- } -- return &EtcDStorage{url, cli, ""}, nil --} -diff --git a/database/etcddb/database_test.go b/database/etcddb/database_test.go -deleted file mode 100644 -index b31e559..0000000 ---- a/database/etcddb/database_test.go -+++ /dev/null -@@ -1,158 +0,0 @@ --package etcddb_test -- --import ( -- "testing" -- -- "github.com/aptly-dev/aptly/database" -- "github.com/aptly-dev/aptly/database/etcddb" -- . "gopkg.in/check.v1" --) -- --// Launch gocheck tests --func Test(t *testing.T) { -- TestingT(t) --} -- --type EtcDDBSuite struct { -- url string -- db database.Storage --} -- --var _ = Suite(&EtcDDBSuite{}) -- --func (s *EtcDDBSuite) SetUpTest(c *C) { -- var err error -- s.db, err = etcddb.NewDB("127.0.0.1:2379") -- c.Assert(err, IsNil) --} -- --func (s *EtcDDBSuite) TestSetUpTest(c *C) { -- var err error -- s.db, err = etcddb.NewDB("127.0.0.1:2379") -- c.Assert(err, IsNil) --} -- --func (s *EtcDDBSuite) TestGetPut(c *C) { -- var ( -- key = []byte("key") -- value = []byte("value") -- ) -- var err error -- -- err = s.db.Put(key, value) -- c.Assert(err, IsNil) -- -- result, err := s.db.Get(key) -- c.Assert(err, IsNil) -- c.Assert(result, DeepEquals, value) --} -- --func (s *EtcDDBSuite) TestDelete(c *C) { -- var ( -- key = []byte("key") -- value = []byte("value") -- ) -- -- err := s.db.Put(key, value) -- c.Assert(err, IsNil) -- -- _, err = s.db.Get(key) -- c.Assert(err, IsNil) -- -- err = s.db.Delete(key) -- c.Assert(err, IsNil) -- --} -- --func (s *EtcDDBSuite) TestByPrefix(c *C) { -- //c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{}) -- -- s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) -- s.db.Put([]byte{0x80, 0x03}, []byte{0x03}) -- s.db.Put([]byte{0x80, 0x02}, []byte{0x02}) -- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) -- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) -- -- s.db.Put([]byte{0x90, 0x01}, []byte{0x04}) -- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) -- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) -- -- s.db.Put([]byte{0x00, 0x01}, []byte{0x05}) -- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) -- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) -- -- keys := [][]byte{} -- values := [][]byte{} -- -- c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error { -- keys = append(keys, append([]byte(nil), k...)) -- values = append(values, append([]byte(nil), v...)) -- return nil -- }), IsNil) -- -- c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) -- c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) -- -- c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error { -- return database.ErrNotFound -- }), Equals, database.ErrNotFound) -- -- c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error { -- return database.ErrNotFound -- }), IsNil) -- -- c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{}) -- c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{}) --} -- --func (s *EtcDDBSuite) TestHasPrefix(c *C) { -- //c.Check(s.db.HasPrefix([]byte(nil)), Equals, false) -- //c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false) -- -- s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) -- -- c.Check(s.db.HasPrefix([]byte(nil)), Equals, true) -- c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true) -- c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false) --} -- --func (s *EtcDDBSuite) TestTransactionCommit(c *C) { -- var ( -- key = []byte("key") -- key2 = []byte("key2") -- value = []byte("value") -- value2 = []byte("value2") -- ) -- transaction, err := s.db.OpenTransaction() -- -- err = s.db.Put(key, value) -- c.Assert(err, IsNil) -- -- c.Assert(err, IsNil) -- transaction.Put(key2, value2) -- v, err := s.db.Get(key) -- c.Check(v, DeepEquals, value) -- err = transaction.Delete(key) -- c.Assert(err, IsNil) -- -- _, err = transaction.Get(key2) -- c.Assert(err, IsNil) -- -- v2, err := transaction.Get(key2) -- c.Check(err, IsNil) -- c.Check(v2, DeepEquals, value2) -- -- _, err = transaction.Get(key) -- c.Assert(err, IsNil) -- -- err = transaction.Commit() -- c.Check(err, IsNil) -- -- v2, err = transaction.Get(key2) -- c.Check(err, IsNil) -- c.Check(v2, DeepEquals, value2) -- -- _, err = transaction.Get(key) -- c.Assert(err, NotNil) --} -- -diff --git a/database/etcddb/storage.go b/database/etcddb/storage.go -deleted file mode 100644 -index efc4cf6..0000000 ---- a/database/etcddb/storage.go -+++ /dev/null -@@ -1,202 +0,0 @@ --package etcddb -- --import ( -- "github.com/aptly-dev/aptly/database" -- "github.com/pborman/uuid" -- clientv3 "go.etcd.io/etcd/client/v3" -- -- "fmt" --) -- --type EtcDStorage struct { -- url string -- db *clientv3.Client -- tmpPrefix string // prefix for temporary DBs --} -- --// CreateTemporary creates new DB of the same type in temp dir --func (s *EtcDStorage) CreateTemporary() (database.Storage, error) { -- tmp := uuid.NewRandom().String() -- return &EtcDStorage{ -- url: s.url, -- db: s.db, -- tmpPrefix: tmp, -- }, nil --} -- --func (s *EtcDStorage) applyPrefix(key []byte) []byte { -- if len(s.tmpPrefix) != 0 { -- return append([]byte(s.tmpPrefix+"/"), key...) -- } -- return key --} -- --// Get key value from etcd --func (s *EtcDStorage) Get(key []byte) (value []byte, err error) { -- realKey := s.applyPrefix(key) -- getResp, err := s.db.Get(Ctx, string(realKey)) -- if err != nil { -- return -- } -- for _, kv := range getResp.Kvs { -- value = kv.Value -- break -- } -- if len(value) == 0 { -- err = database.ErrNotFound -- return -- } -- return --} -- --// Put saves key to etcd, if key has the same value in DB already, it is not saved --func (s *EtcDStorage) Put(key []byte, value []byte) (err error) { -- realKey := s.applyPrefix(key) -- _, err = s.db.Put(Ctx, string(realKey), string(value)) -- if err != nil { -- return -- } -- return --} -- --// Delete removes key from etcd --func (s *EtcDStorage) Delete(key []byte) (err error) { -- realKey := s.applyPrefix(key) -- _, err = s.db.Delete(Ctx, string(realKey)) -- if err != nil { -- return -- } -- return --} -- --// KeysByPrefix returns all keys that start with prefix --func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte { -- realPrefix := s.applyPrefix(prefix) -- result := make([][]byte, 0, 20) -- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) -- if err != nil { -- return nil -- } -- for _, ev := range getResp.Kvs { -- key := ev.Key -- keyc := make([]byte, len(key)) -- copy(keyc, key) -- result = append(result, key) -- } -- return result --} -- --// FetchByPrefix returns all values with keys that start with prefix --func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte { -- realPrefix := s.applyPrefix(prefix) -- result := make([][]byte, 0, 20) -- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) -- if err != nil { -- return nil -- } -- for _, kv := range getResp.Kvs { -- valc := make([]byte, len(kv.Value)) -- copy(valc, kv.Value) -- result = append(result, kv.Value) -- } -- -- return result --} -- --// HasPrefix checks whether it can find any key with given prefix and returns true if one exists --func (s *EtcDStorage) HasPrefix(prefix []byte) bool { -- realPrefix := s.applyPrefix(prefix) -- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) -- if err != nil { -- return false -- } -- return getResp.Count > 0 --} -- --// ProcessByPrefix iterates through all entries where key starts with prefix and calls --// StorageProcessor on key value pair --func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error { -- realPrefix := s.applyPrefix(prefix) -- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) -- if err != nil { -- return err -- } -- -- for _, kv := range getResp.Kvs { -- err := proc(kv.Key, kv.Value) -- if err != nil { -- return err -- } -- } -- return nil --} -- --// Close finishes etcd connect --func (s *EtcDStorage) Close() error { -- // do not close temporary db -- if len(s.tmpPrefix) != 0 { -- return nil -- } -- if s.db == nil { -- return nil -- } -- err := s.db.Close() -- s.db = nil -- return err --} -- --// Reopen tries to open (re-open) the database --func (s *EtcDStorage) Open() error { -- if s.db != nil { -- return nil -- } -- var err error -- s.db, err = internalOpen(s.url) -- return err --} -- --// CreateBatch creates a Batch object --func (s *EtcDStorage) CreateBatch() database.Batch { -- if s.db == nil { -- return nil -- } -- return &EtcDBatch{ -- s: s, -- } --} -- --// OpenTransaction creates new transaction. --func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) { -- tmpdb, err := s.CreateTemporary() -- if err != nil { -- return nil, err -- } -- return &transaction{s: s, tmpdb: tmpdb}, nil --} -- --// CompactDB does nothing for etcd --func (s *EtcDStorage) CompactDB() error { -- return nil --} -- --// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys) --func (s *EtcDStorage) Drop() error { -- if len(s.tmpPrefix) != 0 { -- getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix()) -- if err != nil { -- return nil -- } -- for _, kv := range getResp.Kvs { -- _, err = s.db.Delete(Ctx, string(kv.Key)) -- if err != nil { -- return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key) -- } -- } -- } -- return nil --} -- --// Check interface --var ( -- _ database.Storage = &EtcDStorage{} --) -diff --git a/database/etcddb/transaction.go b/database/etcddb/transaction.go -deleted file mode 100644 -index 01f54da..0000000 ---- a/database/etcddb/transaction.go -+++ /dev/null -@@ -1,75 +0,0 @@ --package etcddb -- --import ( -- "github.com/aptly-dev/aptly/database" -- clientv3 "go.etcd.io/etcd/client/v3" --) -- --type transaction struct { -- s *EtcDStorage -- tmpdb database.Storage -- ops []clientv3.Op --} -- --// Get implements database.Reader interface. --func (t *transaction) Get(key []byte) (value []byte, err error) { -- value, err = t.tmpdb.Get(key) -- // if not found, search main db -- if err != nil { -- value, err = t.s.Get(key) -- } -- return --} -- --// Put implements database.Writer interface. --func (t *transaction) Put(key, value []byte) (err error) { -- err = t.tmpdb.Put(key, value) -- if err != nil { -- return -- } -- t.ops = append(t.ops, clientv3.OpPut(string(key), string(value))) -- return --} -- --// Delete implements database.Writer interface. --func (t *transaction) Delete(key []byte) (err error) { -- err = t.tmpdb.Delete(key) -- if err != nil { -- return -- } -- t.ops = append(t.ops, clientv3.OpDelete(string(key))) -- return --} -- --func (t *transaction) Commit() (err error) { -- kv := clientv3.NewKV(t.s.db) -- -- batchSize := 128 -- for i := 0; i < len(t.ops); i += batchSize { -- txn := kv.Txn(Ctx) -- end := i + batchSize -- if end > len(t.ops) { -- end = len(t.ops) -- } -- -- batch := t.ops[i:end] -- txn.Then(batch...) -- _, err = txn.Commit() -- if err != nil { -- panic(err) -- } -- } -- t.ops = []clientv3.Op{} -- -- return --} -- --// Discard is safe to call after Commit(), it would be no-op --func (t *transaction) Discard() { -- t.ops = []clientv3.Op{} -- t.tmpdb.Drop() -- return --} -- --// transaction should implement database.Transaction --var _ database.Transaction = &transaction{} diff --git a/debian/patches/series b/debian/patches/series index 58a506d2..2b7c0d33 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,2 +1 @@ 0001-disable-swagger.patch -0002-debian-disable-etcd.patch