From a0610292a7c435ee507bac971f44453901203c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Delafond?= Date: Sat, 16 Nov 2024 17:08:02 +0100 Subject: [PATCH] debian: disable etcd --- debian/control | 1 - debian/patches/0002-debian-disable-etcd.patch | 589 ++++++++++++++++++ debian/patches/series | 1 + 3 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 debian/patches/0002-debian-disable-etcd.patch diff --git a/debian/control b/debian/control index 09d0b824..702a2900 100644 --- a/debian/control +++ b/debian/control @@ -76,7 +76,6 @@ Build-Depends: bash-completion, golang-gopkg-yaml.v2-dev, golang-go.uber-multierr-dev, golang-go.uber-zap-dev, - golang-etcd-server-dev (>= 3.5.15-7), git Standards-Version: 4.7.0 Homepage: https://www.aptly.info diff --git a/debian/patches/0002-debian-disable-etcd.patch b/debian/patches/0002-debian-disable-etcd.patch new file mode 100644 index 00000000..29c7206a --- /dev/null +++ b/debian/patches/0002-debian-disable-etcd.patch @@ -0,0 +1,589 @@ +From: =?utf-8?q?Andr=C3=A9_Roth?= +Date: Thu, 26 Sep 2024 17:37:11 +0200 +Subject: debian: disable etcd + +--- + context/context.go | 3 - + database/etcddb/batch.go | 53 ---------- + database/etcddb/database.go | 32 ------- + database/etcddb/database_test.go | 158 ------------------------------ + database/etcddb/storage.go | 202 --------------------------------------- + database/etcddb/transaction.go | 75 --------------- + 6 files changed, 523 deletions(-) + delete mode 100644 database/etcddb/batch.go + delete mode 100644 database/etcddb/database.go + delete mode 100644 database/etcddb/database_test.go + delete mode 100644 database/etcddb/storage.go + delete mode 100644 database/etcddb/transaction.go + +diff --git a/context/context.go b/context/context.go +index 7e468b0..85c9637 100644 +--- a/context/context.go ++++ b/context/context.go +@@ -20,7 +20,6 @@ + "github.com/aptly-dev/aptly/azure" + "github.com/aptly-dev/aptly/console" + "github.com/aptly-dev/aptly/database" +- "github.com/aptly-dev/aptly/database/etcddb" + "github.com/aptly-dev/aptly/database/goleveldb" + "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/files" +@@ -299,8 +298,6 @@ func (context *AptlyContext) _database() (database.Storage, error) { + } + dbPath := filepath.Join(context.config().RootDir, context.config().DatabaseBackend.DbPath) + context.database, err = goleveldb.NewDB(dbPath) +- case "etcd": +- context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL) + default: + context.database, err = goleveldb.NewDB(context.dbPath()) + } +diff --git a/database/etcddb/batch.go b/database/etcddb/batch.go +deleted file mode 100644 +index 24b83de..0000000 +--- a/database/etcddb/batch.go ++++ /dev/null +@@ -1,53 +0,0 @@ +-package etcddb +- +-import ( +- "github.com/aptly-dev/aptly/database" +- clientv3 "go.etcd.io/etcd/client/v3" +-) +- +-type EtcDBatch struct { +- s *EtcDStorage +- ops []clientv3.Op +-} +- +-type WriteOptions struct { +- NoWriteMerge bool +- Sync bool +-} +- +-func (b *EtcDBatch) Put(key []byte, value []byte) (err error) { +- b.ops = append(b.ops, clientv3.OpPut(string(key), string(value))) +- return +-} +- +-func (b *EtcDBatch) Delete(key []byte) (err error) { +- b.ops = append(b.ops, clientv3.OpDelete(string(key))) +- return +-} +- +-func (b *EtcDBatch) Write() (err error) { +- kv := clientv3.NewKV(b.s.db) +- +- batchSize := 128 +- for i := 0; i < len(b.ops); i += batchSize { +- txn := kv.Txn(Ctx) +- end := i + batchSize +- if end > len(b.ops) { +- end = len(b.ops) +- } +- +- batch := b.ops[i:end] +- txn.Then(batch...) +- _, err = txn.Commit() +- if err != nil { +- panic(err) +- } +- } +- +- return +-} +- +-// batch should implement database.Batch +-var ( +- _ database.Batch = &EtcDBatch{} +-) +diff --git a/database/etcddb/database.go b/database/etcddb/database.go +deleted file mode 100644 +index 37a222e..0000000 +--- a/database/etcddb/database.go ++++ /dev/null +@@ -1,32 +0,0 @@ +-package etcddb +- +-import ( +- "context" +- "time" +- +- "github.com/aptly-dev/aptly/database" +- clientv3 "go.etcd.io/etcd/client/v3" +-) +- +-var Ctx = context.TODO() +- +-func internalOpen(url string) (cli *clientv3.Client, err error) { +- cfg := clientv3.Config{ +- Endpoints: []string{url}, +- DialTimeout: 30 * time.Second, +- MaxCallSendMsgSize: 2147483647, // (2048 * 1024 * 1024) - 1 +- MaxCallRecvMsgSize: 2147483647, +- DialKeepAliveTimeout: 7200 * time.Second, +- } +- +- cli, err = clientv3.New(cfg) +- return +-} +- +-func NewDB(url string) (database.Storage, error) { +- cli, err := internalOpen(url) +- if err != nil { +- return nil, err +- } +- return &EtcDStorage{url, cli, ""}, nil +-} +diff --git a/database/etcddb/database_test.go b/database/etcddb/database_test.go +deleted file mode 100644 +index b31e559..0000000 +--- a/database/etcddb/database_test.go ++++ /dev/null +@@ -1,158 +0,0 @@ +-package etcddb_test +- +-import ( +- "testing" +- +- "github.com/aptly-dev/aptly/database" +- "github.com/aptly-dev/aptly/database/etcddb" +- . "gopkg.in/check.v1" +-) +- +-// Launch gocheck tests +-func Test(t *testing.T) { +- TestingT(t) +-} +- +-type EtcDDBSuite struct { +- url string +- db database.Storage +-} +- +-var _ = Suite(&EtcDDBSuite{}) +- +-func (s *EtcDDBSuite) SetUpTest(c *C) { +- var err error +- s.db, err = etcddb.NewDB("127.0.0.1:2379") +- c.Assert(err, IsNil) +-} +- +-func (s *EtcDDBSuite) TestSetUpTest(c *C) { +- var err error +- s.db, err = etcddb.NewDB("127.0.0.1:2379") +- c.Assert(err, IsNil) +-} +- +-func (s *EtcDDBSuite) TestGetPut(c *C) { +- var ( +- key = []byte("key") +- value = []byte("value") +- ) +- var err error +- +- err = s.db.Put(key, value) +- c.Assert(err, IsNil) +- +- result, err := s.db.Get(key) +- c.Assert(err, IsNil) +- c.Assert(result, DeepEquals, value) +-} +- +-func (s *EtcDDBSuite) TestDelete(c *C) { +- var ( +- key = []byte("key") +- value = []byte("value") +- ) +- +- err := s.db.Put(key, value) +- c.Assert(err, IsNil) +- +- _, err = s.db.Get(key) +- c.Assert(err, IsNil) +- +- err = s.db.Delete(key) +- c.Assert(err, IsNil) +- +-} +- +-func (s *EtcDDBSuite) TestByPrefix(c *C) { +- //c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{}) +- +- s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) +- s.db.Put([]byte{0x80, 0x03}, []byte{0x03}) +- s.db.Put([]byte{0x80, 0x02}, []byte{0x02}) +- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) +- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) +- +- s.db.Put([]byte{0x90, 0x01}, []byte{0x04}) +- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) +- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) +- +- s.db.Put([]byte{0x00, 0x01}, []byte{0x05}) +- c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) +- c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) +- +- keys := [][]byte{} +- values := [][]byte{} +- +- c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error { +- keys = append(keys, append([]byte(nil), k...)) +- values = append(values, append([]byte(nil), v...)) +- return nil +- }), IsNil) +- +- c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) +- c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) +- +- c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error { +- return database.ErrNotFound +- }), Equals, database.ErrNotFound) +- +- c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error { +- return database.ErrNotFound +- }), IsNil) +- +- c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{}) +- c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{}) +-} +- +-func (s *EtcDDBSuite) TestHasPrefix(c *C) { +- //c.Check(s.db.HasPrefix([]byte(nil)), Equals, false) +- //c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false) +- +- s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) +- +- c.Check(s.db.HasPrefix([]byte(nil)), Equals, true) +- c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true) +- c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false) +-} +- +-func (s *EtcDDBSuite) TestTransactionCommit(c *C) { +- var ( +- key = []byte("key") +- key2 = []byte("key2") +- value = []byte("value") +- value2 = []byte("value2") +- ) +- transaction, err := s.db.OpenTransaction() +- +- err = s.db.Put(key, value) +- c.Assert(err, IsNil) +- +- c.Assert(err, IsNil) +- transaction.Put(key2, value2) +- v, err := s.db.Get(key) +- c.Check(v, DeepEquals, value) +- err = transaction.Delete(key) +- c.Assert(err, IsNil) +- +- _, err = transaction.Get(key2) +- c.Assert(err, IsNil) +- +- v2, err := transaction.Get(key2) +- c.Check(err, IsNil) +- c.Check(v2, DeepEquals, value2) +- +- _, err = transaction.Get(key) +- c.Assert(err, IsNil) +- +- err = transaction.Commit() +- c.Check(err, IsNil) +- +- v2, err = transaction.Get(key2) +- c.Check(err, IsNil) +- c.Check(v2, DeepEquals, value2) +- +- _, err = transaction.Get(key) +- c.Assert(err, NotNil) +-} +- +diff --git a/database/etcddb/storage.go b/database/etcddb/storage.go +deleted file mode 100644 +index efc4cf6..0000000 +--- a/database/etcddb/storage.go ++++ /dev/null +@@ -1,202 +0,0 @@ +-package etcddb +- +-import ( +- "github.com/aptly-dev/aptly/database" +- "github.com/pborman/uuid" +- clientv3 "go.etcd.io/etcd/client/v3" +- +- "fmt" +-) +- +-type EtcDStorage struct { +- url string +- db *clientv3.Client +- tmpPrefix string // prefix for temporary DBs +-} +- +-// CreateTemporary creates new DB of the same type in temp dir +-func (s *EtcDStorage) CreateTemporary() (database.Storage, error) { +- tmp := uuid.NewRandom().String() +- return &EtcDStorage{ +- url: s.url, +- db: s.db, +- tmpPrefix: tmp, +- }, nil +-} +- +-func (s *EtcDStorage) applyPrefix(key []byte) []byte { +- if len(s.tmpPrefix) != 0 { +- return append([]byte(s.tmpPrefix+"/"), key...) +- } +- return key +-} +- +-// Get key value from etcd +-func (s *EtcDStorage) Get(key []byte) (value []byte, err error) { +- realKey := s.applyPrefix(key) +- getResp, err := s.db.Get(Ctx, string(realKey)) +- if err != nil { +- return +- } +- for _, kv := range getResp.Kvs { +- value = kv.Value +- break +- } +- if len(value) == 0 { +- err = database.ErrNotFound +- return +- } +- return +-} +- +-// Put saves key to etcd, if key has the same value in DB already, it is not saved +-func (s *EtcDStorage) Put(key []byte, value []byte) (err error) { +- realKey := s.applyPrefix(key) +- _, err = s.db.Put(Ctx, string(realKey), string(value)) +- if err != nil { +- return +- } +- return +-} +- +-// Delete removes key from etcd +-func (s *EtcDStorage) Delete(key []byte) (err error) { +- realKey := s.applyPrefix(key) +- _, err = s.db.Delete(Ctx, string(realKey)) +- if err != nil { +- return +- } +- return +-} +- +-// KeysByPrefix returns all keys that start with prefix +-func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte { +- realPrefix := s.applyPrefix(prefix) +- result := make([][]byte, 0, 20) +- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) +- if err != nil { +- return nil +- } +- for _, ev := range getResp.Kvs { +- key := ev.Key +- keyc := make([]byte, len(key)) +- copy(keyc, key) +- result = append(result, key) +- } +- return result +-} +- +-// FetchByPrefix returns all values with keys that start with prefix +-func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte { +- realPrefix := s.applyPrefix(prefix) +- result := make([][]byte, 0, 20) +- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) +- if err != nil { +- return nil +- } +- for _, kv := range getResp.Kvs { +- valc := make([]byte, len(kv.Value)) +- copy(valc, kv.Value) +- result = append(result, kv.Value) +- } +- +- return result +-} +- +-// HasPrefix checks whether it can find any key with given prefix and returns true if one exists +-func (s *EtcDStorage) HasPrefix(prefix []byte) bool { +- realPrefix := s.applyPrefix(prefix) +- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) +- if err != nil { +- return false +- } +- return getResp.Count > 0 +-} +- +-// ProcessByPrefix iterates through all entries where key starts with prefix and calls +-// StorageProcessor on key value pair +-func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error { +- realPrefix := s.applyPrefix(prefix) +- getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix()) +- if err != nil { +- return err +- } +- +- for _, kv := range getResp.Kvs { +- err := proc(kv.Key, kv.Value) +- if err != nil { +- return err +- } +- } +- return nil +-} +- +-// Close finishes etcd connect +-func (s *EtcDStorage) Close() error { +- // do not close temporary db +- if len(s.tmpPrefix) != 0 { +- return nil +- } +- if s.db == nil { +- return nil +- } +- err := s.db.Close() +- s.db = nil +- return err +-} +- +-// Reopen tries to open (re-open) the database +-func (s *EtcDStorage) Open() error { +- if s.db != nil { +- return nil +- } +- var err error +- s.db, err = internalOpen(s.url) +- return err +-} +- +-// CreateBatch creates a Batch object +-func (s *EtcDStorage) CreateBatch() database.Batch { +- if s.db == nil { +- return nil +- } +- return &EtcDBatch{ +- s: s, +- } +-} +- +-// OpenTransaction creates new transaction. +-func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) { +- tmpdb, err := s.CreateTemporary() +- if err != nil { +- return nil, err +- } +- return &transaction{s: s, tmpdb: tmpdb}, nil +-} +- +-// CompactDB does nothing for etcd +-func (s *EtcDStorage) CompactDB() error { +- return nil +-} +- +-// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys) +-func (s *EtcDStorage) Drop() error { +- if len(s.tmpPrefix) != 0 { +- getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix()) +- if err != nil { +- return nil +- } +- for _, kv := range getResp.Kvs { +- _, err = s.db.Delete(Ctx, string(kv.Key)) +- if err != nil { +- return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key) +- } +- } +- } +- return nil +-} +- +-// Check interface +-var ( +- _ database.Storage = &EtcDStorage{} +-) +diff --git a/database/etcddb/transaction.go b/database/etcddb/transaction.go +deleted file mode 100644 +index 01f54da..0000000 +--- a/database/etcddb/transaction.go ++++ /dev/null +@@ -1,75 +0,0 @@ +-package etcddb +- +-import ( +- "github.com/aptly-dev/aptly/database" +- clientv3 "go.etcd.io/etcd/client/v3" +-) +- +-type transaction struct { +- s *EtcDStorage +- tmpdb database.Storage +- ops []clientv3.Op +-} +- +-// Get implements database.Reader interface. +-func (t *transaction) Get(key []byte) (value []byte, err error) { +- value, err = t.tmpdb.Get(key) +- // if not found, search main db +- if err != nil { +- value, err = t.s.Get(key) +- } +- return +-} +- +-// Put implements database.Writer interface. +-func (t *transaction) Put(key, value []byte) (err error) { +- err = t.tmpdb.Put(key, value) +- if err != nil { +- return +- } +- t.ops = append(t.ops, clientv3.OpPut(string(key), string(value))) +- return +-} +- +-// Delete implements database.Writer interface. +-func (t *transaction) Delete(key []byte) (err error) { +- err = t.tmpdb.Delete(key) +- if err != nil { +- return +- } +- t.ops = append(t.ops, clientv3.OpDelete(string(key))) +- return +-} +- +-func (t *transaction) Commit() (err error) { +- kv := clientv3.NewKV(t.s.db) +- +- batchSize := 128 +- for i := 0; i < len(t.ops); i += batchSize { +- txn := kv.Txn(Ctx) +- end := i + batchSize +- if end > len(t.ops) { +- end = len(t.ops) +- } +- +- batch := t.ops[i:end] +- txn.Then(batch...) +- _, err = txn.Commit() +- if err != nil { +- panic(err) +- } +- } +- t.ops = []clientv3.Op{} +- +- return +-} +- +-// Discard is safe to call after Commit(), it would be no-op +-func (t *transaction) Discard() { +- t.ops = []clientv3.Op{} +- t.tmpdb.Drop() +- return +-} +- +-// transaction should implement database.Transaction +-var _ database.Transaction = &transaction{} diff --git a/debian/patches/series b/debian/patches/series index 2b7c0d33..58a506d2 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1 +1,2 @@ 0001-disable-swagger.patch +0002-debian-disable-etcd.patch