Merge pull request #1406 from aptly-dev/release/1.6.0

Release/1.6.0

.github/workflows/ci.yml

@@ -30,12 +30,12 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: "Install Packages"
+      - name: "Install packages"
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
 
-      - name: "Checkout Repository"
+      - name: "Checkout repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
@@ -45,7 +45,7 @@ jobs:
         run: |
           make flake8
 
-      - name: "Read Go Version"
+      - name: "Read go version from go.mod"
         run: |
           gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
           echo "Go Version: $gover"
@@ -91,12 +91,12 @@ jobs:
           sudo mkdir -p /srv ; sudo chown runner /srv
           COVERAGE_DIR=${{ runner.temp }} make system-test
 
-      - name: "Merge Code Coverage"
+      - name: "Merge code coverage"
         run: |
           go install github.com/wadey/gocovmerge@latest
           ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
 
-      - name: "Upload Code Coverage"
+      - name: "Upload code coverage"
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -148,13 +148,13 @@ jobs:
                           gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
-      - name: "Checkout Repository"
+      - name: "Checkout repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: "Read Go Version"
+      - name: "Read go version from go.mod"
         run: |
           gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
           echo "Go Version: $gover"
@@ -221,13 +221,13 @@ jobs:
           - goos: darwin
             goarch: arm
     steps:
-      - name: "Checkout Repository"
+      - name: "Checkout repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: "Read Go Version"
+      - name: "Read go version from go.mod"
         run: |
           echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
         id: goversion
@@ -274,31 +274,12 @@ jobs:
     needs: ci-binary-build
     if: startsWith(github.event.ref, 'refs/tags')
     steps:
-      - name: "Checkout Repository"
-        uses: actions/checkout@v4
-
-      - name: "Get aptly version"
-        env:
-          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
-        run: |
-           aptlyver=$(make -s version)
-           echo "Aptly Version: $aptlyver"
-           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
-        id: releaseversion
-
       - name: "Download Artifacts"
         uses: actions/download-artifact@v4
         with:
           path: out/
 
-      - name: "Create Release Notes"
-        run: |
-          echo -e "## Changes\n\n" > out/release-notes.md
-          dpkg-parsechangelog -S Changes | tail -n +4 >> out/release-notes.md
-
       - name: "Release"
         uses: softprops/action-gh-release@v2
         with:
-          name: "Aptly Release ${{ steps.releaseversion.outputs.VERSION }}"
           files: "out/**/aptly_*.zip"
-          body_path: "out/release-notes.md"

.github/workflows/scripts/upload-artifacts.sh

@@ -67,14 +67,11 @@ cleanup() {
 }
 trap cleanup EXIT
 
-sleeptime=5
-retries=60
 wait_task()
 {
     _id=$1
     _success=0
-    sleep $sleeptime
-    for t in `seq $retries`
+    for t in `seq 180`
     do
         jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
         _state=`echo $jsonret | jq .State`
@@ -87,7 +84,7 @@ wait_task()
             echo Error: task failed
             return 1
         fi
-        sleep $sleeptime
+        sleep 1
     done
     if [ "$_success" -ne 1 ]; then
         echo Error: task timeout

Makefile

@@ -140,7 +140,7 @@ dpkg: prepare swagger  ## Build debian packages
 	mkdir -p build && mv ../*.deb build/ ; \
 	cd build && ls -l *.deb
 
-binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
+binaries: prepare swagger  ## Build binary releases (FreeBSD, MacOS, Linux tar)
 	# build aptly
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
 	# install
@@ -195,7 +195,7 @@ docker-serve:  ## Run development server (auto recompiling) on http://localhost:
 docker-lint:  ## Run golangci-lint in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
 
-docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+docker-binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
 
 docker-man:  ## Create man page in docker container

README.rst

@@ -1,3 +1,7 @@
+=====
+aptly
+=====
+
 .. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
     :target: https://github.com/aptly-dev/aptly/actions
 
@@ -10,18 +14,15 @@
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly
 
-aptly
-=====
-
 Aptly is a swiss army knife for Debian repository management.
 
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
-open `issues <https://github.com/aptly-dev/aptly/issues>`_ or `discussions <https://github.com/aptly-dev/aptly/discussions>`_.
+mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
-Aptly features:
+Aptly features: ("+" means planned features)
 
 * make mirrors of remote Debian/Ubuntu repositories, limiting by components/architectures
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
@@ -31,50 +32,48 @@ Aptly features:
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
 * REST API for remote access
+* mirror repositories "as-is" (without resigning with user's key) (+)
+* support for yum repositories (+)
 
-Any contributions are welcome! Please see `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+Current limitations:
 
-Installation
-=============
+* translations are not supported yet
 
-Aptly can be installed on several operating systems.
+Install Stable Version
+-----------------------
 
-Debian / Ubuntu
-----------------
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
-Aptly is provided in the following debian packages:
+    deb http://repo.aptly.info/ squeeze main
 
-* **aptly**: Includes the main Aptly binary, man pages, and shell completions
-* **aptly-api**: A systemd service for the REST API, using the global /etc/aptly.conf
-* **aptly-dbg**: Debug symbols for troubleshooting
+And import key that is used to sign the release::
 
-The packages can be installed on official `Debian <https://packages.debian.org/search?keywords=aptly>`_ and `Ubuntu <https://packages.ubuntu.com/search?keywords=aptly>`_ distributions.
+    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
 
-Upstream Debian Packages
-~~~~~~~~~~~~~~~~~~~~~~~~~
+After that you can install aptly as any other software package::
 
-If a newer version (not available in Debian/Ubuntu) of aptly is required, upstream debian packages (built from git tags) can be installed as follows:
+    $ apt-get update
+    $ apt-get install aptly
 
-Install the following APT key (as root)::
+Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
+Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
-    wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
+Other Binaries
+~~~~~~~~~~~~~~~~~
 
-Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
+Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
 
-    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
+Install CI Version
+--------------------
 
-Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
+More recent versions are available as CI builds (development, might be unstable).
 
-Install aptly packages::
+Debian GNU/Linux
+~~~~~~~~~~~~~~~~~
 
-    apt-get update
-    apt-get install aptly
-    apt-get install aptly-api  # REST API systemd service
+Install the following APT key::
 
-CI Builds
-~~~~~~~~~~
-
-For testing new features or bugfixes, recent builds are available as CI builds (built from master, may be unstable!) and can be installed as follows:
+    sudo wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
 
 Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
 
@@ -82,19 +81,13 @@ Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
 
 Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
-Note: same gpg key is used as for the Upstream Debian Packages.
+Contributing
+------------
 
-Other Operating Systems
-------------------------
-
-Binary executables (depends almost only on libc) are available on `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_ for:
-
-- macOS / darwin (amd64, arm64)
-- FreeBSD (amd64, arm64, 386, arm)
-- Generic Linux (amd64, arm64, 386, arm)
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
 Integrations
-=============
+------------
 
 Vagrant:
 
@@ -131,7 +124,3 @@ GUI for aptly API:
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar
-
-Molior:
-
--   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth

Releasing.md

@@ -1,16 +0,0 @@
-# Creating a Release
-
-- create branch release/1.x.y
-- update debian/changelog
-- create PR, merge when approved
-- on updated master, create release:
-```
-version=$(dpkg-parsechangelog -S Version)
-echo Releasing prod version $version
-git tag -a v$version -m 'aptly: release $version'
-git push origin v$version master
-```
-- run swagger locally (`make docker-serve`)
-- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
-- releae www.aptly.info
-- create release announcement on https://github.com/aptly-dev/aptly/discussions

api/api.go

@@ -229,7 +229,7 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb.CollectionFactory) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)

api/db.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 
 	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
@@ -29,22 +28,18 @@ func apiDbCleanup(c *gin.Context) {
 
 		collectionFactory := context.NewCollectionFactory()
 
-		// collect information about referenced packages and their reflist buckets...
-		existingPackageRefs := deb.NewSplitRefList()
-		existingBuckets := deb.NewRefListDigestSet()
-
-		reflistMigration := collectionFactory.RefListCollection().NewMigration()
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
 
 		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
 		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-			sl := deb.NewSplitRefList()
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
-			if e != nil && e != database.ErrNotFound {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
 				return e
 			}
-
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-			existingBuckets.AddAllInRefList(sl)
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
 
 			return nil
 		})
@@ -53,14 +48,14 @@ func apiDbCleanup(c *gin.Context) {
 		}
 
 		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-			sl := deb.NewSplitRefList()
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
-			if e != nil && e != database.ErrNotFound {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
 				return e
 			}
 
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-			existingBuckets.AddAllInRefList(sl)
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
 
 			return nil
 		})
@@ -69,14 +64,12 @@ func apiDbCleanup(c *gin.Context) {
 		}
 
 		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-			sl := deb.NewSplitRefList()
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, snapshot.RefKey(), reflistMigration)
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if e != nil {
 				return e
 			}
 
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-			existingBuckets.AddAllInRefList(sl)
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
 
 			return nil
 		})
@@ -88,16 +81,13 @@ func apiDbCleanup(c *gin.Context) {
 			if published.SourceKind != deb.SourceLocalRepo {
 				return nil
 			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
 
 			for _, component := range published.Components() {
-				sl := deb.NewSplitRefList()
-				e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, published.RefKey(component), reflistMigration)
-				if e != nil {
-					return e
-				}
-
-				existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-				existingBuckets.AddAllInRefList(sl)
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
 			}
 			return nil
 		})
@@ -105,20 +95,11 @@ func apiDbCleanup(c *gin.Context) {
 			return nil, err
 		}
 
-		err = reflistMigration.Flush()
-		if err != nil {
-			return nil, err
-		}
-		if stats := reflistMigration.Stats(); stats.Reflists > 0 {
-			out.Printf("Split %d reflist(s) into %d bucket(s) (%d segment(s))",
-				stats.Reflists, stats.Buckets, stats.Segments)
-		}
-
 		// ... and compare it to the list of all packages
 		out.Printf("Loading list of all packages...")
 		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
-		toDelete := allPackageRefs.Subtract(existingPackageRefs.Flatten())
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 		// delete packages that are no longer referenced
 		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
@@ -139,28 +120,6 @@ func apiDbCleanup(c *gin.Context) {
 			}
 		}
 
-		bucketsToDelete, err := collectionFactory.RefListCollection().AllBucketDigests()
-		if err != nil {
-			return nil, err
-		}
-
-		bucketsToDelete.RemoveAll(existingBuckets)
-
-		out.Printf("Deleting unreferenced reflist buckets (%d)...", bucketsToDelete.Len())
-		if bucketsToDelete.Len() > 0 {
-			batch := db.CreateBatch()
-			err := bucketsToDelete.ForEach(func(digest []byte) error {
-				return collectionFactory.RefListCollection().UnsafeDropBucket(digest, batch)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if err := batch.Write(); err != nil {
-				return nil, err
-			}
-		}
-
 		// now, build a list of files that should be present in Repository (package pool)
 		out.Printf("Building list of files referenced by packages...")
 		referencedFiles := make([]string, 0, existingPackageRefs.Len())

api/files.go

@@ -170,7 +170,11 @@ func apiFilesListFiles(c *gin.Context) {
 	listLock := &sync.Mutex{}
 	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := walker.Walk(root, func(path string, _ os.FileInfo) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
 		if path == root {
 			return nil
 		}

api/metrics.go

@@ -102,7 +102,7 @@ func countPackagesByRepos() {
 
 		components := repo.Components()
 		for _, c := range components {
-			count := float64(repo.RefList(c).Len())
+			count := float64(len(repo.RefList(c).Refs))
 			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
 		}
 

api/mirror.go

@@ -150,7 +150,7 @@ func apiMirrorsCreate(c *gin.Context) {
 		return
 	}
 
-	err = collection.Add(repo, collectionFactory.RefListCollection())
+	err = collection.Add(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
 		return
@@ -229,7 +229,7 @@ func apiMirrorsShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
@@ -260,7 +260,7 @@ func apiMirrorsPackages(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
@@ -491,12 +491,12 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				collection.Update(remote, collectionFactory.RefListCollection())
+				collection.Update(remote)
 			}
 		}()
 
 		remote.MarkAsUpdating()
-		err = collection.Update(remote, collectionFactory.RefListCollection())
+		err = collection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -654,7 +654,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 
 		log.Info().Msgf("%s: Finalizing download...", b.Name)
 		remote.FinalizeDownload(collectionFactory, out)
-		err = collectionFactory.RemoteRepoCollection().Update(remote, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

api/publish.go

@@ -297,10 +297,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			switch s := source.(type) {
 			case *deb.Snapshot:
 				snapshotCollection := collectionFactory.SnapshotCollection()
-				err = snapshotCollection.LoadComplete(s, collectionFactory.RefListCollection())
+				err = snapshotCollection.LoadComplete(s)
 			case *deb.LocalRepo:
 				localCollection := collectionFactory.LocalRepoCollection()
-				err = localCollection.LoadComplete(s, collectionFactory.RefListCollection())
+				err = localCollection.LoadComplete(s)
 			default:
 				err = fmt.Errorf("unexpected type for source: %T", source)
 			}
@@ -352,7 +352,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		err = collection.Add(published, collectionFactory.RefListCollection())
+		err = collection.Add(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -469,7 +469,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(published, collectionFactory, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(published, collectionFactory)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
 		}
@@ -495,7 +495,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
 		}
 
-		err = collection.Update(published, collectionFactory.RefListCollection())
+		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}

api/repos.go

@@ -155,13 +155,6 @@ func apiReposCreate(c *gin.Context) {
 		return
 	}
 
-	collection := collectionFactory.LocalRepoCollection()
-	err := collection.Add(repo, collectionFactory.RefListCollection())
-	if err != nil {
-		AbortWithJSONError(c, 400, err)
-		return
-	}
-
 	err := localRepoCollection.Add(repo)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusInternalServerError, err)
@@ -225,7 +218,7 @@ func apiReposEdit(c *gin.Context) {
 		repo.DefaultComponent = *b.DefaultComponent
 	}
 
-	err = collection.Update(repo, collectionFactory.RefListCollection())
+	err = collection.Update(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -333,7 +326,7 @@ func apiReposPackagesShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -367,7 +360,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	resources := []string{string(repo.Key())}
 
 	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -396,9 +389,9 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 			}
 		}
 
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -524,7 +517,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -559,9 +552,9 @@ func apiReposPackageFromDir(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
 		}
 
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -874,7 +867,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		_, failedFiles2, err = deb.ImportChangesFiles(
 			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
 			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-			collectionFactory.RefListCollection(), context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
 		failedFiles = append(failedFiles, failedFiles2...)
 
 		if err != nil {

api/snapshot.go

@@ -93,7 +93,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
 		}
 
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -107,7 +107,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -174,7 +174,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 
 	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		for i := range sources {
-			err = snapshotCollection.LoadComplete(sources[i], collectionFactory.RefListCollection())
+			err = snapshotCollection.LoadComplete(sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
@@ -197,9 +197,9 @@ func apiSnapshotsCreate(c *gin.Context) {
 			}
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewSplitRefListFromPackageList(list), b.Description)
+		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -255,7 +255,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err := collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -269,7 +269,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -333,7 +333,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = collectionFactory.SnapshotCollection().Update(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Update(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -360,7 +360,7 @@ func apiSnapshotsShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshot)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -452,20 +452,20 @@ func apiSnapshotsDiff(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshotA, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshotA)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
 	}
 
-	err = collection.LoadComplete(snapshotB, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshotB)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection(), nil)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -508,7 +508,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshot)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -584,13 +584,13 @@ func apiSnapshotsMerge(c *gin.Context) {
 	}
 
 	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = snapshotCollection.LoadComplete(sources[0], collectionFactory.RefListCollection())
+		err = snapshotCollection.LoadComplete(sources[0])
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 		result := sources[0].RefList()
 		for i := 1; i < len(sources); i++ {
-			err = snapshotCollection.LoadComplete(sources[i], collectionFactory.RefListCollection())
+			err = snapshotCollection.LoadComplete(sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
@@ -609,7 +609,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
 		}

azure/public.go

@@ -12,7 +12,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
 )
 
@@ -179,7 +179,7 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 // Internal copy or move implementation
 func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
 	const leaseDuration = 30
-	leaseID := uuid.NewString()
+	leaseID := uuid.NewRandom().String()
 
 	serviceClient := storage.az.client.ServiceClient()
 	containerClient := serviceClient.NewContainerClient(storage.az.container)

cmd/cmd.go

@@ -21,7 +21,7 @@ const (
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist deb.AnyRefList, collectionFactory *deb.CollectionFactory) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {

cmd/db_cleanup.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
@@ -25,20 +24,12 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
 	collectionFactory := context.NewCollectionFactory()
 
-	// collect information about references packages and their reflistbuckets...
-	existingPackageRefs := deb.NewSplitRefList()
-	existingBuckets := deb.NewRefListDigestSet()
+	// collect information about references packages...
+	existingPackageRefs := deb.NewPackageRefList()
 
 	// used only in verbose mode to report package use source
 	packageRefSources := map[string][]string{}
 
-	var reflistMigration *deb.RefListMigration
-	if !dryRun {
-		reflistMigration = collectionFactory.RefListCollection().NewMigration()
-	} else {
-		reflistMigration = collectionFactory.RefListCollection().NewMigrationDryRun()
-	}
-
 	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
@@ -48,21 +39,20 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		sl := deb.NewSplitRefList()
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
-		if e != nil && e != database.ErrNotFound {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
 			return e
 		}
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-		existingBuckets.AddAllInRefList(sl)
-
-		if verbose {
-			description := fmt.Sprintf("mirror %s", repo.Name)
-			sl.ForEach(func(key []byte) error {
-				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
-				return nil
-			})
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 
 		return nil
@@ -81,23 +71,21 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		sl := deb.NewSplitRefList()
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
-		if e != nil && e != database.ErrNotFound {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
 			return e
 		}
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-		existingBuckets.AddAllInRefList(sl)
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-
-		if verbose {
-			description := fmt.Sprintf("local repo %s", repo.Name)
-			sl.ForEach(func(key []byte) error {
-				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
-				return nil
-			})
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 
 		return nil
@@ -116,18 +104,16 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		sl := deb.NewSplitRefList()
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, snapshot.RefKey(), reflistMigration)
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-		existingBuckets.AddAllInRefList(sl)
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
 
 		if verbose {
 			description := fmt.Sprintf("snapshot %s", snapshot.Name)
-			sl.ForEach(func(key []byte) error {
+			snapshot.RefList().ForEach(func(key []byte) error {
 				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 				return nil
 			})
@@ -150,21 +136,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+		if e != nil {
+			return e
+		}
 
 		for _, component := range published.Components() {
-			sl := deb.NewSplitRefList()
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, published.RefKey(component), reflistMigration)
-			if e != nil {
-				return e
-			}
-
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-			existingBuckets.AddAllInRefList(sl)
-
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
 			if verbose {
 				description := fmt.Sprintf("published repository %s:%s/%s component %s",
 					published.Storage, published.Prefix, published.Distribution, component)
-				sl.ForEach(func(key []byte) error {
+				published.RefList(component).ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -178,29 +160,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	collectionFactory.Flush()
 
-	err = reflistMigration.Flush()
-	if err != nil {
-		return err
-	}
-
-	if verbose {
-		if stats := reflistMigration.Stats(); stats.Reflists > 0 {
-			if !dryRun {
-				context.Progress().ColoredPrintf("@{w!}Split %d reflist(s) into %d bucket(s) (%d segment(s))@|",
-					stats.Reflists, stats.Buckets, stats.Segments)
-			} else {
-				context.Progress().ColoredPrintf(
-					"@{y!}Skipped splitting %d reflist(s) into %d bucket(s) (%d segment(s)), as -dry-run has been requested.@|",
-					stats.Reflists, stats.Buckets, stats.Segments)
-			}
-		}
-	}
-
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Subtract(existingPackageRefs.Flatten())
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
 	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
@@ -238,32 +202,6 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	bucketsToDelete, err := collectionFactory.RefListCollection().AllBucketDigests()
-	if err != nil {
-		return err
-	}
-
-	bucketsToDelete.RemoveAll(existingBuckets)
-
-	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced reflist buckets (%d)...@|", bucketsToDelete.Len())
-	if bucketsToDelete.Len() > 0 {
-		if !dryRun {
-			batch := db.CreateBatch()
-			err := bucketsToDelete.ForEach(func(digest []byte) error {
-				return collectionFactory.RefListCollection().UnsafeDropBucket(digest, batch)
-			})
-			if err != nil {
-				return err
-			}
-
-			if err := batch.Write(); err != nil {
-				return err
-			}
-		} else {
-			context.Progress().ColoredPrintf("@{y!}Skipped reflist deletion, as -dry-run has been requested.@|")
-		}
-	}
-
 	collectionFactory.Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)

cmd/mirror_create.go

@@ -69,7 +69,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	err = collectionFactory.RemoteRepoCollection().Add(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}

cmd/mirror_edit.go

@@ -78,7 +78,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/mirror_rename.go

@@ -37,7 +37,7 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 	}
 
 	repo.Name = newName
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/mirror_show.go

@@ -38,7 +38,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -98,13 +98,12 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 
 	name := args[0]
 
-	collectionFactory := context.NewCollectionFactory()
-	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -114,7 +113,7 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if repo.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err != nil {
 				return fmt.Errorf("unable to get package list: %s", err)
 			}

cmd/mirror_update.go

@@ -29,7 +29,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -101,12 +101,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+			collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
 	repo.MarkAsUpdating()
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -262,7 +262,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	repo.FinalizeDownload(collectionFactory, context.Progress())
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}

cmd/package_show.go

@@ -14,7 +14,7 @@ import (
 
 func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -30,7 +30,7 @@ func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory)
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -46,7 +46,7 @@ func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory)
 	}
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}

cmd/publish_snapshot.go

@@ -49,7 +49,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -85,7 +85,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo, collectionFactory.RefListCollection())
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -175,7 +175,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Add(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}

cmd/publish_switch.go

@@ -72,7 +72,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -108,7 +108,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Update(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}

cmd/publish_update.go

@@ -69,7 +69,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Update(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}

cmd/repo_add.go

@@ -28,7 +28,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
@@ -58,9 +58,9 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	processedFiles = append(processedFiles, otherFiles...)
 
-	repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}

cmd/repo_create.go

@@ -36,7 +36,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
@@ -44,7 +44,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		repo.UpdateRefList(snapshot.RefList())
 	}
 
-	err = collectionFactory.LocalRepoCollection().Add(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}

cmd/repo_edit.go

@@ -22,7 +22,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -53,7 +53,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/repo_include.go

@@ -67,7 +67,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	_, failedFiles2, err = deb.ImportChangesFiles(
 		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
 		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-		collectionFactory.RefListCollection(), context.PackagePool(), collectionFactory.ChecksumCollection,
+		context.PackagePool(), collectionFactory.ChecksumCollection,
 		uploaders, query.Parse)
 	failedFiles = append(failedFiles, failedFiles2...)
 

cmd/repo_list.go

@@ -36,7 +36,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 			if e != nil {
 				return e
 			}
@@ -77,8 +77,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
 	i := 0
 	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		collectionFactory := context.NewCollectionFactory()
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}

cmd/repo_move.go

@@ -25,13 +25,13 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var (
-		srcRefList *deb.SplitRefList
+		srcRefList *deb.PackageRefList
 		srcRepo    *deb.LocalRepo
 	)
 
@@ -45,7 +45,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -59,12 +59,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		if srcRemoteRepo.RefList().Len() == 0 {
+		if srcRemoteRepo.RefList() == nil {
 			return fmt.Errorf("unable to %s: mirror not updated", command)
 		}
 
@@ -161,17 +161,17 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		dstRepo.UpdateRefList(deb.NewSplitRefListFromPackageList(dstList))
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = collectionFactory.LocalRepoCollection().Update(dstRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
 		if command == "move" { // nolint: goconst
-			srcRepo.UpdateRefList(deb.NewSplitRefListFromPackageList(srcList))
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = collectionFactory.LocalRepoCollection().Update(srcRepo, collectionFactory.RefListCollection())
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}

cmd/repo_remove.go

@@ -24,7 +24,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -63,9 +63,9 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}

cmd/repo_rename.go

@@ -32,7 +32,7 @@ func aptlyRepoRename(cmd *commander.Command, args []string) error {
 	}
 
 	repo.Name = newName
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/repo_show.go

@@ -36,7 +36,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -63,13 +63,12 @@ func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 
 	name := args[0]
 
-	collectionFactory := context.NewCollectionFactory()
-	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -80,7 +79,7 @@ func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if repo.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err == nil {
 				packageList = list.FullNames()
 			}

cmd/snapshot_create.go

@@ -30,7 +30,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -50,7 +50,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -71,7 +71,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	err = collectionFactory.SnapshotCollection().Add(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}

cmd/snapshot_diff.go

@@ -23,7 +23,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
@@ -34,13 +34,13 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection(), nil)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_filter.go

@@ -27,7 +27,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(source, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
@@ -87,7 +87,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
 		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
 
-	err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_merge.go

@@ -24,7 +24,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i], collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
@@ -57,7 +57,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_pull.go

@@ -29,7 +29,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -40,7 +40,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(source, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -149,7 +149,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}

cmd/snapshot_rename.go

@@ -32,7 +32,7 @@ func aptlySnapshotRename(cmd *commander.Command, args []string) error {
 	}
 
 	snapshot.Name = newName
-	err = collectionFactory.SnapshotCollection().Update(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/snapshot_search.go

@@ -25,7 +25,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	command := cmd.Parent.Name()
 	collectionFactory := context.NewCollectionFactory()
 
-	var reflist *deb.SplitRefList
+	var reflist *deb.PackageRefList
 
 	if command == "snapshot" { // nolint: goconst
 		var snapshot *deb.Snapshot
@@ -34,7 +34,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -47,7 +47,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -60,7 +60,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}

cmd/snapshot_show.go

@@ -35,7 +35,7 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -86,17 +86,16 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 }
 
 func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
-	collectionFactory := context.NewCollectionFactory()
 	var err error
 
 	name := args[0]
 
-	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	snapshot, err := context.NewCollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -106,14 +105,14 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 		for _, sourceID := range snapshot.SourceIDs {
 			if snapshot.SourceKind == deb.SourceSnapshot {
 				var source *deb.Snapshot
-				source, err = collectionFactory.SnapshotCollection().ByUUID(sourceID)
+				source, err = context.NewCollectionFactory().SnapshotCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				snapshot.Snapshots = append(snapshot.Snapshots, source)
 			} else if snapshot.SourceKind == deb.SourceLocalRepo {
 				var source *deb.LocalRepo
-				source, err = collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+				source, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
@@ -134,7 +133,7 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if snapshot.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err != nil {
 				return fmt.Errorf("unable to get package list: %s", err)
 			}

cmd/snapshot_verify.go

@@ -23,7 +23,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i], collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}

database/database.go

@@ -48,8 +48,6 @@ type Storage interface {
 
 	CreateTemporary() (Storage, error)
 
-	GetRecommendedMaxKVSize() int
-
 	Open() error
 	Close() error
 	CompactDB() error

database/etcddb/storage.go

@@ -1,11 +1,11 @@
 package etcddb
 
 import (
-	"fmt"
-
 	"github.com/aptly-dev/aptly/database"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"fmt"
 )
 
 type EtcDStorage struct {
@@ -16,7 +16,7 @@ type EtcDStorage struct {
 
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
-	tmp := uuid.NewString()
+	tmp := uuid.NewRandom().String()
 	return &EtcDStorage{
 		url:       s.url,
 		db:        s.db,

database/goleveldb/database.go

@@ -9,13 +9,10 @@ import (
 	"github.com/aptly-dev/aptly/database"
 )
 
-const blockSize = 4 * 1024
-
 func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	o := &opt.Options{
 		Filter:                 filter.NewBloomFilter(10),
 		OpenFilesCacheCapacity: 256,
-		BlockSize:              blockSize,
 	}
 
 	if throttleCompaction {

database/goleveldb/storage.go

@@ -16,17 +16,6 @@ type storage struct {
 	db   *leveldb.DB
 }
 
-func (s *storage) GetRecommendedMaxKVSize() int {
-	// The block size configured is not actually a *set* block size, but rather a
-	// *minimum*. LevelDB only checks if a block is full after a new key/value pair is
-	// written, meaning that blocks will tend to overflow a bit.
-	// Therefore, using the default block size as the max value size will ensure
-	// that a new block will only contain a single value and that the size will
-	// only ever be as large as around double the block size (if the block was
-	// nearly full before the new items were added).
-	return blockSize
-}
-
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *storage) CreateTemporary() (database.Storage, error) {
 	tempdir, err := os.MkdirTemp("", "aptly")

deb/changes.go

@@ -291,8 +291,7 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 // ImportChangesFiles imports referenced files in changes files into local repository
 func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
 	verifier pgp.Verifier, repoTemplate *template.Template, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
-	reflistCollection *RefListCollection, pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders,
-	parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
 
 	for _, path := range changesFiles {
 		var changes *Changes
@@ -360,7 +359,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 			}
 		}
 
-		err = localRepoCollection.LoadComplete(repo, reflistCollection)
+		err = localRepoCollection.LoadComplete(repo)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
 		}
@@ -383,9 +382,9 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
 		}
 
-		repo.UpdateRefList(NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
 
-		err = localRepoCollection.Update(repo, reflistCollection)
+		err = localRepoCollection.Update(repo)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to save: %s", err)
 		}

deb/changes_test.go

@@ -21,7 +21,6 @@ type ChangesSuite struct {
 	db                  database.Storage
 	localRepoCollection *LocalRepoCollection
 	packageCollection   *PackageCollection
-	reflistCollection   *RefListCollection
 	packagePool         aptly.PackagePool
 	checksumStorage     aptly.ChecksumStorage
 	progress            aptly.Progress
@@ -43,7 +42,6 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.localRepoCollection = NewLocalRepoCollection(s.db)
 	s.packageCollection = NewPackageCollection(s.db)
-	s.reflistCollection = NewRefListCollection(s.db)
 
 	s.checksumStorage = files.NewMockChecksumStorage()
 	s.packagePool = files.NewPackagePool(s.Dir, false)
@@ -90,7 +88,7 @@ func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
 
 func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	repo := NewLocalRepo("test", "Test Comment")
-	c.Assert(s.localRepoCollection.Add(repo, s.reflistCollection), IsNil)
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
 
 	origFailedFiles := []string{
 		"testdata/changes/calamares.changes",
@@ -126,8 +124,7 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	processedFiles, failedFiles, err := ImportChangesFiles(
 		append(changesFiles, "testdata/changes/notexistent.changes"),
 		s.Reporter, true, true, false, false, &NullVerifier{},
-		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.reflistCollection, s.packagePool,
-		func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
@@ -136,7 +133,7 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 
 func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
 	repo := NewLocalRepo("test", "Test Comment")
-	c.Assert(s.localRepoCollection.Add(repo, s.reflistCollection), IsNil)
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
 
 	changesFiles, failedFiles := CollectChangesFiles(
 		[]string{"testdata/dbgsym-with-source-version"}, s.Reporter)
@@ -145,8 +142,7 @@ func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
 
 	_, failedFiles, err := ImportChangesFiles(
 		changesFiles, s.Reporter, true, true, false, true, &NullVerifier{},
-		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.reflistCollection, s.packagePool,
-		func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, IsNil)

deb/collections.go

@@ -16,7 +16,6 @@ type CollectionFactory struct {
 	snapshots      *SnapshotCollection
 	localRepos     *LocalRepoCollection
 	publishedRepos *PublishedRepoCollection
-	reflists       *RefListCollection
 	checksums      *ChecksumCollection
 }
 
@@ -92,17 +91,6 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 	return factory.publishedRepos
 }
 
-func (factory *CollectionFactory) RefListCollection() *RefListCollection {
-	factory.Lock()
-	defer factory.Unlock()
-
-	if factory.reflists == nil {
-		factory.reflists = NewRefListCollection(factory.db)
-	}
-
-	return factory.reflists
-}
-
 // ChecksumCollection returns (or creates) new ChecksumCollection
 func (factory *CollectionFactory) ChecksumCollection(db database.ReaderWriter) aptly.ChecksumStorage {
 	factory.Lock()

deb/contents.go

@@ -7,7 +7,7 @@ import (
 	"io"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 )
 
 // ContentsIndex calculates mapping from files to packages, with sorting and aggregation
@@ -20,7 +20,7 @@ type ContentsIndex struct {
 func NewContentsIndex(db database.Storage) *ContentsIndex {
 	return &ContentsIndex{
 		db:     db,
-		prefix: []byte(uuid.NewString()),
+		prefix: []byte(uuid.New()),
 	}
 }
 

deb/graph.go

@@ -33,7 +33,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
-		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -55,7 +55,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -81,7 +81,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	})
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
-		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}

deb/list.go

@@ -91,7 +91,7 @@ func NewPackageListWithDuplicates(duplicates bool, capacity int) *PackageList {
 }
 
 // NewPackageListFromRefList loads packages list from PackageRefList
-func NewPackageListFromRefList(reflist AnyRefList, collection *PackageCollection, progress aptly.Progress) (*PackageList, error) {
+func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageCollection, progress aptly.Progress) (*PackageList, error) {
 	// empty reflist
 	if reflist == nil {
 		return NewPackageList(), nil

deb/local.go

@@ -7,7 +7,7 @@ import (
 	"log"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -26,13 +26,13 @@ type LocalRepo struct {
 	// Uploaders configuration
 	Uploaders *Uploaders `codec:"Uploaders,omitempty" json:"-"`
 	// "Snapshot" of current list of packages
-	packageRefs *SplitRefList
+	packageRefs *PackageRefList
 }
 
 // NewLocalRepo creates new instance of Debian local repository
 func NewLocalRepo(name string, comment string) *LocalRepo {
 	return &LocalRepo{
-		UUID:    uuid.NewString(),
+		UUID:    uuid.New(),
 		Name:    name,
 		Comment: comment,
 	}
@@ -55,13 +55,13 @@ func (repo *LocalRepo) NumPackages() int {
 }
 
 // RefList returns package list for repo
-func (repo *LocalRepo) RefList() *SplitRefList {
+func (repo *LocalRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
 // UpdateRefList changes package list for local repo
-func (repo *LocalRepo) UpdateRefList(sl *SplitRefList) {
-	repo.packageRefs = sl
+func (repo *LocalRepo) UpdateRefList(reflist *PackageRefList) {
+	repo.packageRefs = reflist
 }
 
 // Encode does msgpack encoding of LocalRepo
@@ -140,14 +140,14 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
 }
 
 // Add appends new repo to collection and saves it
-func (collection *LocalRepoCollection) Add(repo *LocalRepo, reflistCollection *RefListCollection) error {
+func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
 	_, err := collection.ByName(repo.Name)
 
 	if err == nil {
 		return fmt.Errorf("local repo with name %s already exists", repo.Name)
 	}
 
-	err = collection.Update(repo, reflistCollection)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
@@ -157,25 +157,27 @@ func (collection *LocalRepoCollection) Add(repo *LocalRepo, reflistCollection *R
 }
 
 // Update stores updated information about repo in DB
-func (collection *LocalRepoCollection) Update(repo *LocalRepo, reflistCollection *RefListCollection) error {
+func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
 	batch := collection.db.CreateBatch()
 	batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		bc := reflistCollection.NewBatch(batch)
-		reflistCollection.UpdateInBatch(repo.packageRefs, repo.RefKey(), bc)
+		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
 	return batch.Write()
 }
 
 // LoadComplete loads additional information for local repo
-func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo, reflistCollection *RefListCollection) error {
-	repo.packageRefs = NewSplitRefList()
-	err := reflistCollection.LoadComplete(repo.packageRefs, repo.RefKey())
+func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
+	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
 	}
+	if err != nil {
+		return err
+	}
 
-	return err
+	repo.packageRefs = &PackageRefList{}
+	return repo.packageRefs.Decode(encoded)
 }
 
 // ByName looks up repository by name

deb/local_test.go

@@ -12,7 +12,7 @@ import (
 type LocalRepoSuite struct {
 	db      database.Storage
 	list    *PackageList
-	reflist *SplitRefList
+	reflist *PackageRefList
 	repo    *LocalRepo
 }
 
@@ -24,7 +24,7 @@ func (s *LocalRepoSuite) SetUpTest(c *C) {
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
 
-	s.reflist = NewSplitRefListFromPackageList(s.list)
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 
 	s.repo = NewLocalRepo("lrepo", "Super repo")
 	s.repo.packageRefs = s.reflist
@@ -75,11 +75,10 @@ func (s *LocalRepoSuite) TestRefKey(c *C) {
 }
 
 type LocalRepoCollectionSuite struct {
-	db                database.Storage
-	collection        *LocalRepoCollection
-	reflistCollection *RefListCollection
-	list              *PackageList
-	reflist           *SplitRefList
+	db         database.Storage
+	collection *LocalRepoCollection
+	list       *PackageList
+	reflist    *PackageRefList
 }
 
 var _ = Suite(&LocalRepoCollectionSuite{})
@@ -87,13 +86,12 @@ var _ = Suite(&LocalRepoCollectionSuite{})
 func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewLocalRepoCollection(s.db)
-	s.reflistCollection = NewRefListCollection(s.db)
 
 	s.list = NewPackageList()
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
 
-	s.reflist = NewSplitRefListFromRefList(NewPackageRefListFromPackageList(s.list))
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 }
 
 func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
@@ -105,8 +103,8 @@ func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
-	c.Assert(s.collection.Add(repo, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(repo, s.reflistCollection), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.Add(repo), IsNil)
+	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
 	r, err := s.collection.ByName("local1")
 	c.Assert(err, IsNil)
@@ -123,7 +121,7 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
-	c.Assert(s.collection.Add(repo, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
@@ -137,7 +135,7 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 
 func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
-	c.Assert(s.collection.Update(repo, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewLocalRepoCollection(s.db)
 	r, err := collection.ByName("local1")
@@ -145,20 +143,20 @@ func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	c.Assert(r.packageRefs, IsNil)
 
 	repo.packageRefs = s.reflist
-	c.Assert(s.collection.Update(repo, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection = NewLocalRepoCollection(s.db)
 	r, err = collection.ByName("local1")
 	c.Assert(err, IsNil)
 	c.Assert(r.packageRefs, IsNil)
 	c.Assert(r.NumPackages(), Equals, 0)
-	c.Assert(s.collection.LoadComplete(r, s.reflistCollection), IsNil)
+	c.Assert(s.collection.LoadComplete(r), IsNil)
 	c.Assert(r.NumPackages(), Equals, 2)
 }
 
 func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
-	s.collection.Add(repo, s.reflistCollection)
+	s.collection.Add(repo)
 
 	count := 0
 	err := s.collection.ForEach(func(*LocalRepo) error {
@@ -180,10 +178,10 @@ func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
 
 func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
 	repo1 := NewLocalRepo("local1", "Comment 1")
-	s.collection.Add(repo1, s.reflistCollection)
+	s.collection.Add(repo1)
 
 	repo2 := NewLocalRepo("local2", "Comment 2")
-	s.collection.Add(repo2, s.reflistCollection)
+	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
 	c.Check(r1, Equals, repo1)

deb/publish.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -37,7 +37,7 @@ type repoSourceItem struct {
 	// Pointer to local repo if SourceKind == "local"
 	localRepo *LocalRepo
 	// Package references is SourceKind == "local"
-	packageRefs *SplitRefList
+	packageRefs *PackageRefList
 }
 
 // PublishedRepo is a published for http/ftp representation of snapshot as Debian repository
@@ -354,7 +354,7 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 func NewPublishedRepo(storage, prefix, distribution string, architectures []string,
 	components []string, sources []interface{}, collectionFactory *CollectionFactory, multiDist bool) (*PublishedRepo, error) {
 	result := &PublishedRepo{
-		UUID:          uuid.NewString(),
+		UUID:          uuid.New(),
 		Storage:       storage,
 		Architectures: architectures,
 		Sources:       make(map[string]string),
@@ -609,7 +609,7 @@ func (p *PublishedRepo) RefKey(component string) []byte {
 }
 
 // RefList returns list of package refs in local repo
-func (p *PublishedRepo) RefList(component string) *SplitRefList {
+func (p *PublishedRepo) RefList(component string) *PackageRefList {
 	item := p.sourceItems[component]
 	if p.SourceKind == SourceLocalRepo {
 		return item.packageRefs
@@ -1243,14 +1243,14 @@ func (collection *PublishedRepoCollection) loadList() {
 }
 
 // Add appends new repo to collection and saves it
-func (collection *PublishedRepoCollection) Add(repo *PublishedRepo, reflistCollection *RefListCollection) error {
+func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 	collection.loadList()
 
 	if collection.CheckDuplicate(repo) != nil {
 		return fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s already exists", repo.Storage, repo.Prefix, repo.Distribution)
 	}
 
-	err := collection.Update(repo, reflistCollection)
+	err := collection.Update(repo)
 	if err != nil {
 		return err
 	}
@@ -1273,14 +1273,13 @@ func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *
 }
 
 // Update stores updated information about repo in DB
-func (collection *PublishedRepoCollection) Update(repo *PublishedRepo, reflistCollection *RefListCollection) error {
+func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
 	batch := collection.db.CreateBatch()
 	batch.Put(repo.Key(), repo.Encode())
 
 	if repo.SourceKind == SourceLocalRepo {
-		rb := reflistCollection.NewBatch(batch)
 		for component, item := range repo.sourceItems {
-			reflistCollection.UpdateInBatch(item.packageRefs, repo.RefKey(component), rb)
+			batch.Put(repo.RefKey(component), item.packageRefs.Encode())
 		}
 	}
 	return batch.Write()
@@ -1313,7 +1312,7 @@ func (collection *PublishedRepoCollection) LoadShallow(repo *PublishedRepo, coll
 				return
 			}
 
-			item.packageRefs = NewSplitRefList()
+			item.packageRefs = &PackageRefList{}
 			repo.sourceItems[component] = item
 		}
 	} else {
@@ -1329,29 +1328,35 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 	if repo.SourceKind == SourceSnapshot {
 		for _, item := range repo.sourceItems {
-			err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot, collectionFactory.RefListCollection())
+			err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot)
 			if err != nil {
 				return
 			}
 		}
 	} else if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
-			err = collectionFactory.LocalRepoCollection().LoadComplete(item.localRepo, collectionFactory.RefListCollection())
+			err = collectionFactory.LocalRepoCollection().LoadComplete(item.localRepo)
 			if err != nil {
 				return
 			}
 
-			err = collectionFactory.RefListCollection().LoadComplete(item.packageRefs, repo.RefKey(component))
+			var encoded []byte
+			encoded, err = collection.db.Get(repo.RefKey(component))
 			if err != nil {
 				// < 0.6 saving w/o component name
 				if err == database.ErrNotFound && len(repo.Sources) == 1 {
-					err = collectionFactory.RefListCollection().LoadComplete(item.packageRefs, repo.RefKey(""))
+					encoded, err = collection.db.Get(repo.RefKey(""))
 				}
 
 				if err != nil {
 					return
 				}
 			}
+
+			err = item.packageRefs.Decode(encoded)
+			if err != nil {
+				return
+			}
 		}
 	} else {
 		panic("unknown SourceKind")
@@ -1456,11 +1461,6 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 	referencedFiles := map[string][]string{}
 	processedComponentRefs := map[string]*PackageRefList{}
 
-	processedComponentBuckets := map[string]*RefListDigestSet{}
-	for _, component := range components {
-		processedComponentBuckets[component] = NewRefListDigestSet()
-	}
-
 	for _, r := range collection.list {
 		if r.Prefix == prefix && !r.MultiDist {
 			matches := false
@@ -1484,51 +1484,36 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 
 			for _, component := range components {
 				if utils.StrSliceHasItem(repoComponents, component) {
-					processedBuckets := processedComponentBuckets[component]
+					unseenRefs := r.RefList(component)
+					processedRefs := processedComponentRefs[component]
+					if processedRefs != nil {
+						unseenRefs = unseenRefs.Subtract(processedRefs)
+					} else {
+						processedRefs = NewPackageRefList()
+					}
 
-					err := r.RefList(component).ForEachBucket(func(digest []byte, bucket *PackageRefList) error {
-						if processedBuckets.Has(digest) {
-							return nil
-						}
-						processedBuckets.Add(digest)
+					if unseenRefs.Len() == 0 {
+						continue
+					}
+					processedComponentRefs[component] = processedRefs.Merge(unseenRefs, false, true)
 
-						unseenRefs := bucket
-						processedRefs := processedComponentRefs[component]
-						if processedRefs != nil {
-							unseenRefs = unseenRefs.Subtract(processedRefs)
-						} else {
-							processedRefs = NewPackageRefList()
-						}
+					packageList, err := NewPackageListFromRefList(unseenRefs, collectionFactory.PackageCollection(), progress)
+					if err != nil {
+						return nil, err
+					}
 
-						if unseenRefs.Len() == 0 {
-							return nil
-						}
-						processedComponentRefs[component] = processedRefs.Merge(unseenRefs, false, true)
-
-						packageList, err := NewPackageListFromRefList(unseenRefs, collectionFactory.PackageCollection(), progress)
+					packageList.ForEach(func(p *Package) error {
+						poolDir, err := p.PoolDirectory()
 						if err != nil {
 							return err
 						}
 
-						packageList.ForEach(func(p *Package) error {
-							poolDir, err := p.PoolDirectory()
-							if err != nil {
-								return err
-							}
-
-							for _, f := range p.Files() {
-								referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, f.Filename))
-							}
-
-							return nil
-						})
+						for _, f := range p.Files() {
+							referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, f.Filename))
+						}
 
 						return nil
 					})
-
-					if err != nil {
-						return nil, err
-					}
 				}
 			}
 		}

deb/publish_bench_test.go

@@ -31,7 +31,6 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 	packageCollection := factory.PackageCollection()
 	repoCollection := factory.LocalRepoCollection()
 	publishCollection := factory.PublishedRepoCollection()
-	reflistCollection := factory.RefListCollection()
 
 	sharedRefs := NewPackageRefList()
 	{
@@ -92,14 +91,14 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 		repo := NewLocalRepo(fmt.Sprintf("repo%d", repoIndex), "comment")
 		repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex)
 		repo.DefaultComponent = defaultComponent
-		repo.UpdateRefList(NewSplitRefListFromRefList(refs.Merge(sharedRefs, false, true)))
-		repoCollection.Add(repo, reflistCollection)
+		repo.UpdateRefList(refs.Merge(sharedRefs, false, true))
+		repoCollection.Add(repo)
 
 		publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false)
 		if err != nil {
 			b.Fatal(err)
 		}
-		publishCollection.Add(publish, reflistCollection)
+		publishCollection.Add(publish)
 	}
 
 	db.CompactDB()

deb/publish_test.go

@@ -83,7 +83,6 @@ type PublishedRepoSuite struct {
 	db                                  database.Storage
 	factory                             *CollectionFactory
 	packageCollection                   *PackageCollection
-	reflistCollection                   *RefListCollection
 }
 
 var _ = Suite(&PublishedRepoSuite{})
@@ -115,22 +114,21 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.p2.UpdateFiles(s.p1.Files())
 	s.p3.UpdateFiles(s.p1.Files())
 
-	s.reflist = NewSplitRefListFromPackageList(s.list)
-	s.reflistCollection = s.factory.RefListCollection()
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
-	s.factory.RemoteRepoCollection().Add(repo, s.reflistCollection)
+	s.factory.RemoteRepoCollection().Add(repo)
 
 	s.localRepo = NewLocalRepo("local1", "comment1")
 	s.localRepo.packageRefs = s.reflist
-	s.factory.LocalRepoCollection().Add(s.localRepo, s.reflistCollection)
+	s.factory.LocalRepoCollection().Add(s.localRepo)
 
 	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
-	s.factory.SnapshotCollection().Add(s.snapshot, s.reflistCollection)
+	s.factory.SnapshotCollection().Add(s.snapshot)
 
 	s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
-	s.factory.SnapshotCollection().Add(s.snapshot2, s.reflistCollection)
+	s.factory.SnapshotCollection().Add(s.snapshot2)
 
 	s.packageCollection = s.factory.PackageCollection()
 	s.packageCollection.Update(s.p1)
@@ -339,7 +337,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 
 	s.localRepo.DefaultDistribution = "precise"
 	s.localRepo.DefaultComponent = "contrib"
-	s.factory.LocalRepoCollection().Update(s.localRepo, s.reflistCollection)
+	s.factory.LocalRepoCollection().Update(s.localRepo)
 
 	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, IsNil)
@@ -540,7 +538,6 @@ type PublishedRepoCollectionSuite struct {
 	db                                database.Storage
 	factory                           *CollectionFactory
 	snapshotCollection                *SnapshotCollection
-	reflistCollection                 *RefListCollection
 	collection                        *PublishedRepoCollection
 	snap1, snap2                      *Snapshot
 	localRepo                         *LocalRepo
@@ -556,23 +553,22 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
-	s.reflistCollection = s.factory.RefListCollection()
 
 	snap1Refs := NewPackageRefList()
 	snap1Refs.Refs = [][]byte{s.p1.Key(""), s.p2.Key("")}
 	sort.Sort(snap1Refs)
-	s.snap1 = NewSnapshotFromRefList("snap1", []*Snapshot{}, NewSplitRefListFromRefList(snap1Refs), "desc1")
+	s.snap1 = NewSnapshotFromRefList("snap1", []*Snapshot{}, snap1Refs, "desc1")
 
 	snap2Refs := NewPackageRefList()
 	snap2Refs.Refs = [][]byte{s.p3.Key("")}
 	sort.Sort(snap2Refs)
-	s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, NewSplitRefListFromRefList(snap2Refs), "desc2")
+	s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2")
 
-	s.snapshotCollection.Add(s.snap1, s.reflistCollection)
-	s.snapshotCollection.Add(s.snap2, s.reflistCollection)
+	s.snapshotCollection.Add(s.snap1)
+	s.snapshotCollection.Add(s.snap2)
 
 	s.localRepo = NewLocalRepo("local1", "comment1")
-	s.factory.LocalRepoCollection().Add(s.localRepo, s.reflistCollection)
+	s.factory.LocalRepoCollection().Add(s.localRepo)
 
 	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
 	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false)
@@ -591,14 +587,14 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	_, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	c.Assert(s.collection.Add(s.repo1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.repo1, s.reflistCollection), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.Add(s.repo1), IsNil)
+	c.Assert(s.collection.Add(s.repo1), ErrorMatches, ".*already exists")
 	c.Assert(s.collection.CheckDuplicate(s.repo2), IsNil)
-	c.Assert(s.collection.Add(s.repo2, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.repo3, s.reflistCollection), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.Add(s.repo2), IsNil)
+	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
 	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
-	c.Assert(s.collection.Add(s.repo4, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.repo5, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.repo4), IsNil)
+	c.Assert(s.collection.Add(s.repo5), IsNil)
 
 	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
@@ -624,7 +620,7 @@ func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	c.Assert(s.collection.Add(s.repo1, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.repo1), IsNil)
 
 	r, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, IsNil)
@@ -635,8 +631,8 @@ func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	c.Assert(s.collection.Update(s.repo1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Update(s.repo4, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Update(s.repo1), IsNil)
+	c.Assert(s.collection.Update(s.repo4), IsNil)
 
 	collection := NewPublishedRepoCollection(s.db)
 	r, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -684,7 +680,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 	encoder.Encode(&old)
 
 	c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
-	c.Assert(s.db.Put(s.repo1.RefKey(""), NewPackageRefList().Encode()), IsNil)
+	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
 
 	collection := NewPublishedRepoCollection(s.db)
 	repo, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -699,7 +695,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
-	s.collection.Add(s.repo1, s.reflistCollection)
+	s.collection.Add(s.repo1)
 
 	count := 0
 	err := s.collection.ForEach(func(*PublishedRepo) error {
@@ -720,17 +716,17 @@ func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
-	c.Check(s.collection.Add(s.repo1, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo2, s.reflistCollection), IsNil)
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
 
 	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1, s.repo2})
 	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
 }
 
 func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
-	c.Check(s.collection.Add(s.repo1, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo4, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo5, s.reflistCollection), IsNil)
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
 
 	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
 }
@@ -740,10 +736,10 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 	c.Check(s.factory.PackageCollection().Update(s.p2), IsNil)
 	c.Check(s.factory.PackageCollection().Update(s.p3), IsNil)
 
-	c.Check(s.collection.Add(s.repo1, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo2, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo4, s.reflistCollection), IsNil)
-	c.Check(s.collection.Add(s.repo5, s.reflistCollection), IsNil)
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
+	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
 
 	files, err := s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
 	c.Assert(err, IsNil)
@@ -759,12 +755,12 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 	})
 
 	snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
-	s.snapshotCollection.Add(snap3, s.reflistCollection)
+	s.snapshotCollection.Add(snap3)
 
 	// Ensure that adding a second publish point with matching files doesn't give duplicate results.
 	repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
 	c.Check(err, IsNil)
-	c.Check(s.collection.Add(repo3, s.reflistCollection), IsNil)
+	c.Check(s.collection.Add(repo3), IsNil)
 
 	files, err = s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
 	c.Assert(err, IsNil)
@@ -785,7 +781,6 @@ type PublishedRepoRemoveSuite struct {
 	db                                  database.Storage
 	factory                             *CollectionFactory
 	snapshotCollection                  *SnapshotCollection
-	reflistCollection                   *RefListCollection
 	collection                          *PublishedRepoCollection
 	root, root2                         string
 	provider                            *FakeStorageProvider
@@ -801,11 +796,10 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
-	s.reflistCollection = s.factory.RefListCollection()
 
 	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
 
-	s.snapshotCollection.Add(s.snap1, s.reflistCollection)
+	s.snapshotCollection.Add(s.snap1)
 
 	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
 	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
@@ -814,11 +808,11 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
 
 	s.collection = s.factory.PublishedRepoCollection()
-	s.collection.Add(s.repo1, s.reflistCollection)
-	s.collection.Add(s.repo2, s.reflistCollection)
-	s.collection.Add(s.repo3, s.reflistCollection)
-	s.collection.Add(s.repo4, s.reflistCollection)
-	s.collection.Add(s.repo5, s.reflistCollection)
+	s.collection.Add(s.repo1)
+	s.collection.Add(s.repo2)
+	s.collection.Add(s.repo3)
+	s.collection.Add(s.repo4)
+	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")

deb/reflist.go

@@ -2,15 +2,10 @@ package deb
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"sort"
 
 	"github.com/AlekSi/pointer"
-	"github.com/aptly-dev/aptly/database"
-	"github.com/cespare/xxhash/v2"
 	"github.com/ugorji/go/codec"
 )
 
@@ -49,13 +44,6 @@ func NewPackageRefListFromPackageList(list *PackageList) *PackageRefList {
 	return reflist
 }
 
-func (l *PackageRefList) Clone() *PackageRefList {
-	clone := &PackageRefList{}
-	clone.Refs = make([][]byte, l.Len())
-	copy(clone.Refs, l.Refs)
-	return clone
-}
-
 // Len returns number of refs
 func (l *PackageRefList) Len() int {
 	return len(l.Refs)
@@ -196,12 +184,8 @@ func (d PackageDiff) MarshalJSON() ([]byte, error) {
 type PackageDiffs []PackageDiff
 
 // Diff calculates difference between two reflists
-func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection, result PackageDiffs) (PackageDiffs, error) {
-	var err error
-
-	if result == nil {
-		result = make(PackageDiffs, 0, 128)
-	}
+func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection) (result PackageDiffs, err error) {
+	result = make(PackageDiffs, 0, 128)
 
 	// pointer to left and right reflists
 	il, ir := 0, 0
@@ -274,7 +258,7 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 		}
 	}
 
-	return result, nil
+	return
 }
 
 // Merge merges reflist r into current reflist. If overrideMatching, merge
@@ -407,773 +391,3 @@ func (l *PackageRefList) FilterLatestRefs() {
 		lastArch, lastName, lastVer = arch, name, ver
 	}
 }
-
-const (
-	reflistBucketCount = 1 << 6
-	reflistBucketMask  = reflistBucketCount - 1
-)
-
-type reflistDigestArray [sha256.Size]byte
-
-func bucketRefPrefix(ref []byte) []byte {
-	const maxPrefixLen = 3
-
-	// Cut out the arch, leaving behind the package name and subsequent info.
-	_, ref, _ = bytes.Cut(ref, []byte{' '})
-
-	// Strip off the lib prefix, so that "libxyz" and "xyz", which are likely
-	// to be updated together, go in the same bucket.
-	libPrefix := []byte("lib")
-	if bytes.HasPrefix(ref, libPrefix) {
-		ref = ref[len(libPrefix):]
-	}
-
-	prefixLen := len(ref)
-	if maxPrefixLen < prefixLen {
-		prefixLen = maxPrefixLen
-	}
-	prefix, _, _ := bytes.Cut(ref[:prefixLen], []byte{' '})
-	return prefix
-}
-
-func bucketIdxForRef(ref []byte) int {
-	return int(xxhash.Sum64(bucketRefPrefix(ref))) & reflistBucketMask
-}
-
-// SplitRefList is a list of package refs, similar to a PackageRefList. However,
-// instead of storing a linear array of refs, SplitRefList splits the refs into
-// PackageRefList "buckets", based on a hash of the package name inside the ref.
-// Each bucket has a digest of its contents that serves as its key in the database.
-//
-// When serialized, a SplitRefList just becomes an array of bucket digests, and
-// the buckets themselves are stored separately. Because the buckets are then
-// referenced by their digests, multiple independent reflists can share buckets,
-// if their buckets have matching digests.
-//
-// Buckets themselves may not be confirmed to a single database value; instead,
-// they're split into "segments", based on the database's preferred maximum
-// value size. This prevents large buckets from slowing down the database.
-type SplitRefList struct {
-	Buckets [][]byte
-
-	bucketRefs []*PackageRefList
-}
-
-// NewSplitRefList creates empty SplitRefList
-func NewSplitRefList() *SplitRefList {
-	sl := &SplitRefList{}
-	sl.reset()
-	return sl
-}
-
-// NewSplitRefListFromRefList creates SplitRefList from PackageRefList
-func NewSplitRefListFromRefList(reflist *PackageRefList) *SplitRefList {
-	sl := NewSplitRefList()
-	sl.Replace(reflist)
-	return sl
-}
-
-// NewSplitRefListFromRefList creates SplitRefList from PackageList
-func NewSplitRefListFromPackageList(list *PackageList) *SplitRefList {
-	return NewSplitRefListFromRefList(NewPackageRefListFromPackageList(list))
-}
-
-func (sl *SplitRefList) reset() {
-	sl.Buckets = make([][]byte, reflistBucketCount)
-	sl.bucketRefs = make([]*PackageRefList, reflistBucketCount)
-}
-
-// Has checks whether package is part of reflist
-func (sl *SplitRefList) Has(p *Package) bool {
-	idx := bucketIdxForRef(p.Key(""))
-	if bucket := sl.bucketRefs[idx]; bucket != nil {
-		return bucket.Has(p)
-	}
-	return false
-}
-
-// Len returns number of refs
-func (sl *SplitRefList) Len() int {
-	total := 0
-	for _, bucket := range sl.bucketRefs {
-		if bucket != nil {
-			total += bucket.Len()
-		}
-	}
-	return total
-}
-
-func reflistDigest(l *PackageRefList) []byte {
-	// Different algorithms on PackageRefLists will sometimes return a nil slice
-	// of refs and other times return an empty slice. Regardless, they should
-	// both be treated identically and be given an empty digest.
-	if len(l.Refs) == 0 {
-		return nil
-	}
-
-	h := sha256.New()
-	for _, ref := range l.Refs {
-		h.Write(ref)
-		h.Write([]byte{0})
-	}
-	return h.Sum(nil)
-}
-
-// Removes all the refs inside and replaces them with those in the given reflist
-func (sl *SplitRefList) Replace(reflist *PackageRefList) {
-	sl.reset()
-
-	for _, ref := range reflist.Refs {
-		idx := bucketIdxForRef(ref)
-
-		bucket := sl.bucketRefs[idx]
-		if bucket == nil {
-			bucket = NewPackageRefList()
-			sl.bucketRefs[idx] = bucket
-		}
-
-		bucket.Refs = append(bucket.Refs, ref)
-	}
-
-	for idx, bucket := range sl.bucketRefs {
-		if bucket != nil {
-			sort.Sort(bucket)
-			sl.Buckets[idx] = reflistDigest(bucket)
-		}
-	}
-}
-
-// Merge merges reflist r into current reflist (see PackageRefList.Merge)
-func (sl *SplitRefList) Merge(r *SplitRefList, overrideMatching, ignoreConflicting bool) (result *SplitRefList) {
-	result = NewSplitRefList()
-
-	var empty PackageRefList
-	for idx, lbucket := range sl.bucketRefs {
-		rbucket := r.bucketRefs[idx]
-		if lbucket == nil && rbucket == nil {
-			continue
-		}
-
-		if lbucket == nil {
-			lbucket = &empty
-		} else if rbucket == nil {
-			rbucket = &empty
-		}
-
-		result.bucketRefs[idx] = lbucket.Merge(rbucket, overrideMatching, ignoreConflicting)
-		result.Buckets[idx] = reflistDigest(result.bucketRefs[idx])
-	}
-
-	return
-}
-
-// Subtract returns all packages in l that are not in r
-func (sl *SplitRefList) Subtract(r *SplitRefList) (result *SplitRefList) {
-	result = NewSplitRefList()
-
-	for idx, lbucket := range sl.bucketRefs {
-		rbucket := r.bucketRefs[idx]
-		if lbucket != nil {
-			if rbucket != nil {
-				result.bucketRefs[idx] = lbucket.Subtract(rbucket)
-				result.Buckets[idx] = reflistDigest(result.bucketRefs[idx])
-			} else {
-				result.bucketRefs[idx] = lbucket.Clone()
-				result.Buckets[idx] = sl.Buckets[idx]
-			}
-		}
-	}
-
-	return
-}
-
-// Diff calculates difference between two reflists
-func (sl *SplitRefList) Diff(r *SplitRefList, packageCollection *PackageCollection, result PackageDiffs) (PackageDiffs, error) {
-	var err error
-
-	if result == nil {
-		result = make(PackageDiffs, 0, 128)
-	}
-
-	var empty PackageRefList
-	for idx, lbucket := range sl.bucketRefs {
-		rbucket := r.bucketRefs[idx]
-		if lbucket != nil {
-			if rbucket != nil {
-				result, err = lbucket.Diff(rbucket, packageCollection, result)
-			} else {
-				result, err = lbucket.Diff(&empty, packageCollection, result)
-			}
-		} else if rbucket != nil {
-			result, err = empty.Diff(rbucket, packageCollection, result)
-		}
-
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	sort.Slice(result, func(i, j int) bool {
-		var ri, rj []byte
-		if result[i].Left != nil {
-			ri = result[i].Left.Key("")
-		} else {
-			ri = result[i].Right.Key("")
-		}
-		if result[j].Left != nil {
-			rj = result[j].Left.Key("")
-		} else {
-			rj = result[j].Right.Key("")
-		}
-
-		return bytes.Compare(ri, rj) < 0
-	})
-
-	return result, nil
-}
-
-// FilterLatestRefs reduces a reflist to the latest of each package (see PackageRefList.FilterLatestRefs)
-func (sl *SplitRefList) FilterLatestRefs() {
-	for idx, bucket := range sl.bucketRefs {
-		if bucket != nil {
-			bucket.FilterLatestRefs()
-			sl.Buckets[idx] = reflistDigest(bucket)
-		}
-	}
-}
-
-// Flatten creates a flat PackageRefList containing all the refs in this reflist
-func (sl *SplitRefList) Flatten() *PackageRefList {
-	reflist := NewPackageRefList()
-	sl.ForEach(func(ref []byte) error {
-		reflist.Refs = append(reflist.Refs, ref)
-		return nil
-	})
-	sort.Sort(reflist)
-	return reflist
-}
-
-// ForEachBucket calls handler for each bucket in list
-func (sl *SplitRefList) ForEachBucket(handler func(digest []byte, bucket *PackageRefList) error) error {
-	for idx, digest := range sl.Buckets {
-		if len(digest) == 0 {
-			continue
-		}
-
-		bucket := sl.bucketRefs[idx]
-		if bucket != nil {
-			if err := handler(digest, bucket); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// ForEach calls handler for each package ref in list
-//
-// IMPORTANT: unlike PackageRefList.ForEach, the order of handler invocations
-// is *not* guaranteed to be sorted.
-func (sl *SplitRefList) ForEach(handler func([]byte) error) error {
-	for idx, digest := range sl.Buckets {
-		if len(digest) == 0 {
-			continue
-		}
-
-		bucket := sl.bucketRefs[idx]
-		if bucket != nil {
-			if err := bucket.ForEach(handler); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// RefListDigestSet is a set of SplitRefList bucket digests
-type RefListDigestSet struct {
-	items map[reflistDigestArray]struct{}
-}
-
-// NewRefListDigestSet creates empty RefListDigestSet
-func NewRefListDigestSet() *RefListDigestSet {
-	return &RefListDigestSet{items: map[reflistDigestArray]struct{}{}}
-}
-
-// Len returns number of digests in the set
-func (set *RefListDigestSet) Len() int {
-	return len(set.items)
-}
-
-// ForEach calls handler for each digest in the set
-func (set *RefListDigestSet) ForEach(handler func(digest []byte) error) error {
-	for digest := range set.items {
-		if err := handler(digest[:]); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// workaround for: conversion of slices to arrays requires go1.20 or later
-func newRefListArray(digest []byte) reflistDigestArray {
-	var array reflistDigestArray
-	copy(array[:], digest)
-	return array
-}
-
-// Add adds digest to set, doing nothing if the digest was already present
-func (set *RefListDigestSet) Add(digest []byte) {
-	set.items[newRefListArray(digest)] = struct{}{}
-}
-
-// AddAllInRefList adds all the bucket digests in a SplitRefList to the set
-func (set *RefListDigestSet) AddAllInRefList(sl *SplitRefList) {
-	for _, digest := range sl.Buckets {
-		if len(digest) > 0 {
-			set.Add(digest)
-		}
-	}
-}
-
-// Has checks whether a digest is part of set
-func (set *RefListDigestSet) Has(digest []byte) bool {
-	_, ok := set.items[newRefListArray(digest)]
-	return ok
-}
-
-// Remove removes a digest from set
-func (set *RefListDigestSet) Remove(digest []byte) {
-	delete(set.items, newRefListArray(digest))
-}
-
-// RemoveAll removes all the digests in other from the current set
-func (set *RefListDigestSet) RemoveAll(other *RefListDigestSet) {
-	for digest := range other.items {
-		delete(set.items, digest)
-	}
-}
-
-// RefListCollection does listing, updating/adding/deleting of SplitRefLists
-type RefListCollection struct {
-	db database.Storage
-
-	cache map[reflistDigestArray]*PackageRefList
-}
-
-// NewRefListCollection creates a RefListCollection
-func NewRefListCollection(db database.Storage) *RefListCollection {
-	return &RefListCollection{db: db, cache: make(map[reflistDigestArray]*PackageRefList)}
-}
-
-type reflistStorageFormat int
-
-const (
-	// (legacy format) all the refs are stored inline in a single value
-	reflistStorageFormatInline reflistStorageFormat = iota
-	// the refs are split into buckets that are stored externally from the value
-	reflistStorageFormatSplit
-)
-
-// NoPadding is used because all digests are the same length, so the padding
-// is useless and only serves to muddy the output.
-var bucketDigestEncoding = base64.StdEncoding.WithPadding(base64.NoPadding)
-
-func segmentPrefix(encodedDigest string) []byte {
-	return []byte(fmt.Sprintf("F%s-", encodedDigest))
-}
-
-// workaround for go 1.19 instead of bytes.Clone
-func cloneBytes(b []byte) []byte {
-	if b == nil {
-		return nil
-	}
-	cloned := make([]byte, len(b))
-	copy(cloned, b)
-	return cloned
-}
-
-func segmentIndexKey(prefix []byte, idx int) []byte {
-	// Assume most buckets won't have more than 0xFFFF = ~65k segments (which
-	// would be an extremely large bucket!).
-	return append(cloneBytes(prefix), []byte(fmt.Sprintf("%04x", idx))...)
-}
-
-// AllBucketDigests returns a set of all the bucket digests in the database
-func (collection *RefListCollection) AllBucketDigests() (*RefListDigestSet, error) {
-	digests := NewRefListDigestSet()
-
-	err := collection.db.ProcessByPrefix([]byte("F"), func(key []byte, _ []byte) error {
-		if !bytes.HasSuffix(key, []byte("-0000")) {
-			// Ignore additional segments for the same digest.
-			return nil
-		}
-
-		encodedDigest, _, foundDash := bytes.Cut(key[1:], []byte("-"))
-		if !foundDash {
-			return fmt.Errorf("invalid key: %s", string(key))
-		}
-		digest := make([]byte, bucketDigestEncoding.DecodedLen(len(encodedDigest)))
-		if _, err := bucketDigestEncoding.Decode(digest, encodedDigest); err != nil {
-			return fmt.Errorf("decoding key %s: %w", string(key), err)
-		}
-
-		digests.Add(digest)
-		return nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-	return digests, nil
-}
-
-// UnsafeDropBucket drops the bucket associated with digest from the database,
-// doing so inside batch
-//
-// This is considered "unsafe" because no checks are performed to ensure that
-// the bucket is no longer referenced by any saved reflists.
-func (collection *RefListCollection) UnsafeDropBucket(digest []byte, batch database.Batch) error {
-	prefix := segmentPrefix(bucketDigestEncoding.EncodeToString(digest))
-	return collection.db.ProcessByPrefix(prefix, func(key []byte, _ []byte) error {
-		return batch.Delete(key)
-	})
-}
-
-func (collection *RefListCollection) load(sl *SplitRefList, key []byte) (reflistStorageFormat, error) {
-	sl.reset()
-
-	data, err := collection.db.Get(key)
-	if err != nil {
-		return 0, err
-	}
-
-	var splitOrInlineRefList struct {
-		*SplitRefList
-		*PackageRefList
-	}
-	handle := &codec.MsgpackHandle{}
-	handle.ZeroCopy = true
-	decoder := codec.NewDecoderBytes(data, handle)
-	if err := decoder.Decode(&splitOrInlineRefList); err != nil {
-		return 0, err
-	}
-
-	if splitOrInlineRefList.SplitRefList != nil {
-		sl.Buckets = splitOrInlineRefList.Buckets
-	} else if splitOrInlineRefList.PackageRefList != nil {
-		sl.Replace(splitOrInlineRefList.PackageRefList)
-		return reflistStorageFormatInline, nil
-	}
-
-	return reflistStorageFormatSplit, nil
-}
-
-func (collection *RefListCollection) loadBuckets(sl *SplitRefList) error {
-	for idx := range sl.Buckets {
-		if sl.bucketRefs[idx] != nil {
-			continue
-		}
-
-		var bucket *PackageRefList
-
-		if digest := sl.Buckets[idx]; len(digest) > 0 {
-			cacheKey := newRefListArray(digest)
-			bucket = collection.cache[cacheKey]
-			if bucket == nil {
-				bucket = NewPackageRefList()
-				prefix := segmentPrefix(bucketDigestEncoding.EncodeToString(digest))
-				err := collection.db.ProcessByPrefix(prefix, func(_ []byte, value []byte) error {
-					var l PackageRefList
-					if err := l.Decode(append([]byte{}, value...)); err != nil {
-						return err
-					}
-
-					bucket.Refs = append(bucket.Refs, l.Refs...)
-					return nil
-				})
-
-				if err != nil {
-					return err
-				}
-
-				// The segments may not have been iterated in order, so make sure to re-sort
-				// here.
-				sort.Sort(bucket)
-				collection.cache[cacheKey] = bucket
-			}
-
-			actualDigest := reflistDigest(bucket)
-			if !bytes.Equal(actualDigest, digest) {
-				return fmt.Errorf("corrupt reflist bucket %d: expected digest %s, got %s",
-					idx,
-					bucketDigestEncoding.EncodeToString(digest),
-					bucketDigestEncoding.EncodeToString(actualDigest))
-			}
-		}
-
-		sl.bucketRefs[idx] = bucket
-	}
-
-	return nil
-}
-
-// LoadComplete loads the reflist stored at the given key, as well as all the
-// buckets referenced by a split reflist
-func (collection *RefListCollection) LoadComplete(sl *SplitRefList, key []byte) error {
-	if _, err := collection.load(sl, key); err != nil {
-		return err
-	}
-
-	return collection.loadBuckets(sl)
-}
-
-// RefListBatch is a wrapper over a database.Batch that tracks already-written
-// reflists to avoid writing them multiple times
-//
-// It is *not* safe to use the same underlying database.Batch that has already
-// been given to UnsafeDropBucket.
-type RefListBatch struct {
-	batch database.Batch
-
-	alreadyWritten *RefListDigestSet
-}
-
-// NewBatch creates a new RefListBatch wrapping the given database.Batch
-func (collection *RefListCollection) NewBatch(batch database.Batch) *RefListBatch {
-	return &RefListBatch{
-		batch:          batch,
-		alreadyWritten: NewRefListDigestSet(),
-	}
-}
-
-type reflistUpdateContext struct {
-	rb    *RefListBatch
-	stats *RefListMigrationStats
-}
-
-func clearSegmentRefs(reflist *PackageRefList, recommendedMaxKVSize int) {
-	avgRefsInSegment := recommendedMaxKVSize / 70
-	reflist.Refs = make([][]byte, 0, avgRefsInSegment)
-}
-
-func flushSegmentRefs(uctx *reflistUpdateContext, prefix []byte, segment int, reflist *PackageRefList) error {
-	encoded := reflist.Encode()
-	err := uctx.rb.batch.Put(segmentIndexKey(prefix, segment), encoded)
-	if err == nil && uctx.stats != nil {
-		uctx.stats.Segments++
-	}
-	return err
-}
-
-func (collection *RefListCollection) updateWithContext(sl *SplitRefList, key []byte, uctx *reflistUpdateContext) error {
-	if sl != nil {
-		recommendedMaxKVSize := collection.db.GetRecommendedMaxKVSize()
-
-		for idx, digest := range sl.Buckets {
-			if len(digest) == 0 {
-				continue
-			}
-
-			if uctx.rb.alreadyWritten.Has(digest) {
-				continue
-			}
-
-			prefix := segmentPrefix(bucketDigestEncoding.EncodeToString(digest))
-			if collection.db.HasPrefix(prefix) {
-				continue
-			}
-
-			// All the sizing information taken from the msgpack spec:
-			// https://github.com/msgpack/msgpack/blob/master/spec.md
-
-			// Assume that a segment will have [16,2^16) elements, which would
-			// fit into an array 16 and thus have 3 bytes of overhead.
-			// (A database would need a massive recommendedMaxKVSize to pass
-			// that limit.)
-			size := len(segmentIndexKey(prefix, 0)) + 3
-			segment := 0
-
-			var reflist PackageRefList
-			clearSegmentRefs(&reflist, recommendedMaxKVSize)
-			for _, ref := range sl.bucketRefs[idx].Refs {
-				// In order to determine the size of the ref in the database,
-				// we need to know how much overhead will be added with by msgpack
-				// encoding.
-				requiredSize := len(ref)
-				if requiredSize < 1<<5 {
-					requiredSize++
-				} else if requiredSize < 1<<8 {
-					requiredSize += 2
-				} else if requiredSize < 1<<16 {
-					requiredSize += 3
-				} else {
-					requiredSize += 4
-				}
-				if size+requiredSize > recommendedMaxKVSize {
-					if err := flushSegmentRefs(uctx, prefix, segment, &reflist); err != nil {
-						return err
-					}
-					clearSegmentRefs(&reflist, recommendedMaxKVSize)
-					segment++
-				}
-
-				reflist.Refs = append(reflist.Refs, ref)
-				size += requiredSize
-			}
-
-			if len(reflist.Refs) > 0 {
-				if err := flushSegmentRefs(uctx, prefix, segment, &reflist); err != nil {
-					return err
-				}
-			}
-
-			uctx.rb.alreadyWritten.Add(digest)
-			if uctx.stats != nil {
-				uctx.stats.Buckets++
-			}
-		}
-	}
-
-	var buf bytes.Buffer
-	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(sl)
-	err := uctx.rb.batch.Put(key, buf.Bytes())
-	if err == nil && uctx.stats != nil {
-		uctx.stats.Reflists++
-	}
-	return err
-}
-
-// UpdateInBatch will save or update the SplitRefList at key, as well as save the buckets inside,
-// as part of the given batch
-func (collection *RefListCollection) UpdateInBatch(sl *SplitRefList, key []byte, batch *RefListBatch) error {
-	return collection.updateWithContext(sl, key, &reflistUpdateContext{rb: batch})
-}
-
-// Update will save or update the SplitRefList at key, as well as save the buckets inside
-func (collection *RefListCollection) Update(sl *SplitRefList, key []byte) error {
-	rb := collection.NewBatch(collection.db.CreateBatch())
-	err := collection.UpdateInBatch(sl, key, rb)
-	if err == nil {
-		err = rb.batch.Write()
-	}
-	return err
-}
-
-// RefListMigrationStats counts a number of reflists, buckets, and segments
-type RefListMigrationStats struct {
-	Reflists, Buckets, Segments int
-}
-
-// RefListMigration wraps a RefListBatch for the purpose of migrating inline format
-// reflists to split reflists
-//
-// Once the batch gets too large, it will automatically be flushed to the database,
-// and a new batch will be created in its place.
-type RefListMigration struct {
-	rb *RefListBatch
-
-	dryRun bool
-
-	// current number of reflists/buckets/segments queued in the current, unwritten batch
-	batchStats RefListMigrationStats
-	flushStats RefListMigrationStats
-}
-
-// NewMigration creates an empty RefListMigration
-func (collection *RefListCollection) NewMigration() *RefListMigration {
-	return &RefListMigration{}
-}
-
-// NewMigrationDryRun creates an empty RefListMigration that will track the
-// changes to make as usual but avoid actually writing to the db
-func (collection *RefListCollection) NewMigrationDryRun() *RefListMigration {
-	return &RefListMigration{dryRun: true}
-}
-
-// Stats returns statistics on the written values in the current migration
-func (migration *RefListMigration) Stats() RefListMigrationStats {
-	return migration.flushStats
-}
-
-// Flush will flush the current batch in the migration to the database
-func (migration *RefListMigration) Flush() error {
-	if migration.batchStats.Segments > 0 {
-		if !migration.dryRun {
-			if err := migration.rb.batch.Write(); err != nil {
-				return err
-			}
-
-			// It's important that we don't clear the batch on dry runs, because
-			// the batch is what contains the list of already-written buckets.
-			// If we're not writing to the database, and we clear that list,
-			// duplicate "writes" will occur.
-			migration.rb = nil
-		}
-
-		migration.flushStats.Reflists += migration.batchStats.Reflists
-		migration.flushStats.Buckets += migration.batchStats.Buckets
-		migration.flushStats.Segments += migration.batchStats.Segments
-		migration.batchStats = RefListMigrationStats{}
-	}
-
-	return nil
-}
-
-// LoadCompleteAndMigrate will load the reflist and its buckets as RefListCollection.LoadComplete,
-// migrating any inline reflists to split ones along the way
-func (collection *RefListCollection) LoadCompleteAndMigrate(sl *SplitRefList, key []byte, migration *RefListMigration) error {
-	// Given enough reflists, the memory used by a batch starts to become massive, so
-	// make sure to flush the written segments periodically. Note that this is only
-	// checked *after* a migration of a full bucket (and all the segments inside)
-	// takes place, as splitting a single bucket write into multiple batches would
-	// be unsafe if an interruption occurs midway.
-	const maxMigratorBatch = 50000
-
-	format, err := collection.load(sl, key)
-	if err != nil {
-		return err
-	}
-
-	switch format {
-	case reflistStorageFormatInline:
-		if migration.rb == nil {
-			migration.rb = collection.NewBatch(collection.db.CreateBatch())
-		}
-
-		collection.updateWithContext(sl, key, &reflistUpdateContext{
-			rb:    migration.rb,
-			stats: &migration.batchStats,
-		})
-
-		if migration.batchStats.Segments > maxMigratorBatch {
-			if err := migration.Flush(); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	case reflistStorageFormatSplit:
-		return collection.loadBuckets(sl)
-	default:
-		panic(fmt.Sprintf("unexpected format %v", format))
-	}
-}
-
-// AnyRefList is implemented by both PackageRefList and SplitRefList
-type AnyRefList interface {
-	Has(p *Package) bool
-	Len() int
-	ForEach(handler func([]byte) error) error
-	FilterLatestRefs()
-}
-
-// Check interface
-var (
-	_ AnyRefList = (*PackageRefList)(nil)
-	_ AnyRefList = (*SplitRefList)(nil)
-)

deb/reflist_bench_test.go

@@ -45,41 +45,3 @@ func BenchmarkReflistDecode(b *testing.B) {
 		(&PackageRefList{}).Decode(data)
 	}
 }
-
-func BenchmarkSplitRefListCreationSmall(b *testing.B) {
-	const count = 400
-
-	l := NewPackageRefList()
-
-	for i := 0; i < count; i++ {
-		l.Refs = append(l.Refs, []byte(fmt.Sprintf("Pamd64 %x %d", i, i)))
-	}
-
-	sort.Sort(l)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for j := 0; j < 8; j++ {
-			NewSplitRefListFromRefList(l)
-		}
-	}
-}
-
-func BenchmarkSplitRefListCreationLarge(b *testing.B) {
-	const count = 4096
-
-	l := NewPackageRefList()
-
-	for i := 0; i < count; i++ {
-		l.Refs = append(l.Refs, []byte(fmt.Sprintf("Pamd64 %x %d", i, i)))
-	}
-
-	sort.Sort(l)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for j := 0; j < 8; j++ {
-			NewSplitRefListFromRefList(l)
-		}
-	}
-}

deb/reflist_test.go

@@ -1,10 +1,7 @@
 package deb
 
 import (
-	"bytes"
-	"encoding/hex"
 	"errors"
-	"fmt"
 
 	"github.com/aptly-dev/aptly/database/goleveldb"
 
@@ -12,83 +9,24 @@ import (
 )
 
 type PackageRefListSuite struct {
+	// Simple list with "real" packages from stanzas
+	list                   *PackageList
 	p1, p2, p3, p4, p5, p6 *Package
 }
 
 var _ = Suite(&PackageRefListSuite{})
 
-func verifyRefListIntegrity(c *C, rl AnyRefList) AnyRefList {
-	if rl, ok := rl.(*SplitRefList); ok {
-		for idx, bucket := range rl.bucketRefs {
-			if bucket == nil {
-				bucket = NewPackageRefList()
-			}
-			c.Check(rl.Buckets[idx], DeepEquals, reflistDigest(bucket))
-		}
-	}
-
-	return rl
-}
-
-func getRefs(rl AnyRefList) (refs [][]byte) {
-	switch rl := rl.(type) {
-	case *PackageRefList:
-		refs = rl.Refs
-	case *SplitRefList:
-		refs = rl.Flatten().Refs
-	default:
-		panic(fmt.Sprintf("unexpected reflist type %t", rl))
-	}
-
-	// Hack so that passing getRefs-returned slices to DeepEquals won't fail given a nil
-	// slice and an empty slice.
-	if len(refs) == 0 {
-		refs = nil
-	}
-	return
-}
-
-func toStrSlice(reflist AnyRefList) (result []string) {
+func toStrSlice(reflist *PackageRefList) (result []string) {
 	result = make([]string, reflist.Len())
-	for i, r := range getRefs(reflist) {
+	for i, r := range reflist.Refs {
 		result[i] = string(r)
 	}
 	return
 }
 
-type reflistFactory struct {
-	new                func() AnyRefList
-	newFromRefs        func(refs ...[]byte) AnyRefList
-	newFromPackageList func(list *PackageList) AnyRefList
-}
-
-func forEachRefList(test func(f reflistFactory)) {
-	test(reflistFactory{
-		new: func() AnyRefList {
-			return NewPackageRefList()
-		},
-		newFromRefs: func(refs ...[]byte) AnyRefList {
-			return &PackageRefList{Refs: refs}
-		},
-		newFromPackageList: func(list *PackageList) AnyRefList {
-			return NewPackageRefListFromPackageList(list)
-		},
-	})
-
-	test(reflistFactory{
-		new: func() AnyRefList {
-			return NewSplitRefList()
-		},
-		newFromRefs: func(refs ...[]byte) AnyRefList {
-			return NewSplitRefListFromRefList(&PackageRefList{Refs: refs})
-		},
-		newFromPackageList: func(list *PackageList) AnyRefList {
-			return NewSplitRefListFromPackageList(list)
-		},
-	})
-}
-
 func (s *PackageRefListSuite) SetUpTest(c *C) {
+	s.list = NewPackageList()
+
 	s.p1 = NewPackageFromControlFile(packageStanza.Copy())
 	s.p2 = NewPackageFromControlFile(packageStanza.Copy())
 	stanza := packageStanza.Copy()
@@ -106,413 +44,58 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		list := NewPackageList()
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+	coll.Update(s.p1)
+	coll.Update(s.p3)
 
-		db, _ := goleveldb.NewOpenDB(c.MkDir())
-		coll := NewPackageCollection(db)
-		coll.Update(s.p1)
-		coll.Update(s.p3)
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
 
-		list.Add(s.p1)
-		list.Add(s.p3)
-		list.Add(s.p5)
-		list.Add(s.p6)
+	reflist := NewPackageRefListFromPackageList(s.list)
 
-		reflist := f.newFromPackageList(list)
+	_, err := NewPackageListFromRefList(reflist, coll, nil)
+	c.Assert(err, ErrorMatches, "unable to load package with key.*")
 
-		_, err := NewPackageListFromRefList(reflist, coll, nil)
-		c.Assert(err, ErrorMatches, "unable to load package with key.*")
+	coll.Update(s.p5)
+	coll.Update(s.p6)
 
-		coll.Update(s.p5)
-		coll.Update(s.p6)
+	list, err := NewPackageListFromRefList(reflist, coll, nil)
+	c.Assert(err, IsNil)
+	c.Check(list.Len(), Equals, 4)
+        c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
 
-		list, err = NewPackageListFromRefList(reflist, coll, nil)
-		c.Assert(err, IsNil)
-		c.Check(list.Len(), Equals, 4)
-		c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
-
-		list, err = NewPackageListFromRefList(nil, coll, nil)
-		c.Assert(err, IsNil)
-		c.Check(list.Len(), Equals, 0)
-	})
+	list, err = NewPackageListFromRefList(nil, coll, nil)
+	c.Assert(err, IsNil)
+	c.Check(list.Len(), Equals, 0)
 }
 
 func (s *PackageRefListSuite) TestNewPackageRefList(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		list := NewPackageList()
-		list.Add(s.p1)
-		list.Add(s.p3)
-		list.Add(s.p5)
-		list.Add(s.p6)
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
 
-		reflist := f.newFromPackageList(list)
-		verifyRefListIntegrity(c, reflist)
-		c.Assert(reflist.Len(), Equals, 4)
-		refs := getRefs(reflist)
-		c.Check(refs[0], DeepEquals, []byte(s.p1.Key("")))
-		c.Check(refs[1], DeepEquals, []byte(s.p6.Key("")))
-		c.Check(refs[2], DeepEquals, []byte(s.p5.Key("")))
-		c.Check(refs[3], DeepEquals, []byte(s.p3.Key("")))
+	reflist := NewPackageRefListFromPackageList(s.list)
+	c.Assert(reflist.Len(), Equals, 4)
+	c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key("")))
+	c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key("")))
+	c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key("")))
+	c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key("")))
 
-		reflist = f.new()
-		c.Check(reflist.Len(), Equals, 0)
-	})
-}
-
-func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		list := NewPackageList()
-		list.Add(s.p1)
-		list.Add(s.p3)
-		list.Add(s.p5)
-		list.Add(s.p6)
-
-		reflist := f.newFromPackageList(list)
-
-		Len := 0
-		err := reflist.ForEach(func([]byte) error {
-			Len++
-			return nil
-		})
-
-		c.Check(Len, Equals, 4)
-		c.Check(err, IsNil)
-
-		e := errors.New("b")
-
-		err = reflist.ForEach(func([]byte) error {
-			return e
-		})
-
-		c.Check(err, Equals, e)
-	})
-}
-
-func (s *PackageRefListSuite) TestHas(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		list := NewPackageList()
-		list.Add(s.p1)
-		list.Add(s.p3)
-		list.Add(s.p5)
-		reflist := f.newFromPackageList(list)
-
-		c.Check(reflist.Has(s.p1), Equals, true)
-		c.Check(reflist.Has(s.p3), Equals, true)
-		c.Check(reflist.Has(s.p5), Equals, true)
-		c.Check(reflist.Has(s.p2), Equals, true)
-		c.Check(reflist.Has(s.p6), Equals, false)
-	})
-}
-
-func subtractRefLists(l, r AnyRefList) AnyRefList {
-	switch l := l.(type) {
-	case *PackageRefList:
-		return l.Subtract(r.(*PackageRefList))
-	case *SplitRefList:
-		return l.Subtract(r.(*SplitRefList))
-	default:
-		panic(fmt.Sprintf("unexpected reflist type %t", l))
-	}
-}
-
-func (s *PackageRefListSuite) TestSubtract(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		r1 := []byte("Pall r1")
-		r2 := []byte("Pall r2")
-		r3 := []byte("Pall r3")
-		r4 := []byte("Pall r4")
-		r5 := []byte("Pall r5")
-
-		empty := f.newFromRefs()
-		l1 := f.newFromRefs(r1, r2, r3, r4)
-		l2 := f.newFromRefs(r1, r3)
-		l3 := f.newFromRefs(r2, r4)
-		l4 := f.newFromRefs(r4, r5)
-		l5 := f.newFromRefs(r1, r2, r3)
-
-		c.Check(getRefs(subtractRefLists(l1, empty)), DeepEquals, getRefs(l1))
-		c.Check(getRefs(subtractRefLists(l1, l2)), DeepEquals, getRefs(l3))
-		c.Check(getRefs(subtractRefLists(l1, l3)), DeepEquals, getRefs(l2))
-		c.Check(getRefs(subtractRefLists(l1, l4)), DeepEquals, getRefs(l5))
-		c.Check(getRefs(subtractRefLists(empty, l1)), DeepEquals, getRefs(empty))
-		c.Check(getRefs(subtractRefLists(l2, l3)), DeepEquals, getRefs(l2))
-	})
-}
-
-func diffRefLists(l, r AnyRefList, packageCollection *PackageCollection) (PackageDiffs, error) {
-	switch l := l.(type) {
-	case *PackageRefList:
-		return l.Diff(r.(*PackageRefList), packageCollection, nil)
-	case *SplitRefList:
-		return l.Diff(r.(*SplitRefList), packageCollection, nil)
-	default:
-		panic(fmt.Sprintf("unexpected reflist type %t", l))
-	}
-}
-
-func (s *PackageRefListSuite) TestDiff(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		db, _ := goleveldb.NewOpenDB(c.MkDir())
-		coll := NewPackageCollection(db)
-
-		packages := []*Package{
-			{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
-			{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
-			{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
-			{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
-			{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
-			{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
-			{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
-		}
-
-		for _, p := range packages {
-			coll.Update(p)
-		}
-
-		listA := NewPackageList()
-		listA.Add(packages[0])
-		listA.Add(packages[1])
-		listA.Add(packages[2])
-		listA.Add(packages[3])
-		listA.Add(packages[6])
-
-		listB := NewPackageList()
-		listB.Add(packages[0])
-		listB.Add(packages[2])
-		listB.Add(packages[4])
-		listB.Add(packages[5])
-
-		reflistA := f.newFromPackageList(listA)
-		reflistB := f.newFromPackageList(listB)
-
-		diffAA, err := diffRefLists(reflistA, reflistA, coll)
-		c.Check(err, IsNil)
-		c.Check(diffAA, HasLen, 0)
-
-		diffAB, err := diffRefLists(reflistA, reflistB, coll)
-		c.Check(err, IsNil)
-		c.Check(diffAB, HasLen, 4)
-
-		c.Check(diffAB[0].Left, IsNil)
-		c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
-
-		c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
-		c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
-
-		c.Check(diffAB[2].Left.String(), Equals, "dpkg_1.7_i386")
-		c.Check(diffAB[2].Right, IsNil)
-
-		c.Check(diffAB[3].Left.String(), Equals, "xyz_3.0_sparc")
-		c.Check(diffAB[3].Right, IsNil)
-
-		diffBA, err := diffRefLists(reflistB, reflistA, coll)
-		c.Check(err, IsNil)
-		c.Check(diffBA, HasLen, 4)
-
-		c.Check(diffBA[0].Right, IsNil)
-		c.Check(diffBA[0].Left.String(), Equals, "app_1.1~bp2_amd64")
-
-		c.Check(diffBA[1].Right.String(), Equals, "app_1.1~bp1_i386")
-		c.Check(diffBA[1].Left.String(), Equals, "app_1.1~bp2_i386")
-
-		c.Check(diffBA[2].Right.String(), Equals, "dpkg_1.7_i386")
-		c.Check(diffBA[2].Left, IsNil)
-
-		c.Check(diffBA[3].Right.String(), Equals, "xyz_3.0_sparc")
-		c.Check(diffBA[3].Left, IsNil)
-	})
-}
-
-func (s *PackageRefListSuite) TestDiffCompactsAtEnd(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		db, _ := goleveldb.NewOpenDB(c.MkDir())
-		coll := NewPackageCollection(db)
-
-		packages := []*Package{
-			{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //0
-			{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //1
-			{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //2
-		}
-
-		for _, p := range packages {
-			coll.Update(p)
-		}
-
-		listA := NewPackageList()
-		listA.Add(packages[0])
-
-		listB := NewPackageList()
-		listB.Add(packages[1])
-		listB.Add(packages[2])
-
-		reflistA := f.newFromPackageList(listA)
-		reflistB := f.newFromPackageList(listB)
-
-		diffAB, err := diffRefLists(reflistA, reflistB, coll)
-		c.Check(err, IsNil)
-		c.Check(diffAB, HasLen, 2)
-
-		c.Check(diffAB[0].Left, IsNil)
-		c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
-
-		c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
-		c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
-	})
-}
-
-func mergeRefLists(l, r AnyRefList, overrideMatching, ignoreConflicting bool) AnyRefList {
-	switch l := l.(type) {
-	case *PackageRefList:
-		return l.Merge(r.(*PackageRefList), overrideMatching, ignoreConflicting)
-	case *SplitRefList:
-		return l.Merge(r.(*SplitRefList), overrideMatching, ignoreConflicting)
-	default:
-		panic(fmt.Sprintf("unexpected reflist type %t", l))
-	}
-}
-
-func (s *PackageRefListSuite) TestMerge(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		db, _ := goleveldb.NewOpenDB(c.MkDir())
-		coll := NewPackageCollection(db)
-
-		packages := []*Package{
-			{Name: "lib", Version: "1.0", Architecture: "i386"},                      //0
-			{Name: "dpkg", Version: "1.7", Architecture: "i386"},                     //1
-			{Name: "data", Version: "1.1~bp1", Architecture: "all"},                  //2
-			{Name: "app", Version: "1.1~bp1", Architecture: "i386"},                  //3
-			{Name: "app", Version: "1.1~bp2", Architecture: "i386"},                  //4
-			{Name: "app", Version: "1.1~bp2", Architecture: "amd64"},                 //5
-			{Name: "dpkg", Version: "1.0", Architecture: "i386"},                     //6
-			{Name: "xyz", Version: "1.0", Architecture: "sparc"},                     //7
-			{Name: "dpkg", Version: "1.0", Architecture: "i386", FilesHash: 0x34445}, //8
-			{Name: "app", Version: "1.1~bp2", Architecture: "i386", FilesHash: 0x44}, //9
-		}
-
-		for _, p := range packages {
-			p.V06Plus = true
-			coll.Update(p)
-		}
-
-		listA := NewPackageList()
-		listA.Add(packages[0])
-		listA.Add(packages[1])
-		listA.Add(packages[2])
-		listA.Add(packages[3])
-		listA.Add(packages[7])
-
-		listB := NewPackageList()
-		listB.Add(packages[0])
-		listB.Add(packages[2])
-		listB.Add(packages[4])
-		listB.Add(packages[5])
-		listB.Add(packages[6])
-
-		listC := NewPackageList()
-		listC.Add(packages[0])
-		listC.Add(packages[8])
-		listC.Add(packages[9])
-
-		reflistA := f.newFromPackageList(listA)
-		reflistB := f.newFromPackageList(listB)
-		reflistC := f.newFromPackageList(listC)
-
-		mergeAB := mergeRefLists(reflistA, reflistB, true, false)
-		mergeBA := mergeRefLists(reflistB, reflistA, true, false)
-		mergeAC := mergeRefLists(reflistA, reflistC, true, false)
-		mergeBC := mergeRefLists(reflistB, reflistC, true, false)
-		mergeCB := mergeRefLists(reflistC, reflistB, true, false)
-
-		verifyRefListIntegrity(c, mergeAB)
-		verifyRefListIntegrity(c, mergeBA)
-		verifyRefListIntegrity(c, mergeAC)
-		verifyRefListIntegrity(c, mergeBC)
-		verifyRefListIntegrity(c, mergeCB)
-
-		c.Check(toStrSlice(mergeAB), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
-		c.Check(toStrSlice(mergeBA), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
-		c.Check(toStrSlice(mergeAC), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
-		c.Check(toStrSlice(mergeBC), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
-		c.Check(toStrSlice(mergeCB), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
-
-		mergeABall := mergeRefLists(reflistA, reflistB, false, false)
-		mergeBAall := mergeRefLists(reflistB, reflistA, false, false)
-		mergeACall := mergeRefLists(reflistA, reflistC, false, false)
-		mergeBCall := mergeRefLists(reflistB, reflistC, false, false)
-		mergeCBall := mergeRefLists(reflistC, reflistB, false, false)
-
-		verifyRefListIntegrity(c, mergeABall)
-		verifyRefListIntegrity(c, mergeBAall)
-		verifyRefListIntegrity(c, mergeACall)
-		verifyRefListIntegrity(c, mergeBCall)
-		verifyRefListIntegrity(c, mergeCBall)
-
-		c.Check(mergeABall, DeepEquals, mergeBAall)
-		c.Check(toStrSlice(mergeBAall), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000000",
-				"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
-
-		c.Check(mergeBCall, Not(DeepEquals), mergeCBall)
-		c.Check(toStrSlice(mergeACall), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
-				"Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
-		c.Check(toStrSlice(mergeBCall), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
-				"Pi386 lib 1.0 00000000"})
-
-		mergeBCwithConflicts := mergeRefLists(reflistB, reflistC, false, true)
-		c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
-			[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
-				"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
-	})
-}
-
-func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
-	forEachRefList(func(f reflistFactory) {
-		packages := []*Package{
-			{Name: "lib", Version: "1.0", Architecture: "i386"},
-			{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
-			{Name: "lib", Version: "1.2", Architecture: "i386"},
-			{Name: "dpkg", Version: "1.2", Architecture: "i386"},
-			{Name: "dpkg", Version: "1.3", Architecture: "i386"},
-			{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
-			{Name: "dpkg", Version: "1.5", Architecture: "i386"},
-			{Name: "dpkg", Version: "1.6", Architecture: "i386"},
-		}
-
-		rl := NewPackageList()
-		rl.Add(packages[0])
-		rl.Add(packages[1])
-		rl.Add(packages[2])
-		rl.Add(packages[3])
-		rl.Add(packages[4])
-		rl.Add(packages[5])
-		rl.Add(packages[6])
-		rl.Add(packages[7])
-
-		result := f.newFromPackageList(rl)
-		result.FilterLatestRefs()
-
-		verifyRefListIntegrity(c, result)
-		c.Check(toStrSlice(result), DeepEquals,
-			[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})
-	})
+	reflist = NewPackageRefList()
+	c.Check(reflist.Len(), Equals, 0)
 }
 
 func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) {
-	list := NewPackageList()
-	list.Add(s.p1)
-	list.Add(s.p3)
-	list.Add(s.p5)
-	list.Add(s.p6)
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
 
-	reflist := NewPackageRefListFromPackageList(list)
+	reflist := NewPackageRefListFromPackageList(s.list)
 
 	reflist2 := &PackageRefList{}
 	err := reflist2.Decode(reflist.Encode())
@@ -521,185 +104,286 @@ func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) {
 	c.Check(reflist2.Refs, DeepEquals, reflist.Refs)
 }
 
-func (s *PackageRefListSuite) TestRefListBucketPrefix(c *C) {
-	c.Check(bucketRefPrefix([]byte("Pall abcd 1.0")), DeepEquals, []byte("abc"))
-	c.Check(bucketRefPrefix([]byte("Pall libabcd 1.0")), DeepEquals, []byte("abc"))
-	c.Check(bucketRefPrefix([]byte("Pamd64 xy 1.0")), DeepEquals, []byte("xy"))
-}
+func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
 
-func (s *PackageRefListSuite) TestRefListBucketIdx(c *C) {
-	c.Check(bucketIdxForRef(s.p1.Key("")), Equals, 46)
-	c.Check(bucketIdxForRef(s.p2.Key("")), Equals, 46)
-	c.Check(bucketIdxForRef(s.p3.Key("")), Equals, 26)
-	c.Check(bucketIdxForRef(s.p4.Key("")), Equals, 46)
-	c.Check(bucketIdxForRef(s.p5.Key("")), Equals, 4)
-	c.Check(bucketIdxForRef(s.p6.Key("")), Equals, 46)
-}
+	reflist := NewPackageRefListFromPackageList(s.list)
 
-func (s *PackageRefListSuite) TestSplitRefListBuckets(c *C) {
-	list := NewPackageList()
-	list.Add(s.p1)
-	list.Add(s.p3)
-	list.Add(s.p4)
-	list.Add(s.p5)
-	list.Add(s.p6)
-
-	sl := NewSplitRefListFromPackageList(list)
-	verifyRefListIntegrity(c, sl)
-
-	c.Check(hex.EncodeToString(sl.Buckets[4]), Equals, "7287aed32daad5d1aab4e89533bde135381d932e60548cfc00b882ca8858ae07")
-	c.Check(toStrSlice(sl.bucketRefs[4]), DeepEquals, []string{string(s.p5.Key(""))})
-	c.Check(hex.EncodeToString(sl.Buckets[26]), Equals, "f31fc28e82368b63c8be47eefc64b8e217e2e5349c7e3827b98f80536b956f6e")
-	c.Check(toStrSlice(sl.bucketRefs[26]), DeepEquals, []string{string(s.p3.Key(""))})
-	c.Check(hex.EncodeToString(sl.Buckets[46]), Equals, "55e70286393afc5da5046d68c632d35f98bec24781ae433bd1a1069b52853367")
-	c.Check(toStrSlice(sl.bucketRefs[46]), DeepEquals, []string{string(s.p1.Key("")), string(s.p6.Key(""))})
-}
-
-func (s *PackageRefListSuite) TestRefListDigestSet(c *C) {
-	list := NewPackageList()
-	list.Add(s.p1)
-	list.Add(s.p3)
-	list.Add(s.p4)
-	list.Add(s.p5)
-	list.Add(s.p6)
-
-	sl := NewSplitRefListFromPackageList(list)
-
-	set := NewRefListDigestSet()
-	c.Check(set.Len(), Equals, 0)
-
-	err := sl.ForEachBucket(func(digest []byte, bucket *PackageRefList) error {
-		c.Check(set.Has(digest), Equals, false)
+	Len := 0
+	err := reflist.ForEach(func([]byte) error {
+		Len++
 		return nil
 	})
-	c.Assert(err, IsNil)
 
-	set.AddAllInRefList(sl)
-	c.Check(set.Len(), Equals, 3)
+	c.Check(Len, Equals, 4)
+	c.Check(err, IsNil)
 
-	err = sl.ForEachBucket(func(digest []byte, bucket *PackageRefList) error {
-		c.Check(set.Has(digest), Equals, true)
-		return nil
+	e := errors.New("b")
+
+	err = reflist.ForEach(func([]byte) error {
+		return e
 	})
-	c.Assert(err, IsNil)
 
-	firstDigest := sl.Buckets[bucketIdxForRef(s.p1.Key(""))]
-	set.Remove(firstDigest)
-	c.Check(set.Len(), Equals, 2)
-
-	err = sl.ForEachBucket(func(digest []byte, bucket *PackageRefList) error {
-		c.Check(set.Has(digest), Equals, !bytes.Equal(digest, firstDigest))
-		return nil
-	})
-	c.Assert(err, IsNil)
-
-	set2 := NewRefListDigestSet()
-	set2.AddAllInRefList(sl)
-	set2.RemoveAll(set)
-
-	err = sl.ForEachBucket(func(digest []byte, bucket *PackageRefList) error {
-		c.Check(set2.Has(digest), Equals, bytes.Equal(digest, firstDigest))
-		return nil
-	})
-	c.Assert(err, IsNil)
+	c.Check(err, Equals, e)
 }
 
-func (s *PackageRefListSuite) TestRefListCollectionLoadSave(c *C) {
+func (s *PackageRefListSuite) TestHas(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	c.Check(reflist.Has(s.p1), Equals, true)
+	c.Check(reflist.Has(s.p3), Equals, true)
+	c.Check(reflist.Has(s.p5), Equals, true)
+	c.Check(reflist.Has(s.p2), Equals, true)
+	c.Check(reflist.Has(s.p6), Equals, false)
+}
+
+func (s *PackageRefListSuite) TestSubstract(c *C) {
+	r1 := []byte("r1")
+	r2 := []byte("r2")
+	r3 := []byte("r3")
+	r4 := []byte("r4")
+	r5 := []byte("r5")
+
+	empty := &PackageRefList{Refs: [][]byte{}}
+	l1 := &PackageRefList{Refs: [][]byte{r1, r2, r3, r4}}
+	l2 := &PackageRefList{Refs: [][]byte{r1, r3}}
+	l3 := &PackageRefList{Refs: [][]byte{r2, r4}}
+	l4 := &PackageRefList{Refs: [][]byte{r4, r5}}
+	l5 := &PackageRefList{Refs: [][]byte{r1, r2, r3}}
+
+	c.Check(l1.Subtract(empty), DeepEquals, l1)
+	c.Check(l1.Subtract(l2), DeepEquals, l3)
+	c.Check(l1.Subtract(l3), DeepEquals, l2)
+	c.Check(l1.Subtract(l4), DeepEquals, l5)
+	c.Check(empty.Subtract(l1), DeepEquals, empty)
+	c.Check(l2.Subtract(l3), DeepEquals, l2)
+}
+
+func (s *PackageRefListSuite) TestDiff(c *C) {
 	db, _ := goleveldb.NewOpenDB(c.MkDir())
-	reflistCollection := NewRefListCollection(db)
-	packageCollection := NewPackageCollection(db)
+	coll := NewPackageCollection(db)
 
-	packageCollection.Update(s.p1)
-	packageCollection.Update(s.p2)
-	packageCollection.Update(s.p3)
-	packageCollection.Update(s.p4)
-	packageCollection.Update(s.p5)
-	packageCollection.Update(s.p6)
+	packages := []*Package{
+		{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
+	}
 
-	list := NewPackageList()
-	list.Add(s.p1)
-	list.Add(s.p2)
-	list.Add(s.p3)
-	list.Add(s.p4)
-	list.Add(s.p5)
+	for _, p := range packages {
+		coll.Update(p)
+	}
 
-	key := []byte("test")
+	listA := NewPackageList()
+	listA.Add(packages[0])
+	listA.Add(packages[1])
+	listA.Add(packages[2])
+	listA.Add(packages[3])
+	listA.Add(packages[6])
 
-	reflist := NewPackageRefListFromPackageList(list)
-	db.Put(key, reflist.Encode())
+	listB := NewPackageList()
+	listB.Add(packages[0])
+	listB.Add(packages[2])
+	listB.Add(packages[4])
+	listB.Add(packages[5])
 
-	sl := NewSplitRefList()
-	err := reflistCollection.LoadComplete(sl, key)
-	c.Assert(err, IsNil)
-	verifyRefListIntegrity(c, sl)
-	c.Check(toStrSlice(sl), DeepEquals, toStrSlice(reflist))
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
 
-	list.Add(s.p6)
-	sl = NewSplitRefListFromPackageList(list)
-	err = reflistCollection.Update(sl, key)
-	c.Assert(err, IsNil)
+	diffAA, err := reflistA.Diff(reflistA, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAA, HasLen, 0)
+
+	diffAB, err := reflistA.Diff(reflistB, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAB, HasLen, 4)
+
+	c.Check(diffAB[0].Left, IsNil)
+	c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
+
+	c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
+
+	c.Check(diffAB[2].Left.String(), Equals, "dpkg_1.7_i386")
+	c.Check(diffAB[2].Right, IsNil)
+
+	c.Check(diffAB[3].Left.String(), Equals, "xyz_3.0_sparc")
+	c.Check(diffAB[3].Right, IsNil)
+
+	diffBA, err := reflistB.Diff(reflistA, coll)
+	c.Check(err, IsNil)
+	c.Check(diffBA, HasLen, 4)
+
+	c.Check(diffBA[0].Right, IsNil)
+	c.Check(diffBA[0].Left.String(), Equals, "app_1.1~bp2_amd64")
+
+	c.Check(diffBA[1].Right.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffBA[1].Left.String(), Equals, "app_1.1~bp2_i386")
+
+	c.Check(diffBA[2].Right.String(), Equals, "dpkg_1.7_i386")
+	c.Check(diffBA[2].Left, IsNil)
+
+	c.Check(diffBA[3].Right.String(), Equals, "xyz_3.0_sparc")
+	c.Check(diffBA[3].Left, IsNil)
 
-	sl = NewSplitRefList()
-	err = reflistCollection.LoadComplete(sl, key)
-	c.Assert(err, IsNil)
-	verifyRefListIntegrity(c, sl)
-	c.Check(toStrSlice(sl), DeepEquals, toStrSlice(NewPackageRefListFromPackageList(list)))
 }
 
-func (s *PackageRefListSuite) TestRefListCollectionMigrate(c *C) {
+func (s *PackageRefListSuite) TestDiffCompactsAtEnd(c *C) {
 	db, _ := goleveldb.NewOpenDB(c.MkDir())
-	reflistCollection := NewRefListCollection(db)
-	packageCollection := NewPackageCollection(db)
+	coll := NewPackageCollection(db)
 
-	packageCollection.Update(s.p1)
-	packageCollection.Update(s.p2)
-	packageCollection.Update(s.p3)
-	packageCollection.Update(s.p4)
-	packageCollection.Update(s.p5)
-	packageCollection.Update(s.p6)
+	packages := []*Package{
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //0
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //1
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //2
+	}
 
-	list := NewPackageList()
-	list.Add(s.p1)
-	list.Add(s.p2)
-	list.Add(s.p3)
-	list.Add(s.p4)
-	list.Add(s.p5)
+	for _, p := range packages {
+		coll.Update(p)
+	}
 
-	key := []byte("test")
+	listA := NewPackageList()
+	listA.Add(packages[0])
 
-	reflist := NewPackageRefListFromPackageList(list)
-	db.Put(key, reflist.Encode())
+	listB := NewPackageList()
+	listB.Add(packages[1])
+	listB.Add(packages[2])
 
-	sl := NewSplitRefList()
-	format, err := reflistCollection.load(sl, key)
-	c.Assert(err, IsNil)
-	c.Check(format, Equals, reflistStorageFormatInline)
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
 
-	migrator := reflistCollection.NewMigration()
-	err = reflistCollection.LoadCompleteAndMigrate(sl, key, migrator)
-	c.Assert(err, IsNil)
-	verifyRefListIntegrity(c, sl)
-	c.Check(toStrSlice(sl), DeepEquals, toStrSlice(NewPackageRefListFromPackageList(list)))
+	diffAB, err := reflistA.Diff(reflistB, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAB, HasLen, 2)
 
-	stats := migrator.Stats()
-	c.Check(stats.Reflists, Equals, 0)
-	c.Check(stats.Buckets, Equals, 0)
-	c.Check(stats.Segments, Equals, 0)
+	c.Check(diffAB[0].Left, IsNil)
+	c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
 
-	err = migrator.Flush()
-	c.Assert(err, IsNil)
-	stats = migrator.Stats()
-	c.Check(stats.Reflists, Equals, 1)
-	c.Check(stats.Buckets, Not(Equals), 0)
-	c.Check(stats.Segments, Equals, stats.Segments)
-
-	sl = NewSplitRefList()
-	err = reflistCollection.LoadComplete(sl, key)
-	c.Assert(err, IsNil)
-	verifyRefListIntegrity(c, sl)
-	c.Check(toStrSlice(sl), DeepEquals, toStrSlice(NewPackageRefListFromPackageList(list)))
-
-	format, err = reflistCollection.load(sl, key)
-	c.Assert(err, IsNil)
-	c.Check(format, Equals, reflistStorageFormatSplit)
+	c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
+}
+
+func (s *PackageRefListSuite) TestMerge(c *C) {
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+
+	packages := []*Package{
+		{Name: "lib", Version: "1.0", Architecture: "i386"},                      //0
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},                     //1
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},                  //2
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},                  //3
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},                  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"},                 //5
+		{Name: "dpkg", Version: "1.0", Architecture: "i386"},                     //6
+		{Name: "xyz", Version: "1.0", Architecture: "sparc"},                     //7
+		{Name: "dpkg", Version: "1.0", Architecture: "i386", FilesHash: 0x34445}, //8
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386", FilesHash: 0x44}, //9
+	}
+
+	for _, p := range packages {
+		p.V06Plus = true
+		coll.Update(p)
+	}
+
+	listA := NewPackageList()
+	listA.Add(packages[0])
+	listA.Add(packages[1])
+	listA.Add(packages[2])
+	listA.Add(packages[3])
+	listA.Add(packages[7])
+
+	listB := NewPackageList()
+	listB.Add(packages[0])
+	listB.Add(packages[2])
+	listB.Add(packages[4])
+	listB.Add(packages[5])
+	listB.Add(packages[6])
+
+	listC := NewPackageList()
+	listC.Add(packages[0])
+	listC.Add(packages[8])
+	listC.Add(packages[9])
+
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
+	reflistC := NewPackageRefListFromPackageList(listC)
+
+	mergeAB := reflistA.Merge(reflistB, true, false)
+	mergeBA := reflistB.Merge(reflistA, true, false)
+	mergeAC := reflistA.Merge(reflistC, true, false)
+	mergeBC := reflistB.Merge(reflistC, true, false)
+	mergeCB := reflistC.Merge(reflistB, true, false)
+
+	c.Check(toStrSlice(mergeAB), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBA), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeAC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
+	c.Check(toStrSlice(mergeCB), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
+
+	mergeABall := reflistA.Merge(reflistB, false, false)
+	mergeBAall := reflistB.Merge(reflistA, false, false)
+	mergeACall := reflistA.Merge(reflistC, false, false)
+	mergeBCall := reflistB.Merge(reflistC, false, false)
+	mergeCBall := reflistC.Merge(reflistB, false, false)
+
+	c.Check(mergeABall, DeepEquals, mergeBAall)
+	c.Check(toStrSlice(mergeBAall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000000",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+
+	c.Check(mergeBCall, Not(DeepEquals), mergeCBall)
+	c.Check(toStrSlice(mergeACall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBCall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 lib 1.0 00000000"})
+
+	mergeBCwithConflicts := reflistB.Merge(reflistC, false, true)
+	c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
+}
+
+func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
+	packages := []*Package{
+		{Name: "lib", Version: "1.0", Architecture: "i386"},
+		{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
+		{Name: "lib", Version: "1.2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+	}
+
+	rl := NewPackageList()
+	rl.Add(packages[0])
+	rl.Add(packages[1])
+	rl.Add(packages[2])
+	rl.Add(packages[3])
+	rl.Add(packages[4])
+	rl.Add(packages[5])
+	rl.Add(packages[6])
+	rl.Add(packages[7])
+
+	result := NewPackageRefListFromPackageList(rl)
+	result.FilterLatestRefs()
+
+	c.Check(toStrSlice(result), DeepEquals,
+		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})
 }

deb/remote.go

@@ -20,7 +20,7 @@ import (
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -73,7 +73,7 @@ type RemoteRepo struct {
 	// Packages for json output
 	Packages []string `codec:"-" json:",omitempty"`
 	// "Snapshot" of current list of packages
-	packageRefs *SplitRefList
+	packageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
 	// Current list of packages (filled while updating mirror)
@@ -84,7 +84,7 @@ type RemoteRepo struct {
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
 	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:              uuid.NewString(),
+		UUID:              uuid.New(),
 		Name:              name,
 		ArchiveRoot:       archiveRoot,
 		Distribution:      distribution,
@@ -170,7 +170,7 @@ func (repo *RemoteRepo) NumPackages() int {
 }
 
 // RefList returns package list for repo
-func (repo *RemoteRepo) RefList() *SplitRefList {
+func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
@@ -686,7 +686,7 @@ func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, p
 	})
 
 	if err == nil {
-		repo.packageRefs = NewSplitRefListFromPackageList(repo.packageList)
+		repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
 		repo.packageList = nil
 	}
 
@@ -828,14 +828,14 @@ func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, un
 }
 
 // Add appends new repo to collection and saves it
-func (collection *RemoteRepoCollection) Add(repo *RemoteRepo, reflistCollection *RefListCollection) error {
+func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
 	_, err := collection.ByName(repo.Name)
 
 	if err == nil {
 		return fmt.Errorf("mirror with name %s already exists", repo.Name)
 	}
 
-	err = collection.Update(repo, reflistCollection)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
@@ -845,26 +845,28 @@ func (collection *RemoteRepoCollection) Add(repo *RemoteRepo, reflistCollection
 }
 
 // Update stores updated information about repo in DB
-func (collection *RemoteRepoCollection) Update(repo *RemoteRepo, reflistCollection *RefListCollection) error {
+func (collection *RemoteRepoCollection) Update(repo *RemoteRepo) error {
 	batch := collection.db.CreateBatch()
 
 	batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		rb := reflistCollection.NewBatch(batch)
-		reflistCollection.UpdateInBatch(repo.packageRefs, repo.RefKey(), rb)
+		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
 	return batch.Write()
 }
 
 // LoadComplete loads additional information for remote repo
-func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo, reflistCollection *RefListCollection) error {
-	repo.packageRefs = NewSplitRefList()
-	err := reflistCollection.LoadComplete(repo.packageRefs, repo.RefKey())
+func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo) error {
+	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
 	}
+	if err != nil {
+		return err
+	}
 
-	return err
+	repo.packageRefs = &PackageRefList{}
+	return repo.packageRefs.Decode(encoded)
 }
 
 // ByName looks up repository by name

deb/remote_test.go

@@ -52,7 +52,7 @@ func (n *NullVerifier) IsClearSigned(clearsign io.Reader) (bool, error) {
 type PackageListMixinSuite struct {
 	p1, p2, p3 *Package
 	list       *PackageList
-	reflist    *SplitRefList
+	reflist    *PackageRefList
 }
 
 func (s *PackageListMixinSuite) SetUpPackages() {
@@ -72,7 +72,7 @@ func (s *PackageListMixinSuite) SetUpPackages() {
 	s.list.Add(s.p2)
 	s.list.Add(s.p3)
 
-	s.reflist = NewSplitRefListFromPackageList(s.list)
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 }
 
 type RemoteRepoSuite struct {
@@ -290,7 +290,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Flatten().Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
@@ -372,12 +372,12 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
 	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Flatten().Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Flatten().Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "installer")
 }
@@ -418,12 +418,12 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Flatten().Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Flatten().Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 
@@ -502,7 +502,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Flatten().Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
@@ -592,12 +592,12 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Flatten().Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Flatten().Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
@@ -657,9 +657,8 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 
 type RemoteRepoCollectionSuite struct {
 	PackageListMixinSuite
-	db                database.Storage
-	collection        *RemoteRepoCollection
-	refListCollection *RefListCollection
+	db         database.Storage
+	collection *RemoteRepoCollection
 }
 
 var _ = Suite(&RemoteRepoCollectionSuite{})
@@ -667,7 +666,6 @@ var _ = Suite(&RemoteRepoCollectionSuite{})
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
-	s.refListCollection = NewRefListCollection(s.db)
 	s.SetUpPackages()
 }
 
@@ -680,8 +678,8 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
-	c.Assert(s.collection.Add(repo, s.refListCollection), IsNil)
-	c.Assert(s.collection.Add(repo, s.refListCollection), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.Add(repo), IsNil)
+	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
 	r, err := s.collection.ByName("yandex")
 	c.Assert(err, IsNil)
@@ -698,7 +696,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
-	c.Assert(s.collection.Add(repo, s.refListCollection), IsNil)
+	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
@@ -712,7 +710,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
-	c.Assert(s.collection.Update(repo, s.refListCollection), IsNil)
+	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
 	r, err := collection.ByName("yandex")
@@ -720,20 +718,20 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	c.Assert(r.packageRefs, IsNil)
 
 	repo.packageRefs = s.reflist
-	c.Assert(s.collection.Update(repo, s.refListCollection), IsNil)
+	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection = NewRemoteRepoCollection(s.db)
 	r, err = collection.ByName("yandex")
 	c.Assert(err, IsNil)
 	c.Assert(r.packageRefs, IsNil)
 	c.Assert(r.NumPackages(), Equals, 0)
-	c.Assert(s.collection.LoadComplete(r, s.refListCollection), IsNil)
+	c.Assert(s.collection.LoadComplete(r), IsNil)
 	c.Assert(r.NumPackages(), Equals, 3)
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
-	s.collection.Add(repo, s.refListCollection)
+	s.collection.Add(repo)
 
 	count := 0
 	err := s.collection.ForEach(func(*RemoteRepo) error {
@@ -755,10 +753,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
 	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
-	s.collection.Add(repo1, s.refListCollection)
+	s.collection.Add(repo1)
 
 	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
-	s.collection.Add(repo2, s.refListCollection)
+	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
 	c.Check(r1, Equals, repo1)

deb/snapshot.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -40,17 +40,17 @@ type Snapshot struct {
 	NotAutomatic         string
 	ButAutomaticUpgrades string
 
-	packageRefs *SplitRefList
+	packageRefs *PackageRefList
 }
 
 // NewSnapshotFromRepository creates snapshot from current state of repository
 func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error) {
-	if repo.packageRefs == nil {
+	if repo.packageRefs == nil || repo.packageRefs.Len() == 0 {
 		return nil, errors.New("mirror not updated")
 	}
 
 	return &Snapshot{
-		UUID:                 uuid.NewString(),
+		UUID:                 uuid.New(),
 		Name:                 name,
 		CreatedAt:            time.Now(),
 		SourceKind:           SourceRemoteRepo,
@@ -66,7 +66,7 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 // NewSnapshotFromLocalRepo creates snapshot from current state of local repository
 func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
 	snap := &Snapshot{
-		UUID:        uuid.NewString(),
+		UUID:        uuid.New(),
 		Name:        name,
 		CreatedAt:   time.Now(),
 		SourceKind:  SourceLocalRepo,
@@ -76,7 +76,7 @@ func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
 	}
 
 	if snap.packageRefs == nil {
-		snap.packageRefs = NewSplitRefList()
+		snap.packageRefs = NewPackageRefList()
 	}
 
 	return snap, nil
@@ -84,20 +84,18 @@ func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
 
 // NewSnapshotFromPackageList creates snapshot from PackageList
 func NewSnapshotFromPackageList(name string, sources []*Snapshot, list *PackageList, description string) *Snapshot {
-	sl := NewSplitRefList()
-	sl.Replace(NewPackageRefListFromPackageList(list))
-	return NewSnapshotFromRefList(name, sources, sl, description)
+	return NewSnapshotFromRefList(name, sources, NewPackageRefListFromPackageList(list), description)
 }
 
-// NewSnapshotFromRefList creates snapshot from SplitRefList
-func NewSnapshotFromRefList(name string, sources []*Snapshot, list *SplitRefList, description string) *Snapshot {
+// NewSnapshotFromRefList creates snapshot from PackageRefList
+func NewSnapshotFromRefList(name string, sources []*Snapshot, list *PackageRefList, description string) *Snapshot {
 	sourceUUIDs := make([]string, len(sources))
 	for i := range sources {
 		sourceUUIDs[i] = sources[i].UUID
 	}
 
 	return &Snapshot{
-		UUID:        uuid.NewString(),
+		UUID:        uuid.New(),
 		Name:        name,
 		CreatedAt:   time.Now(),
 		SourceKind:  "snapshot",
@@ -114,14 +112,11 @@ func (s *Snapshot) String() string {
 
 // NumPackages returns number of packages in snapshot
 func (s *Snapshot) NumPackages() int {
-	if s.packageRefs == nil {
-		return 0
-	}
 	return s.packageRefs.Len()
 }
 
 // RefList returns list of package refs in snapshot
-func (s *Snapshot) RefList() *SplitRefList {
+func (s *Snapshot) RefList() *PackageRefList {
 	return s.packageRefs
 }
 
@@ -214,13 +209,13 @@ func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
 }
 
 // Add appends new repo to collection and saves it
-func (collection *SnapshotCollection) Add(snapshot *Snapshot, reflistCollection *RefListCollection) error {
+func (collection *SnapshotCollection) Add(snapshot *Snapshot) error {
 	_, err := collection.ByName(snapshot.Name)
 	if err == nil {
 		return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
 	}
 
-	err = collection.Update(snapshot, reflistCollection)
+	err = collection.Update(snapshot)
 	if err != nil {
 		return err
 	}
@@ -230,22 +225,26 @@ func (collection *SnapshotCollection) Add(snapshot *Snapshot, reflistCollection
 }
 
 // Update stores updated information about snapshot in DB
-func (collection *SnapshotCollection) Update(snapshot *Snapshot, reflistCollection *RefListCollection) error {
+func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	batch := collection.db.CreateBatch()
 
 	batch.Put(snapshot.Key(), snapshot.Encode())
 	if snapshot.packageRefs != nil {
-		rb := reflistCollection.NewBatch(batch)
-		reflistCollection.UpdateInBatch(snapshot.packageRefs, snapshot.RefKey(), rb)
+		batch.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
 	}
 
 	return batch.Write()
 }
 
 // LoadComplete loads additional information about snapshot
-func (collection *SnapshotCollection) LoadComplete(snapshot *Snapshot, reflistCollection *RefListCollection) error {
-	snapshot.packageRefs = NewSplitRefList()
-	return reflistCollection.LoadComplete(snapshot.packageRefs, snapshot.RefKey())
+func (collection *SnapshotCollection) LoadComplete(snapshot *Snapshot) error {
+	encoded, err := collection.db.Get(snapshot.RefKey())
+	if err != nil {
+		return err
+	}
+
+	snapshot.packageRefs = &PackageRefList{}
+	return snapshot.packageRefs.Decode(encoded)
 }
 
 func (collection *SnapshotCollection) search(filter func(*Snapshot) bool, unique bool) []*Snapshot {

deb/snapshot_bench_test.go

@@ -18,11 +18,10 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) {
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)
-	reflistCollection := NewRefListCollection(db)
 
 	for i := 0; i < count; i++ {
-		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewSplitRefList(), fmt.Sprintf("Snapshot number %d", i))
-		if collection.Add(snapshot, reflistCollection) != nil {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
 			b.FailNow()
 		}
 	}
@@ -48,12 +47,11 @@ func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)
-	reflistCollection := NewRefListCollection(db)
 
 	uuids := []string{}
 	for i := 0; i < count; i++ {
-		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewSplitRefList(), fmt.Sprintf("Snapshot number %d", i))
-		if collection.Add(snapshot, reflistCollection) != nil {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
 			b.FailNow()
 		}
 		uuids = append(uuids, snapshot.UUID)
@@ -80,11 +78,10 @@ func BenchmarkSnapshotCollectionByName(b *testing.B) {
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)
-	reflistCollection := NewRefListCollection(db)
 
 	for i := 0; i < count; i++ {
-		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewSplitRefList(), fmt.Sprintf("Snapshot number %d", i))
-		if collection.Add(snapshot, reflistCollection) != nil {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
 			b.FailNow()
 		}
 	}

deb/snapshot_test.go

@@ -109,7 +109,6 @@ type SnapshotCollectionSuite struct {
 	snapshot1, snapshot2 *Snapshot
 	snapshot3, snapshot4 *Snapshot
 	collection           *SnapshotCollection
-	reflistCollection    *RefListCollection
 }
 
 var _ = Suite(&SnapshotCollectionSuite{})
@@ -117,7 +116,6 @@ var _ = Suite(&SnapshotCollectionSuite{})
 func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewSnapshotCollection(s.db)
-	s.reflistCollection = NewRefListCollection(s.db)
 	s.SetUpPackages()
 
 	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
@@ -145,10 +143,10 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 	_, err := s.collection.ByName("snap1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	c.Assert(s.collection.Add(s.snapshot1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot1, s.reflistCollection), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot1), ErrorMatches, ".*already exists")
 
-	c.Assert(s.collection.Add(s.snapshot2, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
 
 	snapshot, err := s.collection.ByName("snap1")
 	c.Assert(err, IsNil)
@@ -169,20 +167,20 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestUpdateLoadComplete(c *C) {
-	c.Assert(s.collection.Update(s.snapshot1, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Update(s.snapshot1), IsNil)
 
 	collection := NewSnapshotCollection(s.db)
 	snapshot, err := collection.ByName("snap1")
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.packageRefs, IsNil)
 
-	c.Assert(s.collection.LoadComplete(snapshot, s.reflistCollection), IsNil)
+	c.Assert(s.collection.LoadComplete(snapshot), IsNil)
 	c.Assert(snapshot.NumPackages(), Equals, 3)
 }
 
 func (s *SnapshotCollectionSuite) TestForEachAndLen(c *C) {
-	s.collection.Add(s.snapshot1, s.reflistCollection)
-	s.collection.Add(s.snapshot2, s.reflistCollection)
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot2)
 
 	count := 0
 	err := s.collection.ForEach(func(*Snapshot) error {
@@ -202,10 +200,10 @@ func (s *SnapshotCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestForEachSorted(c *C) {
-	s.collection.Add(s.snapshot2, s.reflistCollection)
-	s.collection.Add(s.snapshot1, s.reflistCollection)
-	s.collection.Add(s.snapshot4, s.reflistCollection)
-	s.collection.Add(s.snapshot3, s.reflistCollection)
+	s.collection.Add(s.snapshot2)
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot4)
+	s.collection.Add(s.snapshot3)
 
 	names := []string{}
 
@@ -219,8 +217,8 @@ func (s *SnapshotCollectionSuite) TestForEachSorted(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
-	c.Assert(s.collection.Add(s.snapshot1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot2, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
 
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
@@ -231,10 +229,10 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestFindByLocalRepoSource(c *C) {
-	c.Assert(s.collection.Add(s.snapshot1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot2, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot3, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot4, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
+	c.Assert(s.collection.Add(s.snapshot3), IsNil)
+	c.Assert(s.collection.Add(s.snapshot4), IsNil)
 
 	c.Check(s.collection.ByLocalRepoSource(s.lrepo1), DeepEquals, []*Snapshot{s.snapshot3})
 	c.Check(s.collection.ByLocalRepoSource(s.lrepo2), DeepEquals, []*Snapshot{s.snapshot4})
@@ -249,11 +247,11 @@ func (s *SnapshotCollectionSuite) TestFindSnapshotSource(c *C) {
 	snapshot4 := NewSnapshotFromRefList("snap4", []*Snapshot{s.snapshot1}, s.reflist, "desc2")
 	snapshot5 := NewSnapshotFromRefList("snap5", []*Snapshot{snapshot3}, s.reflist, "desc3")
 
-	c.Assert(s.collection.Add(s.snapshot1, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(s.snapshot2, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(snapshot3, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(snapshot4, s.reflistCollection), IsNil)
-	c.Assert(s.collection.Add(snapshot5, s.reflistCollection), IsNil)
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
+	c.Assert(s.collection.Add(snapshot3), IsNil)
+	c.Assert(s.collection.Add(snapshot4), IsNil)
+	c.Assert(s.collection.Add(snapshot5), IsNil)
 
 	list := s.collection.BySnapshotSource(s.snapshot1)
 	sorter, _ := newSnapshotSorter("name", list)
@@ -265,8 +263,8 @@ func (s *SnapshotCollectionSuite) TestFindSnapshotSource(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestDrop(c *C) {
-	s.collection.Add(s.snapshot1, s.reflistCollection)
-	s.collection.Add(s.snapshot2, s.reflistCollection)
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot2)
 
 	snap, _ := s.collection.ByUUID(s.snapshot1.UUID)
 	c.Check(snap, Equals, s.snapshot1)

debian/aptly-api.postrm

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# source debconf library
-. /usr/share/debconf/confmodule
-
-case "$1" in
-    purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
-	# only remove aptly-api user and its homedir on purge
-	if [ "${1}" = "purge" ] ; then
-	    userdel -r aptly-api
-	fi
-    ;;
-
-    *)
-        echo "postrm called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-#DEBHELPER#
-
-exit 0

debian/aptly.conf

@@ -228,7 +228,7 @@ s3_publish_endpoints:
     #     # Encryption Method (optional)
     #     # Server-side encryption method, defaults to none. Currently
     #     # the only available encryption method is `AES256`
-    #     encryption_method: ""
+    #     encryption_method: none
     #     # Plus Workaround (optional)
     #     # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by
     #     # creating two copies of package files with `+` in filename: one original

debian/changelog

@@ -1,13 +1,3 @@
-aptly (1.6.1) stable; urgency=medium
-
-  * update golang-github-syndtr-goleveldb-dev dependency (v1.0.1-0.20220721030215-126854af5e6d) to fix segfault on arm64
-    (bug in golang-github-golang-snappy-dev)
-  * allow snapshotting empty mirrors again (regression)
-  * debian compliance: add postrm (note: `apt purge aptly-api` will remove all data in ~aptly-api/)
-  * update other dependencies (x/net 0.33.0, gin-gonic/gin 1.9.1)
-
- -- André Roth <neolynx@gmail.com>  Sat, 15 Feb 2025 13:03:16 +0100
-
 aptly (1.6.0) stable; urgency=medium
 
   * support reading filters from file or stdin

debian/control

@@ -45,6 +45,7 @@ Build-Depends: bash-completion,
                golang-github-munnerz-goautoneg-dev,
                golang-github-mxk-go-flowrate-dev,
                golang-github-ncw-swift-dev,
+               golang-github-pborman-uuid-dev,
                golang-github-pelletier-go-toml,
                golang-github-pkg-errors-dev,
                golang-github-prometheus-client-golang-dev,

files/package_pool.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/google/uuid"
+	"github.com/pborman/uuid"
 	"github.com/saracen/walker"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -417,7 +417,7 @@ func (pool *PackagePool) FullPath(path string) string {
 
 // GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
 func (pool *PackagePool) GenerateTempPath(filename string) (string, error) {
-	random := uuid.NewString()
+	random := uuid.NewRandom().String()
 
 	return filepath.Join(pool.rootPath, random[0:2], random[2:4], random[4:]+filename), nil
 }

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/awalterschulze/gographviz v2.0.1+incompatible
 	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/cheggaaa/pb v1.0.25
-	github.com/gin-gonic/gin v1.9.1
-	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/gin-gonic/gin v1.9.0
+	github.com/go-playground/validator/v10 v10.11.2 // indirect
 	github.com/h2non/filetype v1.1.3
 	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/kjk/lzma v0.0.0-20120628231508-2a7c55cad4a2
@@ -21,6 +21,7 @@ require (
 	github.com/mkrautz/goar v0.0.0-20150919110319-282caa8bd9da
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f
 	github.com/ncw/swift v1.0.53
+	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.0
 	github.com/rs/zerolog v1.29.1
@@ -29,7 +30,7 @@ require (
 	github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76
 	github.com/smira/go-ftp-protocol v0.0.0-20140829150050-066b75c2b70d
 	github.com/smira/go-xz v0.1.0
-	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
+	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	github.com/ugorji/go/codec v1.2.11
 	github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0
 	golang.org/x/crypto v0.31.0 // indirect
@@ -59,14 +60,13 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/bytedance/sonic v1.8.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cloudflare/circl v1.4.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/fatih/color v1.17.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
@@ -77,15 +77,16 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang/snappy v0.0.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -102,8 +103,8 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
+	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
@@ -123,7 +124,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.46
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1
 	github.com/aws/smithy-go v1.22.1
-	github.com/google/uuid v1.6.0
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.3

go.sum

@@ -64,8 +64,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
-github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
-github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
+github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA=
+github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
 github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
 github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -75,9 +75,6 @@ github.com/cheggaaa/pb v1.0.25/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXH
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY=
 github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=
@@ -92,17 +89,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
 github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
-github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
 github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
-github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8=
+github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -119,9 +113,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
-github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU=
+github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -136,20 +129,18 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
@@ -160,7 +151,6 @@ github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
 github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -173,9 +163,8 @@ github.com/kjk/lzma v0.0.0-20120628231508-2a7c55cad4a2 h1:TVZQgMi+I83S3rCuE65Hnm
 github.com/kjk/lzma v0.0.0-20120628231508-2a7c55cad4a2/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
-github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -188,8 +177,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
-github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
@@ -219,20 +208,17 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks=
 github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
-github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
@@ -273,14 +259,11 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
@@ -290,8 +273,8 @@ github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+z
 github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo=
 github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
 github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
-github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
-github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
+github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk=
+github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
@@ -313,9 +296,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
-golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -335,19 +317,17 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -363,20 +343,16 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -393,6 +369,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -406,7 +383,6 @@ golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
@@ -415,8 +391,8 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxb
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
 google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
@@ -429,8 +405,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

system/t02_config/CreateConfigTest_gold

@@ -228,7 +228,7 @@ s3_publish_endpoints:
     #     # Encryption Method (optional)
     #     # Server-side encryption method, defaults to none. Currently
     #     # the only available encryption method is `AES256`
-    #     encryption_method: ""
+    #     encryption_method: none
     #     # Plus Workaround (optional)
     #     # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by
     #     # creating two copies of package files with `+` in filename: one original

system/t05_snapshot/CreateSnapshot10Test_gold

@@ -1,3 +0,0 @@
-
-Snapshot oh-snap successfully created.
-You can run 'aptly publish snapshot oh-snap' to publish snapshot as Debian repository.

system/t05_snapshot/create.py

@@ -106,14 +106,3 @@ class CreateSnapshot9Test(BaseTest):
         "aptly repo create local-repo",
     ]
     runCmd = "aptly snapshot create snap9 from repo local-repo"
-
-
-class CreateSnapshot10Test(BaseTest):
-    """
-    create snapshot: from empty mirror
-    """
-    fixtureCmds = [
-        "aptly mirror create -ignore-signatures non-free http://repo.aptly.info/system-tests/security.debian.org/debian-security bookworm-security updates/non-free",
-        "aptly mirror update -ignore-signatures non-free"
-    ]
-    runCmd = "aptly snapshot create oh-snap from mirror non-free"

system/t08_db/CleanupDB10Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB11Test_gold

@@ -14,7 +14,6 @@ Loading mirrors:
 Loading local repos:
 Loading snapshots:
 Loading published repositories:
-Split 11 reflist(s) into 510 bucket(s) (123181 segment(s))
 Loading list of all packages...
 Deleting unreferenced packages (7)...
 List of package keys to delete:
@@ -25,7 +24,6 @@ List of package keys to delete:
  - Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815
  - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
  - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (9)...

system/t08_db/CleanupDB12Test_gold

@@ -14,7 +14,6 @@ Loading mirrors:
 Loading local repos:
 Loading snapshots:
 Loading published repositories:
-Skipped splitting 11 reflist(s) into 510 bucket(s) (123181 segment(s)), as -dry-run has been requested.
 Loading list of all packages...
 Deleting unreferenced packages (7)...
 List of package keys to delete:
@@ -26,7 +25,6 @@ List of package keys to delete:
  - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
  - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
 Skipped deletion, as -dry-run has been requested.
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (9)...

system/t08_db/CleanupDB1Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB2Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (73270)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB3Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (7)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (9)...

system/t08_db/CleanupDB4Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB5Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (7)...
-Deleting unreferenced reflist buckets (1)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (9)...

system/t08_db/CleanupDB6Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB7Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...

system/t08_db/CleanupDB8Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (4)...
-Deleting unreferenced reflist buckets (1)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (6)...

system/t08_db/CleanupDB9Test_gold

@@ -1,7 +1,6 @@
 Loading mirrors, local repos, snapshots and published repos...
 Loading list of all packages...
 Deleting unreferenced packages (0)...
-Deleting unreferenced reflist buckets (0)...
 Building list of files referenced by packages...
 Building list of files in package pool...
 Deleting unreferenced files (0)...