Remove corrupt package references in db recover

When aptly crashes it is possible to get a corrupt database with a dangling key reference.
This results in an error with 'key not found', eg:

    ERROR: unable to load package Pall example-package 1.2.3 778cf6f877bf6e2d: key not found

This change makes `db recover` fix this situation by removing the dangling references.

system/files/corruptdb.go

@@ -0,0 +1,42 @@
+// This utility corrupts a database by deleting matching package entries.
+// Do not use it outside system tests.
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+)
+
+func main() {
+	var dbPath, prefix string
+
+	flag.StringVar(&dbPath, "db", "", "Path to DB to corrupt")
+	flag.StringVar(&prefix, "prefix", "P", "Path to DB to corrupt")
+	flag.Parse()
+
+	db, err := goleveldb.NewOpenDB(dbPath)
+	if err != nil {
+		log.Fatalf("Error opening DB %q: %s", dbPath, err)
+	}
+	defer db.Close()
+
+	keys := db.KeysByPrefix([]byte(prefix))
+	if len(keys) == 0 {
+		keys2 := db.KeysByPrefix([]byte{})
+		for _, key := range keys2 {
+			log.Printf("Have: %q", key)
+		}
+
+		log.Fatal("No keys to delete")
+	}
+
+	for _, key := range keys {
+		log.Printf("Deleting %q", key)
+
+		if err = db.Delete(key); err != nil {
+			log.Fatalf("Error deleting key: %s", err)
+		}
+	}
+}

system/t08_db/RecoverDB1Test_gold

@@ -1 +1,2 @@
 Recovering database...
+Checking database integrity...

system/t08_db/RecoverDB2Test_gold

@@ -1 +1,2 @@
 Recovering database...
+Checking database integrity...

system/t08_db/RecoverDB3Test_cleanup

@@ -0,0 +1,8 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading list of all packages...
+Deleting unreferenced packages (0)...
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (1)...
+Disk space freed: 12.18 KiB...
+Compacting database...

system/t08_db/RecoverDB3Test_gold

@@ -0,0 +1,3 @@
+Recovering database...
+Checking database integrity...
+Removing dangling database reference "Pamd64 hardlink 0.2.1 daf8fcecbf8210ad"

system/t08_db/recover.py

@@ -1,3 +1,5 @@
+import os
+
 from lib import BaseTest
 
 
@@ -23,3 +25,27 @@ class RecoverDB2Test(BaseTest):
     def check(self):
         self.check_output()
         self.check_cmd_output("aptly mirror list", "mirror_list")
+
+
+class RecoverDB3Test(BaseTest):
+    """
+    recover db: dangling reference
+    """
+    fixtureDB = True
+    fixtureCmds = [
+        "aptly repo create db3test",
+        "aptly repo add db3test changes/hardlink_0.2.1_amd64.deb",
+    ]
+
+    runCmd = "aptly db recover"
+
+    def prepare(self):
+        super(RecoverDB3Test, self).prepare()
+
+        self.run_cmd(["go", "run", "files/corruptdb.go",
+                      "-db", os.path.join(os.environ["HOME"], self.aptlyDir, "db"),
+                      "-prefix", "Pamd64 hardlink 0.2.1"])
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly db cleanup", "cleanup")