Allow using files from the pool while importing source packages

Sometimes source packages reference files already present in the pool.

Allow for those file to be omitted when importing packages either via
`repo add` or `repo include`. If file is missing, aptly would make
an attempt to look up file in the package pool (by checksum) and
use it.

Fixes: #278

deb/import.go

@@ -147,14 +147,34 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		// go over all the other files
 		for i := range files {
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+
+			_, err = os.Stat(sourceFile)
+			if err == nil {
+				files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+				if err == nil {
+					candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+				}
+			} else if os.IsNotExist(err) {
+				// if file is not present, try to find it in the pool
+				var (
+					err2  error
+					found bool
+				)
+
+				files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage)
+				if err2 != nil {
+					err = err2
+				} else if found {
+					// clear error, file is already in the package pool
+					err = nil
+				}
+			}
+
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
 				break
 			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
 		}
 		if err != nil {
 			// some files haven't been imported

system/t09_repo/AddRepo16Test_gold

@@ -0,0 +1,2 @@
+Loading packages...
+[+] pyspi_0.6.1-1.3_source added

system/t09_repo/AddRepo16Test_repo_show

@@ -0,0 +1,5 @@
+Name: repo2
+Comment: 
+Default Distribution: 
+Default Component: main
+Number of packages: 1

system/t09_repo/AddRepo5Test_gold

@@ -1,5 +1,5 @@
 Loading packages...
-[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: open /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory
+[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: stat /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory
 [!] Some files were skipped due to errors:
   /02/03/pyspi_0.6.1-1.3.dsc
 ERROR: some files failed to be added

system/t09_repo/IncludeRepo22Test_gold

@@ -0,0 +1,3 @@
+Loading repository unstable for changes file hardlink_0.2.1_amd64.changes...
+[+] hardlink_0.2.1_source added
+[+] hardlink_0.2.1_amd64 added

system/t09_repo/add.py

@@ -310,3 +310,37 @@ class AddRepo15Test(BaseTest):
     def outputMatchPrepare(self, s):
         return s.replace(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__), ""). \
                          replace(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), "")
+
+
+class AddRepo16Test(BaseTest):
+    """
+    add package to local repo: some source files missing, but already in the pool
+    """
+    fixtureCmds = [
+        "aptly repo create repo1",
+        "aptly repo create repo2",
+        "aptly repo add repo1 ${files}"
+    ]
+    runCmd = "aptly repo add repo2 "
+
+    def outputMatchPrepare(self, s):
+        return s.replace(self.tempSrcDir, "")
+
+    def prepare(self):
+        super(AddRepo16Test, self).prepare()
+
+        self.tempSrcDir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755)
+
+        shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1-1.3.dsc"),
+                    os.path.join(self.tempSrcDir, "02", "03"))
+        shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1.orig.tar.gz"),
+                    os.path.join(self.tempSrcDir, "02", "03"))
+
+        self.runCmd += self.tempSrcDir
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly repo show repo2", "repo_show")
+
+        shutil.rmtree(self.tempSrcDir)

system/t09_repo/include.py

@@ -516,3 +516,44 @@ class IncludeRepo21Test(BaseTest):
             super(IncludeRepo21Test, self).check()
         finally:
             shutil.rmtree(self.tempSrcDir)
+
+
+class IncludeRepo22Test(BaseTest):
+    """
+    include packages to local repo: missing files, but files aready in the pool
+    """
+    fixtureCmds = [
+        "aptly repo create stable",
+        "aptly repo create unstable",
+        "aptly repo add stable ${changes}"
+    ]
+    runCmd = "aptly repo include -ignore-signatures -keyring=${files}/aptly.pub "
+
+    def outputMatchPrepare(self, s):
+        return gpgRemove(self, tempDirRemove(self, s))
+
+    def prepare(self):
+        super(IncludeRepo22Test, self).prepare()
+
+        self.tempSrcDir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755)
+
+        for path in ["hardlink_0.2.1.dsc", "hardlink_0.2.1_amd64.deb"]:
+            shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path),
+                        os.path.join(self.tempSrcDir, "01", path))
+
+        path = "hardlink_0.2.1_amd64.changes"
+        with open(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), "r") as source:
+            with open(os.path.join(self.tempSrcDir, "01", path), "w") as dest:
+                content = source.readlines()
+                # remove reference to .tar.gz file
+                content = [line for line in content if "hardlink_0.2.1.tar.gz" not in line]
+                dest.write("".join(content))
+
+        self.runCmd += self.tempSrcDir
+
+    def check(self):
+        try:
+            super(IncludeRepo22Test, self).check()
+        finally:
+            shutil.rmtree(self.tempSrcDir)