Merge pull request #650 from smira/278-import-files-from-pool

Allow using files from the pool while importing source packages

deb/import.go

@@ -147,14 +147,34 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		// go over all the other files
 		for i := range files {
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+
+			_, err = os.Stat(sourceFile)
+			if err == nil {
+				files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+				if err == nil {
+					candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+				}
+			} else if os.IsNotExist(err) {
+				// if file is not present, try to find it in the pool
+				var (
+					err2  error
+					found bool
+				)
+
+				files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage)
+				if err2 != nil {
+					err = err2
+				} else if found {
+					// clear error, file is already in the package pool
+					err = nil
+				}
+			}
+
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
 				break
 			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
 		}
 		if err != nil {
 			// some files haven't been imported

system/t09_repo/AddRepo16Test_gold

@@ -0,0 +1,2 @@
+Loading packages...
+[+] pyspi_0.6.1-1.3_source added

system/t09_repo/AddRepo16Test_repo_show

@@ -0,0 +1,5 @@
+Name: repo2
+Comment: 
+Default Distribution: 
+Default Component: main
+Number of packages: 1

system/t09_repo/AddRepo5Test_gold

@@ -1,5 +1,5 @@
 Loading packages...
-[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: open /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory
+[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: stat /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory
 [!] Some files were skipped due to errors:
   /02/03/pyspi_0.6.1-1.3.dsc
 ERROR: some files failed to be added

system/t09_repo/IncludeRepo22Test_gold

@@ -0,0 +1,3 @@
+Loading repository unstable for changes file hardlink_0.2.1_amd64.changes...
+[+] hardlink_0.2.1_source added
+[+] hardlink_0.2.1_amd64 added

system/t09_repo/add.py

@@ -310,3 +310,37 @@ class AddRepo15Test(BaseTest):
     def outputMatchPrepare(self, s):
         return s.replace(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__), ""). \
                          replace(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), "")
+
+
+class AddRepo16Test(BaseTest):
+    """
+    add package to local repo: some source files missing, but already in the pool
+    """
+    fixtureCmds = [
+        "aptly repo create repo1",
+        "aptly repo create repo2",
+        "aptly repo add repo1 ${files}"
+    ]
+    runCmd = "aptly repo add repo2 "
+
+    def outputMatchPrepare(self, s):
+        return s.replace(self.tempSrcDir, "")
+
+    def prepare(self):
+        super(AddRepo16Test, self).prepare()
+
+        self.tempSrcDir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755)
+
+        shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1-1.3.dsc"),
+                    os.path.join(self.tempSrcDir, "02", "03"))
+        shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1.orig.tar.gz"),
+                    os.path.join(self.tempSrcDir, "02", "03"))
+
+        self.runCmd += self.tempSrcDir
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly repo show repo2", "repo_show")
+
+        shutil.rmtree(self.tempSrcDir)

system/t09_repo/include.py

@@ -516,3 +516,44 @@ class IncludeRepo21Test(BaseTest):
             super(IncludeRepo21Test, self).check()
         finally:
             shutil.rmtree(self.tempSrcDir)
+
+
+class IncludeRepo22Test(BaseTest):
+    """
+    include packages to local repo: missing files, but files aready in the pool
+    """
+    fixtureCmds = [
+        "aptly repo create stable",
+        "aptly repo create unstable",
+        "aptly repo add stable ${changes}"
+    ]
+    runCmd = "aptly repo include -ignore-signatures -keyring=${files}/aptly.pub "
+
+    def outputMatchPrepare(self, s):
+        return gpgRemove(self, tempDirRemove(self, s))
+
+    def prepare(self):
+        super(IncludeRepo22Test, self).prepare()
+
+        self.tempSrcDir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755)
+
+        for path in ["hardlink_0.2.1.dsc", "hardlink_0.2.1_amd64.deb"]:
+            shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path),
+                        os.path.join(self.tempSrcDir, "01", path))
+
+        path = "hardlink_0.2.1_amd64.changes"
+        with open(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), "r") as source:
+            with open(os.path.join(self.tempSrcDir, "01", path), "w") as dest:
+                content = source.readlines()
+                # remove reference to .tar.gz file
+                content = [line for line in content if "hardlink_0.2.1.tar.gz" not in line]
+                dest.write("".join(content))
+
+        self.runCmd += self.tempSrcDir
+
+    def check(self):
+        try:
+            super(IncludeRepo22Test, self).check()
+        finally:
+            shutil.rmtree(self.tempSrcDir)