From 0d94f29c277a94f67dec6d13c4ffd30c07b24485 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 29 Sep 2017 22:39:51 +0300 Subject: [PATCH] Allow using files from the pool while importing source packages Sometimes source packages reference files already present in the pool. Allow for those file to be omitted when importing packages either via `repo add` or `repo include`. If file is missing, aptly would make an attempt to look up file in the package pool (by checksum) and use it. Fixes: #278 --- deb/import.go | 26 ++++++++++++++-- system/t09_repo/AddRepo16Test_gold | 2 ++ system/t09_repo/AddRepo16Test_repo_show | 5 +++ system/t09_repo/AddRepo5Test_gold | 2 +- system/t09_repo/IncludeRepo22Test_gold | 3 ++ system/t09_repo/add.py | 34 ++++++++++++++++++++ system/t09_repo/include.py | 41 +++++++++++++++++++++++++ 7 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 system/t09_repo/AddRepo16Test_gold create mode 100644 system/t09_repo/AddRepo16Test_repo_show create mode 100644 system/t09_repo/IncludeRepo22Test_gold diff --git a/deb/import.go b/deb/import.go index 0e9d227b..74afa631 100644 --- a/deb/import.go +++ b/deb/import.go @@ -147,14 +147,34 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b // go over all the other files for i := range files { sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename)) - files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage) + + _, err = os.Stat(sourceFile) + if err == nil { + files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage) + if err == nil { + candidateProcessedFiles = append(candidateProcessedFiles, sourceFile) + } + } else if os.IsNotExist(err) { + // if file is not present, try to find it in the pool + var ( + err2 error + found bool + ) + + files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage) + if err2 != nil { + err = err2 + } else if found { + // clear error, file is already in the package pool + err = nil + } + } + if err != nil { reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err) failedFiles = append(failedFiles, file) break } - - candidateProcessedFiles = append(candidateProcessedFiles, sourceFile) } if err != nil { // some files haven't been imported diff --git a/system/t09_repo/AddRepo16Test_gold b/system/t09_repo/AddRepo16Test_gold new file mode 100644 index 00000000..1897e606 --- /dev/null +++ b/system/t09_repo/AddRepo16Test_gold @@ -0,0 +1,2 @@ +Loading packages... +[+] pyspi_0.6.1-1.3_source added diff --git a/system/t09_repo/AddRepo16Test_repo_show b/system/t09_repo/AddRepo16Test_repo_show new file mode 100644 index 00000000..26baf543 --- /dev/null +++ b/system/t09_repo/AddRepo16Test_repo_show @@ -0,0 +1,5 @@ +Name: repo2 +Comment: +Default Distribution: +Default Component: main +Number of packages: 1 diff --git a/system/t09_repo/AddRepo5Test_gold b/system/t09_repo/AddRepo5Test_gold index d0ab5bce..cbf6989f 100644 --- a/system/t09_repo/AddRepo5Test_gold +++ b/system/t09_repo/AddRepo5Test_gold @@ -1,5 +1,5 @@ Loading packages... -[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: open /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory +[!] Unable to import file /02/03/pyspi_0.6.1-1.3.diff.gz into pool: stat /02/03/pyspi_0.6.1-1.3.diff.gz: no such file or directory [!] Some files were skipped due to errors: /02/03/pyspi_0.6.1-1.3.dsc ERROR: some files failed to be added diff --git a/system/t09_repo/IncludeRepo22Test_gold b/system/t09_repo/IncludeRepo22Test_gold new file mode 100644 index 00000000..aad7ca65 --- /dev/null +++ b/system/t09_repo/IncludeRepo22Test_gold @@ -0,0 +1,3 @@ +Loading repository unstable for changes file hardlink_0.2.1_amd64.changes... +[+] hardlink_0.2.1_source added +[+] hardlink_0.2.1_amd64 added diff --git a/system/t09_repo/add.py b/system/t09_repo/add.py index a83db8bb..dea199b9 100644 --- a/system/t09_repo/add.py +++ b/system/t09_repo/add.py @@ -310,3 +310,37 @@ class AddRepo15Test(BaseTest): def outputMatchPrepare(self, s): return s.replace(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__), ""). \ replace(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), "") + + +class AddRepo16Test(BaseTest): + """ + add package to local repo: some source files missing, but already in the pool + """ + fixtureCmds = [ + "aptly repo create repo1", + "aptly repo create repo2", + "aptly repo add repo1 ${files}" + ] + runCmd = "aptly repo add repo2 " + + def outputMatchPrepare(self, s): + return s.replace(self.tempSrcDir, "") + + def prepare(self): + super(AddRepo16Test, self).prepare() + + self.tempSrcDir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755) + + shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1-1.3.dsc"), + os.path.join(self.tempSrcDir, "02", "03")) + shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1.orig.tar.gz"), + os.path.join(self.tempSrcDir, "02", "03")) + + self.runCmd += self.tempSrcDir + + def check(self): + self.check_output() + self.check_cmd_output("aptly repo show repo2", "repo_show") + + shutil.rmtree(self.tempSrcDir) diff --git a/system/t09_repo/include.py b/system/t09_repo/include.py index 434387c1..2ae6b99a 100644 --- a/system/t09_repo/include.py +++ b/system/t09_repo/include.py @@ -516,3 +516,44 @@ class IncludeRepo21Test(BaseTest): super(IncludeRepo21Test, self).check() finally: shutil.rmtree(self.tempSrcDir) + + +class IncludeRepo22Test(BaseTest): + """ + include packages to local repo: missing files, but files aready in the pool + """ + fixtureCmds = [ + "aptly repo create stable", + "aptly repo create unstable", + "aptly repo add stable ${changes}" + ] + runCmd = "aptly repo include -ignore-signatures -keyring=${files}/aptly.pub " + + def outputMatchPrepare(self, s): + return gpgRemove(self, tempDirRemove(self, s)) + + def prepare(self): + super(IncludeRepo22Test, self).prepare() + + self.tempSrcDir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755) + + for path in ["hardlink_0.2.1.dsc", "hardlink_0.2.1_amd64.deb"]: + shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), + os.path.join(self.tempSrcDir, "01", path)) + + path = "hardlink_0.2.1_amd64.changes" + with open(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), "r") as source: + with open(os.path.join(self.tempSrcDir, "01", path), "w") as dest: + content = source.readlines() + # remove reference to .tar.gz file + content = [line for line in content if "hardlink_0.2.1.tar.gz" not in line] + dest.write("".join(content)) + + self.runCmd += self.tempSrcDir + + def check(self): + try: + super(IncludeRepo22Test, self).check() + finally: + shutil.rmtree(self.tempSrcDir)