diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 94e7db92..43a0b0bb 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -44,6 +44,12 @@ type LocalPackagePool interface { GenerateTempPath(filename string) (string, error) // Link generates hardlink to destination path Link(path, dstPath string) error + // Symlink generates symlink to destination path + Symlink(path, dstPath string) error + // FullPath generates full path to the file in pool + // + // Please use with care: it's not supposed to be used to access files + FullPath(path string) string } // PublishedStorage is abstraction of filesystem storing all published repositories diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 66136b09..b176b1f1 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -123,14 +123,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { ch := make(chan error, count) // In separate goroutine (to avoid blocking main), push queue to downloader - go func() { + /*go func() { for _, task := range queue { context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries) } // We don't need queue after this point queue = nil - }() + }()*/ // Wait for all downloads to finish var errors []string diff --git a/cmd/package_show.go b/cmd/package_show.go index 4b8f5939..ba9ab0b8 100644 --- a/cmd/package_show.go +++ b/cmd/package_show.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/smira/aptly/aptly" "github.com/smira/aptly/deb" "github.com/smira/aptly/query" "github.com/smira/commander" @@ -87,11 +88,17 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error { if withFiles { fmt.Printf("Files in the pool:\n") + packagePool := context.PackagePool() for _, f := range p.Files() { - path, err := context.PackagePool().Path(f.Filename, f.Checksums) + path, err := f.GetPoolPath(packagePool) if err != nil { return err } + + if pp, ok := packagePool.(aptly.LocalPackagePool); ok { + path = pp.FullPath(path) + } + fmt.Printf(" %s\n", path) } fmt.Printf("\n") diff --git a/context/context.go b/context/context.go index 049e7c4e..2a95fcb8 100644 --- a/context/context.go +++ b/context/context.go @@ -204,8 +204,7 @@ func (context *AptlyContext) Downloader() aptly.Downloader { if downloadLimit == 0 { downloadLimit = context.config().DownloadLimit } - context.downloader = http.NewDownloader(context.config().DownloadConcurrency, - downloadLimit*1024, context._progress()) + context.downloader = http.NewDownloader(downloadLimit*1024, context._progress()) } return context.downloader @@ -416,7 +415,6 @@ func (context *AptlyContext) Shutdown() { context.database = nil } if context.downloader != nil { - context.downloader.Abort() context.downloader = nil } if context.progress != nil { @@ -431,7 +429,6 @@ func (context *AptlyContext) Cleanup() { defer context.Unlock() if context.downloader != nil { - context.downloader.Shutdown() context.downloader = nil } if context.progress != nil { diff --git a/deb/deb.go b/deb/deb.go index 50c7b7cb..af74e29e 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -12,7 +12,9 @@ import ( "github.com/h2non/filetype/matchers" "github.com/mkrautz/goar" + "github.com/pkg/errors" + "github.com/smira/aptly/aptly" "github.com/smira/aptly/utils" "github.com/smira/go-xz" "github.com/smira/lzma" @@ -105,13 +107,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err } // GetContentsFromDeb returns list of files installed by .deb package -func GetContentsFromDeb(packageFile string) ([]string, error) { - file, err := os.Open(packageFile) - if err != nil { - return nil, err - } - defer file.Close() - +func GetContentsFromDeb(file aptly.ReadSeekerCloser, packageFile string) ([]string, error) { library := ar.NewReader(file) for { header, err := library.Next() @@ -119,7 +115,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) { return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile) } if err != nil { - return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err) + return nil, errors.Wrapf(err, "unable to read .deb archive from %s", packageFile) } if strings.HasPrefix(header.Name, "data.tar") { @@ -142,7 +138,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) { } else { ungzip, err := gzip.NewReader(bufReader) if err != nil { - return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err) + return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile) } defer ungzip.Close() tarInput = ungzip @@ -152,7 +148,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) { case "data.tar.xz": unxz, err := xz.NewReader(bufReader) if err != nil { - return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err) + return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile) } defer unxz.Close() tarInput = unxz @@ -172,7 +168,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) { return results, nil } if err != nil { - return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err) + return nil, errors.Wrapf(err, "unable to read .tar archive from %s", packageFile) } if tarHeader.Typeflag == tar.TypeDir { diff --git a/deb/deb_test.go b/deb/deb_test.go index 0260044d..646f3780 100644 --- a/deb/deb_test.go +++ b/deb/deb_test.go @@ -1,6 +1,7 @@ package deb import ( + "os" "path/filepath" "runtime" @@ -59,13 +60,19 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) { } func (s *DebSuite) TestGetContentsFromDeb(c *C) { - contents, err := GetContentsFromDeb(s.debFile) + f, err := os.Open(s.debFile) + c.Assert(err, IsNil) + contents, err := GetContentsFromDeb(f, s.debFile) c.Check(err, IsNil) c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz", "usr/share/doc/libboost-program-options-dev/copyright"}) + c.Assert(f.Close(), IsNil) - contents, err = GetContentsFromDeb(s.debFile2) + f, err = os.Open(s.debFile2) + c.Assert(err, IsNil) + contents, err = GetContentsFromDeb(f, s.debFile2) c.Check(err, IsNil) c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz", "usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"}) + c.Assert(f.Close(), IsNil) } diff --git a/deb/import.go b/deb/import.go index 9282d7e5..81468f4a 100644 --- a/deb/import.go +++ b/deb/import.go @@ -116,19 +116,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b continue } + var files PackageFiles + + if isSourcePackage { + files = p.Files() + } + var checksums utils.ChecksumInfo checksums, err = utils.ChecksumsForFile(file) if err != nil { return nil, nil, err } - if isSourcePackage { - p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums})) - } else { - p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}}) + mainPackageFile := PackageFile{ + Filename: filepath.Base(file), + Checksums: checksums, } - err = pool.Import(file, checksums) + mainPackageFile.PoolPath, err = pool.Import(file, mainPackageFile.Filename, &mainPackageFile.Checksums, false) if err != nil { reporter.Warning("Unable to import file %s into pool: %s", file, err) failedFiles = append(failedFiles, file) @@ -137,13 +142,10 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b candidateProcessedFiles = append(candidateProcessedFiles, file) - // go over all files, except for the last one (.dsc/.deb itself) - for _, f := range p.Files() { - if filepath.Base(f.Filename) == filepath.Base(file) { - continue - } + // go over all the other files + for _, f := range files { sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename)) - err = pool.Import(sourceFile, f.Checksums) + f.PoolPath, err = pool.Import(sourceFile, f.Filename, &f.Checksums, false) if err != nil { reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err) failedFiles = append(failedFiles, file) @@ -157,6 +159,8 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b continue } + p.UpdateFiles(append(files, mainPackageFile)) + if restriction != nil && !restriction.Matches(p) { reporter.Warning("%s has been ignored as it doesn't match restriction", p) failedFiles = append(failedFiles, file) diff --git a/deb/package.go b/deb/package.go index 71bc4aed..d2463df7 100644 --- a/deb/package.go +++ b/deb/package.go @@ -418,7 +418,7 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl } file := p.Files()[0] - path, err := packagePool.Path(file.Filename, file.Checksums) + poolPath, err := file.GetPoolPath(packagePool) if err != nil { if progress != nil { progress.ColoredPrintf("@y[!]@| @!Failed to build pool path: @| %s", err) @@ -426,7 +426,16 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl return nil, err } - contents, err := GetContentsFromDeb(path) + reader, err := packagePool.Open(poolPath) + if err != nil { + if progress != nil { + progress.ColoredPrintf("@y[!]@| @!Failed to open package in pool: @| %s", err) + } + return nil, err + } + defer reader.Close() + + contents, err := GetContentsFromDeb(reader, file.Filename) if err != nil { if progress != nil { progress.ColoredPrintf("@y[!]@| @!Failed to generate package contents: @| %s", err) @@ -547,7 +556,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP } for i, f := range p.Files() { - sourcePath, err := packagePool.Path(f.Filename, f.Checksums) + sourcePoolPath, err := f.GetPoolPath(packagePool) if err != nil { return err } @@ -555,7 +564,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP relPath := filepath.Join("pool", component, poolDir) publishedDirectory := filepath.Join(prefix, relPath) - err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums, force) + err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePoolPath, f.Checksums, force) if err != nil { return err } @@ -604,14 +613,9 @@ type PackageDownloadTask struct { // DownloadList returns list of missing package files for download in format // [[srcpath, dstpath]] func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) { - result = make([]PackageDownloadTask, 0, 1) + /*result = make([]PackageDownloadTask, 0, 1) for _, f := range p.Files() { - poolPath, err := packagePool.Path(f.Filename, f.Checksums) - if err != nil { - return nil, err - } - verified, err := f.Verify(packagePool) if err != nil { return nil, err @@ -622,7 +626,8 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD } } - return result, nil + return result, nil*/ + panic("NEEDS REWORK") } // VerifyFiles verifies that all package files have neen correctly downloaded @@ -645,7 +650,7 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) result := make([]string, len(p.Files())) for i, f := range p.Files() { - result[i], err = packagePool.RelativePath(f.Filename, f.Checksums) + result[i], err = f.GetPoolPath(packagePool) if err != nil { return nil, err } diff --git a/deb/package_files.go b/deb/package_files.go index 94f6dbe1..4019c1dd 100644 --- a/deb/package_files.go +++ b/deb/package_files.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "fmt" "hash/fnv" - "os" "path/filepath" "sort" "strconv" @@ -20,18 +19,20 @@ type PackageFile struct { Filename string // Hashes for the file Checksums utils.ChecksumInfo + // PoolPath persists relative path to file in the package pool + PoolPath string // Temporary field used while downloading, stored relative path on the mirror downloadPath string } // Verify that package file is present and correct func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) { - poolPath, err := packagePool.Path(f.Filename, f.Checksums) + poolPath, err := f.GetPoolPath(packagePool) if err != nil { return false, err } - st, err := os.Stat(poolPath) + st, err := packagePool.Stat(poolPath) if err != nil { return false, nil } @@ -41,6 +42,19 @@ func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) { return st.Size() == f.Checksums.Size, nil } +// GetPoolPath returns path to the file in the pool +// +// For legacy packages which do not have PoolPath field set, that calculates LegacyPath via pool +func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) { + var err error + + if f.PoolPath == "" { + f.PoolPath, err = packagePool.LegacyPath(f.Filename, &f.Checksums) + } + + return f.PoolPath, err +} + // DownloadURL return relative URL to package download location func (f *PackageFile) DownloadURL() string { return filepath.Join(f.downloadPath, f.Filename) diff --git a/deb/package_files_test.go b/deb/package_files_test.go index 83604788..41ebea15 100644 --- a/deb/package_files_test.go +++ b/deb/package_files_test.go @@ -1,7 +1,7 @@ package deb import ( - "os" + "io/ioutil" "path/filepath" "github.com/smira/aptly/files" @@ -30,19 +30,15 @@ func (s *PackageFilesSuite) SetUpTest(c *C) { func (s *PackageFilesSuite) TestVerify(c *C) { packagePool := files.NewPackagePool(c.MkDir()) - poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums) result, err := s.files[0].Verify(packagePool) c.Check(err, IsNil) c.Check(result, Equals, false) - err = os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) + tmpFilepath := filepath.Join(c.MkDir(), "file") + c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil) - file, err := os.Create(poolPath) - c.Assert(err, IsNil) - file.WriteString("abcde") - file.Close() + s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false) result, err = s.files[0].Verify(packagePool) c.Check(err, IsNil) diff --git a/deb/package_test.go b/deb/package_test.go index 1d999c1d..043b33bb 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -2,12 +2,11 @@ package deb import ( "bytes" - "os" + "io/ioutil" "path/filepath" "regexp" "github.com/smira/aptly/files" - "github.com/smira/aptly/utils" . "gopkg.in/check.v1" ) @@ -367,15 +366,12 @@ func (s *PackageSuite) TestLinkFromPool(c *C) { publishedStorage := files.NewPublishedStorage(c.MkDir(), "", "") p := NewPackageFromControlFile(s.stanza) - poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums) - err := os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) + tmpFilepath := filepath.Join(c.MkDir(), "file") + c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil) - file, err := os.Create(poolPath) - c.Assert(err, IsNil) - file.Close() + p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false) - err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) + err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) c.Check(err, IsNil) c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena") @@ -396,48 +392,46 @@ func (s *PackageSuite) TestFilepathList(c *C) { } func (s *PackageSuite) TestDownloadList(c *C) { - packagePool := files.NewPackagePool(c.MkDir()) - p := NewPackageFromControlFile(s.stanza) - p.Files()[0].Checksums.Size = 5 - poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums) + c.Fail() + /* + packagePool := files.NewPackagePool(c.MkDir()) + p := NewPackageFromControlFile(s.stanza) + p.Files()[0].Checksums.Size = 5 + poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums) - list, err := p.DownloadList(packagePool) - c.Check(err, IsNil) - c.Check(list, DeepEquals, []PackageDownloadTask{ - { - RepoURI: "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", - DestinationPath: poolPath, - Checksums: utils.ChecksumInfo{Size: 5, - MD5: "1e8cba92c41420aa7baa8a5718d67122", - SHA1: "46955e48cad27410a83740a21d766ce362364024", - SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}}) + list, err := p.DownloadList(packagePool) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []PackageDownloadTask{ + { + RepoURI: "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", + DestinationPath: poolPath, + Checksums: utils.ChecksumInfo{Size: 5, + MD5: "1e8cba92c41420aa7baa8a5718d67122", + SHA1: "46955e48cad27410a83740a21d766ce362364024", + SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}}) - err = os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) + err = os.MkdirAll(filepath.Dir(poolPath), 0755) + c.Assert(err, IsNil) - file, err := os.Create(poolPath) - c.Assert(err, IsNil) - file.WriteString("abcde") - file.Close() + file, err := os.Create(poolPath) + c.Assert(err, IsNil) + file.WriteString("abcde") + file.Close() - list, err = p.DownloadList(packagePool) - c.Check(err, IsNil) - c.Check(list, DeepEquals, []PackageDownloadTask{}) + list, err = p.DownloadList(packagePool) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []PackageDownloadTask{})*/ } func (s *PackageSuite) TestVerifyFiles(c *C) { p := NewPackageFromControlFile(s.stanza) packagePool := files.NewPackagePool(c.MkDir()) - poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums) - err := os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) + tmpFilepath := filepath.Join(c.MkDir(), "file") + c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil) - file, err := os.Create(poolPath) - c.Assert(err, IsNil) - file.WriteString("abcde") - file.Close() + p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false) result, err := p.VerifyFiles(packagePool) c.Check(err, IsNil) diff --git a/deb/publish_test.go b/deb/publish_test.go index b526315c..56d95137 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -132,11 +132,10 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory) s.repo5.SkipContents = true - poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums) - err := os.MkdirAll(filepath.Dir(poolPath), 0755) - f, err := os.Create(poolPath) - c.Assert(err, IsNil) - f.Close() + tmpFilepath := filepath.Join(c.MkDir(), "file") + c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil) + + s.p1.Files()[0].PoolPath, _ = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false) } func (s *PublishedRepoSuite) TearDownTest(c *C) { diff --git a/files/package_pool.go b/files/package_pool.go index b6fce2d5..775eba88 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -164,7 +164,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } // trying to overwrite file? - return "", fmt.Errorf("unable to import into pool: file %s already exists", poolPath) + return "", fmt.Errorf("unable to import into pool: file %s already exists", fullPoolPath) } if pool.supportLegacyPaths { @@ -254,6 +254,18 @@ func (pool *PackagePool) Link(path, dstPath string) error { return os.Link(filepath.Join(pool.rootPath, path), dstPath) } +// Symlink generates symlink to destination path +func (pool *PackagePool) Symlink(path, dstPath string) error { + return os.Symlink(filepath.Join(pool.rootPath, path), dstPath) +} + +// FullPath generates full path to the file in pool +// +// Please use with care: it's not supposed to be used to access files +func (pool *PackagePool) FullPath(path string) string { + return filepath.Join(pool.rootPath, path) +} + // GenerateTempPath generates temporary path for download (which is fast to import into package pool later on) func (pool *PackagePool) GenerateTempPath(filename string) (string, error) { random := uuid.NewRandom().String() diff --git a/files/package_pool_test.go b/files/package_pool_test.go index 3b9e583f..1734b370 100644 --- a/files/package_pool_test.go +++ b/files/package_pool_test.go @@ -189,6 +189,24 @@ func (s *PackagePoolSuite) TestLink(c *C) { c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true) } +func (s *PackagePoolSuite) TestSymlink(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) + c.Check(err, IsNil) + + tmpDir := c.MkDir() + dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile)) + c.Check(s.pool.Symlink(path, dstPath), IsNil) + + info, err := os.Stat(dstPath) + c.Assert(err, IsNil) + c.Check(info.Size(), Equals, int64(2738)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true) + + info, err = os.Lstat(dstPath) + c.Assert(err, IsNil) + c.Check(info.Sys().(*syscall.Stat_t).Mode&syscall.S_IFMT, Equals, uint16(syscall.S_IFLNK)) +} + func (s *PackagePoolSuite) TestGenerateRandomPath(c *C) { path, err := s.pool.GenerateTempPath("a.deb") c.Check(err, IsNil)