vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-02 04:50:49 +00:00
Code Issues Projects Releases Wiki Activity

Final round of updates, everything except mirror download should be ready

Browse Source
This commit is contained in:
Andrey Smirnov
2017-04-08 23:59:37 +03:00
parent 2535367c3c
commit 72d233b587
14 changed files with 154 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deb/deb.go
+7 -11
View File
@@ -12,7 +12,9 @@ import (
"github.com/h2non/filetype/matchers"
"github.com/mkrautz/goar"
"github.com/pkg/errors"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/smira/go-xz"
"github.com/smira/lzma"
@@ -105,13 +107,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
}
// GetContentsFromDeb returns list of files installed by .deb package
func GetContentsFromDeb(packageFile string) ([]string, error) {
file, err := os.Open(packageFile)
if err != nil {
return nil, err
}
defer file.Close()
func GetContentsFromDeb(file aptly.ReadSeekerCloser, packageFile string) ([]string, error) {
library := ar.NewReader(file)
for {
header, err := library.Next()
@@ -119,7 +115,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
}
if err != nil {
return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
return nil, errors.Wrapf(err, "unable to read .deb archive from %s", packageFile)
}
if strings.HasPrefix(header.Name, "data.tar") {
@@ -142,7 +138,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
} else {
ungzip, err := gzip.NewReader(bufReader)
if err != nil {
return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
}
defer ungzip.Close()
tarInput = ungzip
@@ -152,7 +148,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
case "data.tar.xz":
unxz, err := xz.NewReader(bufReader)
if err != nil {
return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
}
defer unxz.Close()
tarInput = unxz
@@ -172,7 +168,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
return results, nil
}
if err != nil {
return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
return nil, errors.Wrapf(err, "unable to read .tar archive from %s", packageFile)
}
if tarHeader.Typeflag == tar.TypeDir {
deb/deb_test.go
+9 -2
View File
@@ -1,6 +1,7 @@
package deb
import (
"os"
"path/filepath"
"runtime"
@@ -59,13 +60,19 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
}
func (s *DebSuite) TestGetContentsFromDeb(c *C) {
contents, err := GetContentsFromDeb(s.debFile)
f, err := os.Open(s.debFile)
c.Assert(err, IsNil)
contents, err := GetContentsFromDeb(f, s.debFile)
c.Check(err, IsNil)
c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
"usr/share/doc/libboost-program-options-dev/copyright"})
c.Assert(f.Close(), IsNil)
contents, err = GetContentsFromDeb(s.debFile2)
f, err = os.Open(s.debFile2)
c.Assert(err, IsNil)
contents, err = GetContentsFromDeb(f, s.debFile2)
c.Check(err, IsNil)
c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
c.Assert(f.Close(), IsNil)
}
deb/import.go
+15 -11
View File
@@ -116,19 +116,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
continue
}
var files PackageFiles
if isSourcePackage {
files = p.Files()
}
var checksums utils.ChecksumInfo
checksums, err = utils.ChecksumsForFile(file)
if err != nil {
return nil, nil, err
}
if isSourcePackage {
p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
} else {
p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
mainPackageFile := PackageFile{
Filename: filepath.Base(file),
Checksums: checksums,
}
err = pool.Import(file, checksums)
mainPackageFile.PoolPath, err = pool.Import(file, mainPackageFile.Filename, &mainPackageFile.Checksums, false)
if err != nil {
reporter.Warning("Unable to import file %s into pool: %s", file, err)
failedFiles = append(failedFiles, file)
@@ -137,13 +142,10 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
candidateProcessedFiles = append(candidateProcessedFiles, file)
// go over all files, except for the last one (.dsc/.deb itself)
for _, f := range p.Files() {
if filepath.Base(f.Filename) == filepath.Base(file) {
continue
}
// go over all the other files
for _, f := range files {
sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
err = pool.Import(sourceFile, f.Checksums)
f.PoolPath, err = pool.Import(sourceFile, f.Filename, &f.Checksums, false)
if err != nil {
reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
failedFiles = append(failedFiles, file)
@@ -157,6 +159,8 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
continue
}
p.UpdateFiles(append(files, mainPackageFile))
if restriction != nil && !restriction.Matches(p) {
reporter.Warning("%s has been ignored as it doesn't match restriction", p)
failedFiles = append(failedFiles, file)
deb/package.go
+17 -12
View File
@@ -418,7 +418,7 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl
}
file := p.Files()[0]
path, err := packagePool.Path(file.Filename, file.Checksums)
poolPath, err := file.GetPoolPath(packagePool)
if err != nil {
if progress != nil {
progress.ColoredPrintf("@y[!]@| @!Failed to build pool path: @| %s", err)
@@ -426,7 +426,16 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl
return nil, err
}
contents, err := GetContentsFromDeb(path)
reader, err := packagePool.Open(poolPath)
if err != nil {
if progress != nil {
progress.ColoredPrintf("@y[!]@| @!Failed to open package in pool: @| %s", err)
}
return nil, err
}
defer reader.Close()
contents, err := GetContentsFromDeb(reader, file.Filename)
if err != nil {
if progress != nil {
progress.ColoredPrintf("@y[!]@| @!Failed to generate package contents: @| %s", err)
@@ -547,7 +556,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
}
for i, f := range p.Files() {
sourcePath, err := packagePool.Path(f.Filename, f.Checksums)
sourcePoolPath, err := f.GetPoolPath(packagePool)
if err != nil {
return err
}
@@ -555,7 +564,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
relPath := filepath.Join("pool", component, poolDir)
publishedDirectory := filepath.Join(prefix, relPath)
err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums, force)
err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePoolPath, f.Checksums, force)
if err != nil {
return err
}
@@ -604,14 +613,9 @@ type PackageDownloadTask struct {
// DownloadList returns list of missing package files for download in format
// [[srcpath, dstpath]]
func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
result = make([]PackageDownloadTask, 0, 1)
/*result = make([]PackageDownloadTask, 0, 1)
for _, f := range p.Files() {
poolPath, err := packagePool.Path(f.Filename, f.Checksums)
if err != nil {
return nil, err
}
verified, err := f.Verify(packagePool)
if err != nil {
return nil, err
@@ -622,7 +626,8 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD
}
}
return result, nil
return result, nil*/
panic("NEEDS REWORK")
}
// VerifyFiles verifies that all package files have neen correctly downloaded
@@ -645,7 +650,7 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error)
result := make([]string, len(p.Files()))
for i, f := range p.Files() {
result[i], err = packagePool.RelativePath(f.Filename, f.Checksums)
result[i], err = f.GetPoolPath(packagePool)
if err != nil {
return nil, err
}
deb/package_files.go
+17 -3
View File
@@ -4,7 +4,6 @@ import (
"encoding/binary"
"fmt"
"hash/fnv"
"os"
"path/filepath"
"sort"
"strconv"
@@ -20,18 +19,20 @@ type PackageFile struct {
Filename string
// Hashes for the file
Checksums utils.ChecksumInfo
// PoolPath persists relative path to file in the package pool
PoolPath string
// Temporary field used while downloading, stored relative path on the mirror
downloadPath string
}
// Verify that package file is present and correct
func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
poolPath, err := packagePool.Path(f.Filename, f.Checksums)
poolPath, err := f.GetPoolPath(packagePool)
if err != nil {
return false, err
}
st, err := os.Stat(poolPath)
st, err := packagePool.Stat(poolPath)
if err != nil {
return false, nil
}
@@ -41,6 +42,19 @@ func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
return st.Size() == f.Checksums.Size, nil
}
// GetPoolPath returns path to the file in the pool
//
// For legacy packages which do not have PoolPath field set, that calculates LegacyPath via pool
func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) {
var err error
if f.PoolPath == "" {
f.PoolPath, err = packagePool.LegacyPath(f.Filename, &f.Checksums)
}
return f.PoolPath, err
}
// DownloadURL return relative URL to package download location
func (f *PackageFile) DownloadURL() string {
return filepath.Join(f.downloadPath, f.Filename)
deb/package_files_test.go
+4 -8
View File
@@ -1,7 +1,7 @@
package deb
import (
"os"
"io/ioutil"
"path/filepath"
"github.com/smira/aptly/files"
@@ -30,19 +30,15 @@ func (s *PackageFilesSuite) SetUpTest(c *C) {
func (s *PackageFilesSuite) TestVerify(c *C) {
packagePool := files.NewPackagePool(c.MkDir())
poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums)
result, err := s.files[0].Verify(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, false)
err = os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
file, err := os.Create(poolPath)
c.Assert(err, IsNil)
file.WriteString("abcde")
file.Close()
s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false)
result, err = s.files[0].Verify(packagePool)
c.Check(err, IsNil)
deb/package_test.go
+33 -39
View File
@@ -2,12 +2,11 @@ package deb
import (
"bytes"
"os"
"io/ioutil"
"path/filepath"
"regexp"
"github.com/smira/aptly/files"
"github.com/smira/aptly/utils"
. "gopkg.in/check.v1"
)
@@ -367,15 +366,12 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
publishedStorage := files.NewPublishedStorage(c.MkDir(), "", "")
p := NewPackageFromControlFile(s.stanza)
poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
file, err := os.Create(poolPath)
c.Assert(err, IsNil)
file.Close()
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false)
err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
c.Check(err, IsNil)
c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
@@ -396,48 +392,46 @@ func (s *PackageSuite) TestFilepathList(c *C) {
}
func (s *PackageSuite) TestDownloadList(c *C) {
packagePool := files.NewPackagePool(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
p.Files()[0].Checksums.Size = 5
poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums)
c.Fail()
/*
packagePool := files.NewPackagePool(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
p.Files()[0].Checksums.Size = 5
poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums)
list, err := p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{
{
RepoURI: "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
DestinationPath: poolPath,
Checksums: utils.ChecksumInfo{Size: 5,
MD5: "1e8cba92c41420aa7baa8a5718d67122",
SHA1: "46955e48cad27410a83740a21d766ce362364024",
SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
list, err := p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{
{
RepoURI: "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
DestinationPath: poolPath,
Checksums: utils.ChecksumInfo{Size: 5,
MD5: "1e8cba92c41420aa7baa8a5718d67122",
SHA1: "46955e48cad27410a83740a21d766ce362364024",
SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
err = os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
err = os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
file, err := os.Create(poolPath)
c.Assert(err, IsNil)
file.WriteString("abcde")
file.Close()
file, err := os.Create(poolPath)
c.Assert(err, IsNil)
file.WriteString("abcde")
file.Close()
list, err = p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{})
list, err = p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{})*/
}
func (s *PackageSuite) TestVerifyFiles(c *C) {
p := NewPackageFromControlFile(s.stanza)
packagePool := files.NewPackagePool(c.MkDir())
poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
file, err := os.Create(poolPath)
c.Assert(err, IsNil)
file.WriteString("abcde")
file.Close()
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false)
result, err := p.VerifyFiles(packagePool)
c.Check(err, IsNil)
deb/publish_test.go
+4 -5
View File
@@ -132,11 +132,10 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo5.SkipContents = true
poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
f, err := os.Create(poolPath)
c.Assert(err, IsNil)
f.Close()
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
s.p1.Files()[0].PoolPath, _ = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false)
}
func (s *PublishedRepoSuite) TearDownTest(c *C) {