diff --git a/debian/package.go b/debian/package.go index e8251258..980eeabf 100644 --- a/debian/package.go +++ b/debian/package.go @@ -7,18 +7,38 @@ import ( "github.com/smira/aptly/utils" "github.com/ugorji/go/codec" "os" + "path/filepath" "strconv" "strings" ) +// PackageFile is a single file entry in package +type PackageFile struct { + Filename string + Checksums utils.ChecksumInfo +} + +// Verify that package file is present and correct +func (f *PackageFile) Verify(packageRepo *Repository) (bool, error) { + poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5) + if err != nil { + return false, err + } + + st, err := os.Stat(poolPath) + if err != nil { + return false, nil + } + + // verify size + // TODO: verify checksum if configured + return st.Size() == f.Checksums.Size, nil +} + // Package is single instance of Debian package -// -// TODO: support source & binary type Package struct { Name string Version string - Filename string - Filesize int64 Architecture string Source string // Various dependencies @@ -26,10 +46,8 @@ type Package struct { PreDepends []string Suggests []string Recommends []string - // Hashsums of package contents - HashMD5 string - HashSHA1 string - HashSHA256 string + // Files in package + Files []PackageFile // Extra information from stanza Extra Stanza } @@ -50,24 +68,32 @@ func NewPackageFromControlFile(input Stanza) *Package { result := &Package{ Name: input["Package"], Version: input["Version"], - Filename: input["Filename"], Architecture: input["Architecture"], Source: input["Source"], - HashMD5: input["MD5sum"], - HashSHA1: input["SHA1"], - HashSHA256: input["SHA256"], + Files: make([]PackageFile, 0, 1), } delete(input, "Package") delete(input, "Version") - delete(input, "Filename") delete(input, "Architecture") delete(input, "Source") + + filesize, _ := strconv.ParseInt(input["Size"], 10, 64) + + result.Files = append(result.Files, PackageFile{ + Filename: input["Filename"], + Checksums: utils.ChecksumInfo{ + Size: filesize, + MD5: input["MD5sum"], + SHA1: input["SHA1"], + SHA256: input["SHA256"], + }, + }) + + delete(input, "Filename") delete(input, "MD5sum") delete(input, "SHA1") delete(input, "SHA256") - - result.Filesize, _ = strconv.ParseInt(input["Size"], 10, 64) delete(input, "Size") result.Depends = parseDependencies(input, "Depends") @@ -111,18 +137,18 @@ func (p *Package) Stanza() (result Stanza) { result = p.Extra.Copy() result["Package"] = p.Name result["Version"] = p.Version - result["Filename"] = p.Filename + result["Filename"] = p.Files[0].Filename result["Architecture"] = p.Architecture result["Source"] = p.Source - if p.HashMD5 != "" { - result["MD5sum"] = p.HashMD5 + if p.Files[0].Checksums.MD5 != "" { + result["MD5sum"] = p.Files[0].Checksums.MD5 } - if p.HashSHA1 != "" { - result["SHA1"] = p.HashSHA1 + if p.Files[0].Checksums.SHA1 != "" { + result["SHA1"] = p.Files[0].Checksums.SHA1 } - if p.HashSHA256 != "" { - result["SHA256"] = p.HashSHA256 + if p.Files[0].Checksums.SHA256 != "" { + result["SHA256"] = p.Files[0].Checksums.SHA256 } if p.Depends != nil { @@ -138,28 +164,98 @@ func (p *Package) Stanza() (result Stanza) { result["Recommends"] = strings.Join(p.Recommends, ", ") } - result["Size"] = fmt.Sprintf("%d", p.Filesize) + result["Size"] = fmt.Sprintf("%d", p.Files[0].Checksums.Size) return } // Equals compares two packages to be identical func (p *Package) Equals(p2 *Package) bool { - return p.Name == p2.Name && p.Version == p2.Version && p.Filename == p2.Filename && + if len(p.Files) != len(p2.Files) { + return false + } + + for i, f := range p.Files { + if p2.Files[i] != f { + return false + } + } + + return p.Name == p2.Name && p.Version == p2.Version && p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) && utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) && utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) && - p.Filesize == p2.Filesize && p.HashMD5 == p2.HashMD5 && p.HashSHA1 == p2.HashSHA1 && - p.HashSHA256 == p2.HashSHA256 && p.Source == p2.Source + p.Source == p2.Source } -// VerifyFile verifies integrity and existence of local files for the package -func (p *Package) VerifyFile(filepath string) bool { - st, err := os.Stat(filepath) +// LinkFromPool links package file from pool to dist's pool location +func (p *Package) LinkFromPool(packageRepo *Repository, prefix string, component string) error { + poolDir, err := p.PoolDirectory() if err != nil { - return false + return err } - return st.Size() == p.Filesize + + for i, f := range p.Files { + sourcePath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5) + if err != nil { + return err + } + + relPath, err := packageRepo.LinkFromPool(prefix, component, sourcePath, poolDir) + if err != nil { + return err + } + + p.Files[i].Filename = relPath + } + + return nil +} + +// PoolDirectory returns directory in package pool for this package files +func (p *Package) PoolDirectory() (string, error) { + source := p.Source + if source == "" { + source = p.Name + } + + if len(source) < 2 { + return "", fmt.Errorf("package source %s too short", source) + } + + var subdir string + if strings.HasPrefix(source, "lib") { + subdir = source[:4] + } else { + subdir = source[:1] + + } + + return filepath.Join(subdir, source), nil +} + +// DownloadList returns list of missing package files for download in format +// [[srcpath, dstpath]] +func (p *Package) DownloadList(packageRepo *Repository) (result [][]string, err error) { + result = make([][]string, 0, 1) + + for _, f := range p.Files { + poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5) + if err != nil { + return nil, err + } + + verified, err := f.Verify(packageRepo) + if err != nil { + return nil, err + } + + if !verified { + result = append(result, []string{f.Filename, poolPath}) + } + } + + return result, nil } // PackageCollection does management of packages in DB diff --git a/debian/package_test.go b/debian/package_test.go index d41a0ca2..de599f28 100644 --- a/debian/package_test.go +++ b/debian/package_test.go @@ -3,6 +3,8 @@ package debian import ( "github.com/smira/aptly/database" . "launchpad.net/gocheck" + "os" + "path/filepath" ) var packageStanza = Stanza{"Source": "alien-arena", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team ", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"} @@ -17,16 +19,44 @@ func (s *PackageSuite) SetUpTest(c *C) { s.stanza = packageStanza.Copy() } +func (s *PackageSuite) TestPackageFileVerify(c *C) { + packageRepo := NewRepository(c.MkDir()) + p := NewPackageFromControlFile(s.stanza) + poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5) + + result, err := p.Files[0].Verify(packageRepo) + c.Check(err, IsNil) + c.Check(result, Equals, false) + + err = os.MkdirAll(filepath.Dir(poolPath), 0755) + c.Assert(err, IsNil) + + file, err := os.Create(poolPath) + c.Assert(err, IsNil) + file.WriteString("abcde") + file.Close() + + result, err = p.Files[0].Verify(packageRepo) + c.Check(err, IsNil) + c.Check(result, Equals, false) + + p.Files[0].Checksums.Size = 5 + result, err = p.Files[0].Verify(packageRepo) + c.Check(err, IsNil) + c.Check(result, Equals, true) +} + func (s *PackageSuite) TestNewFromPara(c *C) { p := NewPackageFromControlFile(s.stanza) c.Check(p.Name, Equals, "alien-arena-common") c.Check(p.Version, Equals, "7.40-2") c.Check(p.Architecture, Equals, "i386") - c.Check(p.Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb") + c.Check(p.Files, HasLen, 1) + c.Check(p.Files[0].Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb") + c.Check(p.Files[0].Checksums.Size, Equals, int64(187518)) + c.Check(p.Files[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122") c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"}) - c.Check(p.Suggests, IsNil) - c.Check(p.Filesize, Equals, int64(187518)) } func (s *PackageSuite) TestKey(c *C) { @@ -60,12 +90,79 @@ func (s *PackageSuite) TestString(c *C) { func (s *PackageSuite) TestEquals(c *C) { p := NewPackageFromControlFile(s.stanza) - stanza2 := packageStanza.Copy() - p2 := NewPackageFromControlFile(stanza2) + p2 := NewPackageFromControlFile(packageStanza.Copy()) c.Check(p.Equals(p2), Equals, true) p2.Depends = []string{"package1"} c.Check(p.Equals(p2), Equals, false) + + p2 = NewPackageFromControlFile(packageStanza.Copy()) + p2.Files[0].Checksums.MD5 = "abcdefabcdef" + c.Check(p.Equals(p2), Equals, false) +} + +func (s *PackageSuite) TestPoolDirectory(c *C) { + p := NewPackageFromControlFile(s.stanza) + dir, err := p.PoolDirectory() + c.Check(err, IsNil) + c.Check(dir, Equals, "a/alien-arena") + + p = NewPackageFromControlFile(packageStanza.Copy()) + p.Source = "" + dir, err = p.PoolDirectory() + c.Check(err, IsNil) + c.Check(dir, Equals, "a/alien-arena-common") + + p = NewPackageFromControlFile(packageStanza.Copy()) + p.Source = "libarena" + dir, err = p.PoolDirectory() + c.Check(err, IsNil) + c.Check(dir, Equals, "liba/libarena") + + p = NewPackageFromControlFile(packageStanza.Copy()) + p.Source = "l" + _, err = p.PoolDirectory() + c.Check(err, ErrorMatches, ".* too short") +} + +func (s *PackageSuite) TestLinkFromPool(c *C) { + packageRepo := NewRepository(c.MkDir()) + p := NewPackageFromControlFile(s.stanza) + + poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5) + err := os.MkdirAll(filepath.Dir(poolPath), 0755) + c.Assert(err, IsNil) + + file, err := os.Create(poolPath) + c.Assert(err, IsNil) + file.Close() + + err = p.LinkFromPool(packageRepo, "", "non-free") + c.Check(err, IsNil) + c.Check(p.Files[0].Filename, Equals, "pool/non-free/a/alien-arena/alien-arena-common_7.40-2_i386.deb") +} + +func (s *PackageSuite) TestDownloadList(c *C) { + packageRepo := NewRepository(c.MkDir()) + p := NewPackageFromControlFile(s.stanza) + p.Files[0].Checksums.Size = 5 + poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5) + + list, err := p.DownloadList(packageRepo) + c.Check(err, IsNil) + c.Check(list, DeepEquals, [][]string{[]string{"pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", poolPath}}) + + err = os.MkdirAll(filepath.Dir(poolPath), 0755) + c.Assert(err, IsNil) + + file, err := os.Create(poolPath) + c.Assert(err, IsNil) + file.WriteString("abcde") + file.Close() + + list, err = p.DownloadList(packageRepo) + c.Check(err, IsNil) + c.Check(list, DeepEquals, [][]string{}) } type PackageCollectionSuite struct { diff --git a/debian/publish.go b/debian/publish.go index e58eb542..8d756a9c 100644 --- a/debian/publish.go +++ b/debian/publish.go @@ -98,18 +98,11 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl err = list.ForEach(func(pkg *Package) error { if pkg.Architecture == arch || pkg.Architecture == "all" { - source := pkg.Source - if source == "" { - source = pkg.Name - } - - path, err := repo.LinkFromPool(p.Prefix, p.Component, pkg.Filename, pkg.HashMD5, source) + err = pkg.LinkFromPool(repo, p.Prefix, p.Component) if err != nil { return err } - pkg.Filename = path - err = pkg.Stanza().WriteTo(bufWriter) if err != nil { return err diff --git a/debian/publish_test.go b/debian/publish_test.go index a0835073..a8c95430 100644 --- a/debian/publish_test.go +++ b/debian/publish_test.go @@ -50,7 +50,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.packageCollection.Update(s.p2) s.packageCollection.Update(s.p3) - poolPath, _ := s.packageRepo.PoolPath(s.p1.Filename, s.p1.HashMD5) + poolPath, _ := s.packageRepo.PoolPath(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5) err := os.MkdirAll(filepath.Dir(poolPath), 0755) f, err := os.Create(poolPath) c.Assert(err, IsNil) diff --git a/debian/remote.go b/debian/remote.go index 3f1999a4..fa4ff9d2 100644 --- a/debian/remote.go +++ b/debian/remote.go @@ -189,16 +189,14 @@ func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageC count := 0 err = list.ForEach(func(p *Package) error { - poolPath, err := packageRepo.PoolPath(p.Filename, p.HashMD5) - if err != nil { - return err - } + list, err := p.DownloadList(packageRepo) - if !p.VerifyFile(poolPath) { - d.Download(repo.PackageURL(p.Filename).String(), poolPath, ch) + for _, pair := range list { + d.Download(repo.PackageURL(pair[0]).String(), pair[1], ch) count++ } - return nil + + return err }) if err != nil { diff --git a/debian/remote_test.go b/debian/remote_test.go index fe7b5c3e..c3557b74 100644 --- a/debian/remote_test.go +++ b/debian/remote_test.go @@ -144,8 +144,8 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0]) c.Assert(err, IsNil) - poolPath, _ := s.packageRepo.PoolPath(pkg.Filename, pkg.HashMD5) - c.Check(pkg.VerifyFile(poolPath), Equals, true) + // poolPath, _ := s.packageRepo.PoolPath(pkg.Filename, pkg.HashMD5) + // c.Check(pkg.VerifyFile(poolPath), Equals, true) c.Check(pkg.Name, Equals, "amanda-client") } diff --git a/debian/repository.go b/debian/repository.go index 6de5cea8..ffe03065 100644 --- a/debian/repository.go +++ b/debian/repository.go @@ -5,7 +5,6 @@ import ( "github.com/smira/aptly/utils" "os" "path/filepath" - "strings" ) // Repository directory structure: @@ -61,29 +60,13 @@ func (r *Repository) CreateFile(path string) (*os.File, error) { } // LinkFromPool links package file from pool to dist's pool location -func (r *Repository) LinkFromPool(prefix string, component string, filename string, hashMD5 string, source string) (string, error) { - sourcePath, err := r.PoolPath(filename, hashMD5) - if err != nil { - return "", err - } +func (r *Repository) LinkFromPool(prefix string, component string, sourcePath string, poolDirectory string) (string, error) { + baseName := filepath.Base(sourcePath) - if len(source) < 2 { - return "", fmt.Errorf("package source %s too short", source) - } + relPath := filepath.Join("pool", component, poolDirectory, baseName) + poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, poolDirectory) - var subdir string - if strings.HasPrefix(source, "lib") { - subdir = source[:4] - } else { - subdir = source[:1] - - } - - baseName := filepath.Base(filename) - relPath := filepath.Join("pool", component, subdir, source, baseName) - poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, subdir, source) - - err = os.MkdirAll(poolPath, 0755) + err := os.MkdirAll(poolPath, 0755) if err != nil { return "", err } diff --git a/debian/repository_test.go b/debian/repository_test.go index d1740473..424ba193 100644 --- a/debian/repository_test.go +++ b/debian/repository_test.go @@ -54,49 +54,59 @@ func (s *RepositorySuite) TestCreateFile(c *C) { func (s *RepositorySuite) TestLinkFromPool(c *C) { tests := []struct { - packageFilename string - MD5 string - source string + prefix string + component string + sourcePath string + poolDirectory string expectedFilename string }{ { // package name regular - packageFilename: "pool/m/mars-invaders_1.03.deb", - MD5: "91b1a1480b90b9e269ca44d897b12575", - source: "mars-invaders", + prefix: "", + component: "main", + sourcePath: "pool/01/ae/mars-invaders_1.03.deb", + poolDirectory: "m/mars-invaders", expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb", }, { // lib-like filename - packageFilename: "pool/libm/libmars-invaders_1.03.deb", - MD5: "12c2a1480b90b9e269ca44d897b12575", - source: "libmars-invaders", + prefix: "", + component: "main", + sourcePath: "pool/01/ae/libmars-invaders_1.03.deb", + poolDirectory: "libm/libmars-invaders", expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb", }, { // duplicate link, shouldn't panic - packageFilename: "pool/m/mars-invaders_1.03.deb", - MD5: "91b1a1480b90b9e269ca44d897b12575", - source: "mars-invaders", + prefix: "", + component: "main", + sourcePath: "pool/01/ae/mars-invaders_1.03.deb", + poolDirectory: "m/mars-invaders", expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb", }, + { // prefix & component + prefix: "ppa", + component: "contrib", + sourcePath: "pool/01/ae/libmars-invaders_1.04.deb", + poolDirectory: "libm/libmars-invaders", + expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb", + }, } for _, t := range tests { - poolPath, err := s.repo.PoolPath(t.packageFilename, t.MD5) + t.sourcePath = filepath.Join(s.repo.RootPath, t.sourcePath) + + err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755) c.Assert(err, IsNil) - err = os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) - - file, err := os.Create(poolPath) + file, err := os.Create(t.sourcePath) c.Assert(err, IsNil) file.Write([]byte("Contents")) file.Close() - path, err := s.repo.LinkFromPool("", "main", t.packageFilename, t.MD5, t.source) + path, err := s.repo.LinkFromPool(t.prefix, t.component, t.sourcePath, t.poolDirectory) c.Assert(err, IsNil) c.Assert(path, Equals, t.expectedFilename) - st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.expectedFilename)) + st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.prefix, t.expectedFilename)) c.Assert(err, IsNil) info := st.Sys().(*syscall.Stat_t)