When downloading/importing packages, enforce all checksums

deb/package_files_test.go

@@ -40,6 +40,7 @@ func (s *PackageFilesSuite) TestVerify(c *C) {
 
 	s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false)
 
+	s.files[0].Checksums.Size = 187518
 	result, err = s.files[0].Verify(packagePool)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)

deb/package_test.go

@@ -423,6 +423,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false)
 
+	p.Files()[0].Checksums.Size = 100
 	result, err := p.VerifyFiles(packagePool)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)

deb/publish_test.go

@@ -98,6 +98,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 		"files:other": s.publishedStorage2}}
 	s.packagePool = files.NewPackagePool(s.root)
 
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+
+	var err error
+	s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false)
+	c.Assert(err, IsNil)
+
+	s.p1.UpdateFiles(s.p1.Files())
+	s.p2.UpdateFiles(s.p1.Files())
+	s.p3.UpdateFiles(s.p1.Files())
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
+
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
@@ -131,11 +144,6 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	s.repo5.SkipContents = true
-
-	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
-
-	s.p1.Files()[0].PoolPath, _ = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false)
 }
 
 func (s *PublishedRepoSuite) TearDownTest(c *C) {

deb/remote.go

@@ -542,7 +542,6 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, skipEx
 // FinalizeDownload swaps for final value of package refs
 func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
 	repo.LastDownloadDate = time.Now()
-	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
 
 	if progress != nil {
 		progress.InitBar(int64(repo.packageList.Len()), true)
@@ -556,9 +555,13 @@ func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, p
 		if progress != nil {
 			progress.SetBar(i)
 		}
+		// download process might have udpated checksums
+		p.UpdateFiles(p.Files())
 		return collectionFactory.PackageCollection().Update(p)
 	})
 
+	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+
 	if progress != nil {
 		progress.ShutdownBar()
 	}

files/package_pool.go

@@ -142,6 +142,14 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 		return "", err
 	}
 
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.Size != sourceInfo.Size() {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
 	// build target path
 	// TODO: replace with new build scheme
 	poolPath, err := pool.LegacyPath(basename, checksums)

files/package_pool_test.go

@@ -24,7 +24,7 @@ var _ = Suite(&PackagePoolSuite{})
 func (s *PackagePoolSuite) SetUpTest(c *C) {
 	s.pool = NewPackagePool(c.MkDir())
 	s.checksum = utils.ChecksumInfo{
-		MD5: "91b1a1480b90b9e269ca44d897b12575",
+		MD5: "0035d7822b2f8f0ec4013f270fd650c2",
 	}
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
@@ -33,7 +33,7 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
 func (s *PackagePoolSuite) TestLegacyPath(c *C) {
 	path, err := s.pool.LegacyPath("a/b/package.deb", &s.checksum)
 	c.Assert(err, IsNil)
-	c.Assert(path, Equals, "91/b1/package.deb")
+	c.Assert(path, Equals, "00/35/package.deb")
 
 	_, err = s.pool.LegacyPath("/", &s.checksum)
 	c.Assert(err, ErrorMatches, ".*is invalid")
@@ -91,7 +91,9 @@ func (s *PackagePoolSuite) TestRemove(c *C) {
 func (s *PackagePoolSuite) TestImportOk(c *C) {
 	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false)
 	c.Check(err, IsNil)
-	c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb")
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(s.checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
 
 	info, err := s.pool.Stat(path)
 	c.Assert(err, IsNil)
@@ -101,12 +103,12 @@ func (s *PackagePoolSuite) TestImportOk(c *C) {
 	// import as different name
 	path, err = s.pool.Import(s.debFile, "some.deb", &s.checksum, false)
 	c.Check(err, IsNil)
-	c.Check(path, Equals, "91/b1/some.deb")
+	c.Check(path, Equals, "00/35/some.deb")
 
 	// double import, should be ok
 	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false)
 	c.Check(err, IsNil)
-	c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb")
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
 }
 
 func (s *PackagePoolSuite) TestImportMove(c *C) {
@@ -127,7 +129,7 @@ func (s *PackagePoolSuite) TestImportMove(c *C) {
 
 	path, err := s.pool.Import(tmpPath, filepath.Base(tmpPath), &s.checksum, true)
 	c.Check(err, IsNil)
-	c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb")
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
 
 	info, err := s.pool.Stat(path)
 	c.Assert(err, IsNil)
@@ -141,8 +143,8 @@ func (s *PackagePoolSuite) TestImportNotExist(c *C) {
 }
 
 func (s *PackagePoolSuite) TestImportOverwrite(c *C) {
-	os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "00", "35"), 0755)
-	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "00", "35", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
 
 	_, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false)
 	c.Check(err, ErrorMatches, "unable to import into pool.*")

http/download.go

@@ -173,6 +173,9 @@ func (downloader *downloaderImpl) download(req *http.Request, url, destination s
 				os.Remove(temppath)
 				return "", err
 			}
+		} else {
+			// update checksums if they match, so that they contain exactly expected set
+			*expected = actual
 		}
 	}
 

http/download_test.go

@@ -98,9 +98,12 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1),
 		ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1),
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
 		IsNil)
+	// download backfills missing checksums
+	c.Check(checksums.SHA512, Equals, "bac18bf4e564856369acc2ed57300fecba3a2c1af5ae8304021e4252488678feb18118466382ee4e1210fe1f065080210e453a80cfb37ccb8752af3269df160e")
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {

system/t06_publish/PublishSnapshot35Test_packages_amd64

@@ -58,6 +58,10 @@ SHA256: 27760f636f6dbfe387dfbede1131fe7a0dd5fd3b0ab562213193ffa7cfcadfb5
 SHA256: 2920249908a8297f85006def6a55fb99abfcc8466cac2b9f28d01ce8315df065
 SHA256: 8361f45f51a7e70e3367e5b2df59fa8defc8648a76afa4159da3f249460f5b33
 SHA256: b626c3320c0ba2c41c5214bf8175c713f3713cc393e9361a977dc0202c197875
+SHA512: 7223dbbf4a847c48f040c3b7dd1e9f3b45b1a837794aa8368456cc2d522a21180751755a6ea0d919f8cb4b092d742e525e138c6ea9185aedea55bc1d85475e76
+SHA512: ad84c2798fd91f94782f08f371007e62fe933bcd079041874e3e54d6e91a66f953a8b8da2d563e88d800fd9819353362f0e700701ddcf5b44f490123e57f8b38
+SHA512: adda521ac2837bba7bf4e4a6a633f79187dff5ec3806d1bbe97544ac0a00024a41583f70434362755ef33aa788f79c7dc1a0bdb3daaf9a0465d1b90600ba30ee
+SHA512: bd1d7374808541d85fdb965c91b1ac2d0a98580cfac524d2bbdd6021fa6689a54ce7effd9c899237300b2c4966a824c7ed4aa6a4f6c4cb869a7163702960d68c
 Section: utils
 Section: utils
 Section: utils

system/t06_publish/PublishSnapshot35Test_packages_i386

@@ -58,6 +58,10 @@ SHA256: 27760f636f6dbfe387dfbede1131fe7a0dd5fd3b0ab562213193ffa7cfcadfb5
 SHA256: 6898801e3f3c97a30bef1ee50381479b69360a28807fb63fcce4abef4da1aec7
 SHA256: b852d7681ea328bd8b45140973624781e65d0363961d92bcc2ab0bbf1cc6ed52
 SHA256: cff40c87faea248c77de7d9fc50fcbc80631cd1bc8cec2b1033e0db452e08ea6
+SHA512: 6e2f3b4add560a19154717ad4dea1af9e07ef251403c85c87ef5b30e3f8639d74ab8bb1a27b7e4d3346a66f0c9180c70435557caa4e6eded9bd9010f3f5d7123
+SHA512: adda521ac2837bba7bf4e4a6a633f79187dff5ec3806d1bbe97544ac0a00024a41583f70434362755ef33aa788f79c7dc1a0bdb3daaf9a0465d1b90600ba30ee
+SHA512: e1c69691ceb3afb10ad8287e34ef4af75046f99e8aa51d5f15c1e8ac904377ac44023aed1bd7572ebc64c68aca9f99dbd485e13952f6b65e41cf47598af5e03f
+SHA512: e6277d5e08210fc7258fc239d1715657a4bd9a4c3c190e41a0b3e4d101bd3abfd7b5c87ed8111a1f3efec239b27938a42cd25a582a6f9d93fdb28fc9684cf14c
 Section: utils
 Section: utils
 Section: utils

system/t06_publish/PublishSnapshot35Test_packages_udeb_amd64

@@ -25,6 +25,8 @@ SHA1: 5d32171182e956f8277d44378b1623bbeae23110
 SHA1: dbe121bae44db6eb6108311f41997c4ede1178b2
 SHA256: 4abcb1191d8a3e58d88fb56084f9d784255ba68c767babc3c2819b7a1a689b78
 SHA256: dd7230f9d025c47e8c94e4101e2970e94aed50ec0c65801f9c7cd0a03d6723e1
+SHA512: 520ff1cc4053499609c87329d9458560817c5638bd6871ba3b0598c3cb95420859601aa09728998d8227a4ab29930f4d0474660e26921829641b7bed2751ec5e
+SHA512: a0d5e55d8e183f19111cf3067fa8d434a263bad5bafce39fedaeb3ef7fd97577fef852e4f2bb5e04e27b2a0b79d3e707ff12af5b54879f33f124cdf1626a3dea
 Section: debian-installer
 Section: debian-installer
 Size: 130734

system/t06_publish/PublishSnapshot35Test_packages_udeb_i386

@@ -25,6 +25,8 @@ SHA1: e64cb327e89ba41ba6aaeca7e9e69cf18479ed40
 SHA1: f6937084ae96b269131a08bb365619e704f91d21
 SHA256: 7d86005e0f2a7bdeff3204ccb0e50d6d06b07011621acb56ad322480bd11494c
 SHA256: 96eae21eb31fa79d196dfbec63594f62c39753aad59d02d69bf9495ad486ec01
+SHA512: 0de8d92708fbdd6c14b196124ff4fb8a047daf75b942eae24987a0707293578ca86b2de8d61aad72472e653e4536ec62b83bb60ee0a422f317212bd6159a1753
+SHA512: 5954d3f4d8960a2444f89192d05781087410f296a9d810a6bff2a7bc3955f952a3f063c47d575b0215dd60681d99c3e08852c9e3df027ad94ec448fc1749da57
 Section: debian-installer
 Section: debian-installer
 Size: 125582