diff --git a/deb/remote_test.go b/deb/remote_test.go index deb32dd3..c632f3c9 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -329,6 +329,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile) downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404}) downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404}) downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile) err := s.flat.Fetch(downloader, nil) @@ -359,9 +360,11 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile) downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404}) downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404}) downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile) downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.HTTPError{Code: 404}) downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.HTTPError{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.HTTPError{Code: 404}) downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile) err := s.flat.Fetch(downloader, nil) diff --git a/http/download.go b/http/download.go index 77af8703..55e7eebb 100644 --- a/http/download.go +++ b/http/download.go @@ -4,16 +4,19 @@ import ( "compress/bzip2" "compress/gzip" "fmt" - "github.com/mxk/go-flowrate/flowrate" - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/utils" - "github.com/smira/go-ftp-protocol/protocol" "io" "io/ioutil" "net/http" "os" "path/filepath" "strings" + "time" + + "github.com/mxk/go-flowrate/flowrate" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/utils" + "github.com/smira/go-ftp-protocol/protocol" + "github.com/smira/go-xz" ) // HTTPError is download error connected to HTTP code @@ -58,8 +61,13 @@ type downloadTask struct { // NewDownloader creates new instance of Downloader which specified number // of threads and download limit in bytes/sec func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader { - transport := *http.DefaultTransport.(*http.Transport) + transport := http.Transport{} + transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy + transport.ResponseHeaderTimeout = 30 * time.Second + transport.TLSHandshakeTimeout = http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout + transport.ExpectContinueTimeout = http.DefaultTransport.(*http.Transport).ExpectContinueTimeout transport.DisableCompression = true + initTransport(&transport) transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{}) downloader := &downloaderImpl{ @@ -134,7 +142,7 @@ func (downloader *downloaderImpl) Download(url string, destination string, resul // DownloadWithChecksum starts new download task with checksum verification func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) { - downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries} + downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries} } // handleTask processes single download task @@ -154,7 +162,6 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) { req.URL.RawQuery = "" } - var temppath string for task.triesLeft > 0 { @@ -322,6 +329,10 @@ var compressionMethods = []struct { extenstion: ".gz", transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) }, }, + { + extenstion: ".xz", + transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) }, + }, { extenstion: "", transformation: func(r io.Reader) (io.Reader, error) { return r, nil }, diff --git a/http/download_go16.go b/http/download_go16.go new file mode 100644 index 00000000..f4df5156 --- /dev/null +++ b/http/download_go16.go @@ -0,0 +1,11 @@ +// +build !go1.7 + +package http + +import ( + "net/http" +) + +func initTransport(transport *http.Transport) { + +} diff --git a/http/download_go17.go b/http/download_go17.go new file mode 100644 index 00000000..a69a210c --- /dev/null +++ b/http/download_go17.go @@ -0,0 +1,13 @@ +// +build go1.7 + +package http + +import ( + "net/http" +) + +func initTransport(transport *http.Transport) { + transport.DialContext = http.DefaultTransport.(*http.Transport).DialContext + transport.MaxIdleConns = http.DefaultTransport.(*http.Transport).MaxIdleConns + transport.IdleConnTimeout = http.DefaultTransport.(*http.Transport).IdleConnTimeout +} diff --git a/http/download_test.go b/http/download_test.go index f198449d..b8ab08fb 100644 --- a/http/download_test.go +++ b/http/download_test.go @@ -3,9 +3,6 @@ package http import ( "errors" "fmt" - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/console" - "github.com/smira/aptly/utils" "io" "io/ioutil" "net" @@ -14,6 +11,10 @@ import ( "runtime" "time" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/console" + "github.com/smira/aptly/utils" + . "gopkg.in/check.v1" ) @@ -204,6 +205,7 @@ func (s *DownloaderSuite) TestDownloadTempError(c *C) { const ( bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00" gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00" + xzData = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a" rawData = "test" ) @@ -213,6 +215,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { expectedChecksums := map[string]utils.ChecksumInfo{ "file.bz2": {Size: int64(len(bzipData))}, "file.gz": {Size: int64(len(gzipData))}, + "file.xz": {Size: int64(len(xzData))}, "file": {Size: int64(len(rawData))}, } @@ -239,11 +242,25 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { c.Assert(string(buf), Equals, rawData) c.Assert(d.Empty(), Equals, true) - // bzip2 & gzip not available, but raw is + // bzip2 & gzip not available, but xz is buf = make([]byte, 4) d = NewFakeDownloader() d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404}) d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404}) + d.ExpectResponse("http://example.com/file.xz", xzData) + r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1) + c.Assert(err, IsNil) + defer file.Close() + io.ReadFull(r, buf) + c.Assert(string(buf), Equals, rawData) + c.Assert(d.Empty(), Equals, true) + + // bzip2, gzip & xz not available, but raw is + buf = make([]byte, 4) + d = NewFakeDownloader() + d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404}) + d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404}) + d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404}) d.ExpectResponse("http://example.com/file", rawData) r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1) c.Assert(err, IsNil) @@ -270,6 +287,7 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) { d = NewFakeDownloader() d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404}) d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404}) + d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404}) d.ExpectError("http://example.com/file", errors.New("403")) _, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1) c.Assert(err, ErrorMatches, "403") @@ -277,10 +295,12 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) { d = NewFakeDownloader() d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404}) d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404}) + d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404}) d.ExpectResponse("http://example.com/file", rawData) expectedChecksums := map[string]utils.ChecksumInfo{ "file.bz2": {Size: 7}, "file.gz": {Size: 7}, + "file.xz": {Size: 7}, "file": {Size: 7}, } _, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1) diff --git a/system/t04_mirror/UpdateMirror4Test_gold b/system/t04_mirror/UpdateMirror4Test_gold index 87d17f2a..dc022217 100644 --- a/system/t04_mirror/UpdateMirror4Test_gold +++ b/system/t04_mirror/UpdateMirror4Test_gold @@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release... Downloading & parsing package files... Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2... Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz... +Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz... Downloading ${url}dists/hardy/main/binary-amd64/Packages... WARNING: ${url}dists/hardy/main/binary-amd64/Packages: sha256 hash mismatch "494414ded24da13c451b13b424928821351c78fce49f93d9e1b55f102790c206" != "8a21688ae769f2b4ffcaa366409f679d" ERROR: unable to update: malformed stanza syntax diff --git a/system/t04_mirror/UpdateMirror6Test_gold b/system/t04_mirror/UpdateMirror6Test_gold index 187a151e..de1c5aa5 100644 --- a/system/t04_mirror/UpdateMirror6Test_gold +++ b/system/t04_mirror/UpdateMirror6Test_gold @@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release... Downloading & parsing package files... Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2... Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz... +Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz... Downloading ${url}dists/hardy/main/binary-amd64/Packages... Building download queue... Download queue: 1 items (30 B)