Merge pull request #505 from smira/rmedaer-master

Added '.xz' extension in HTTP download.

deb/remote_test.go

@@ -329,6 +329,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
 
 	err := s.flat.Fetch(downloader, nil)
@@ -359,9 +360,11 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.HTTPError{Code: 404})
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.HTTPError{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
 
 	err := s.flat.Fetch(downloader, nil)

http/download.go

@@ -4,16 +4,19 @@ import (
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
-	"github.com/mxk/go-flowrate/flowrate"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/go-ftp-protocol/protocol"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
+
+	"github.com/mxk/go-flowrate/flowrate"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/go-ftp-protocol/protocol"
+	"github.com/smira/go-xz"
 )
 
 // HTTPError is download error connected to HTTP code
@@ -58,8 +61,13 @@ type downloadTask struct {
 // NewDownloader creates new instance of Downloader which specified number
 // of threads and download limit in bytes/sec
 func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
-	transport := *http.DefaultTransport.(*http.Transport)
+	transport := http.Transport{}
+	transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
+	transport.ResponseHeaderTimeout = 30 * time.Second
+	transport.TLSHandshakeTimeout = http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout
+	transport.ExpectContinueTimeout = http.DefaultTransport.(*http.Transport).ExpectContinueTimeout
 	transport.DisableCompression = true
+	initTransport(&transport)
 	transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
 
 	downloader := &downloaderImpl{
@@ -134,7 +142,7 @@ func (downloader *downloaderImpl) Download(url string, destination string, resul
 // DownloadWithChecksum starts new download task with checksum verification
 func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
 	expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
-		downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
+	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
 }
 
 // handleTask processes single download task
@@ -154,7 +162,6 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		req.URL.RawQuery = ""
 	}
 
-
 	var temppath string
 	for task.triesLeft > 0 {
 
@@ -322,6 +329,10 @@ var compressionMethods = []struct {
 		extenstion:     ".gz",
 		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
 	},
+	{
+		extenstion:     ".xz",
+		transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
+	},
 	{
 		extenstion:     "",
 		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },

http/download_go16.go

@@ -0,0 +1,11 @@
+// +build !go1.7
+
+package http
+
+import (
+	"net/http"
+)
+
+func initTransport(transport *http.Transport) {
+
+}

http/download_go17.go

@@ -0,0 +1,13 @@
+// +build go1.7
+
+package http
+
+import (
+	"net/http"
+)
+
+func initTransport(transport *http.Transport) {
+	transport.DialContext = http.DefaultTransport.(*http.Transport).DialContext
+	transport.MaxIdleConns = http.DefaultTransport.(*http.Transport).MaxIdleConns
+	transport.IdleConnTimeout = http.DefaultTransport.(*http.Transport).IdleConnTimeout
+}

http/download_test.go

@@ -3,9 +3,6 @@ package http
 import (
 	"errors"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/utils"
 	"io"
 	"io/ioutil"
 	"net"
@@ -14,6 +11,10 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
@@ -204,6 +205,7 @@ func (s *DownloaderSuite) TestDownloadTempError(c *C) {
 const (
 	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
 	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
+	xzData   = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
 	rawData  = "test"
 )
 
@@ -213,6 +215,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	expectedChecksums := map[string]utils.ChecksumInfo{
 		"file.bz2": {Size: int64(len(bzipData))},
 		"file.gz":  {Size: int64(len(gzipData))},
+		"file.xz":  {Size: int64(len(xzData))},
 		"file":     {Size: int64(len(rawData))},
 	}
 
@@ -239,11 +242,25 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	c.Assert(string(buf), Equals, rawData)
 	c.Assert(d.Empty(), Equals, true)
 
-	// bzip2 & gzip not available, but raw is
+	// bzip2 & gzip not available, but xz is
 	buf = make([]byte, 4)
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
+	d.ExpectResponse("http://example.com/file.xz", xzData)
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2, gzip & xz not available, but raw is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
 	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
@@ -270,6 +287,7 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
 	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
 	c.Assert(err, ErrorMatches, "403")
@@ -277,10 +295,12 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
 	expectedChecksums := map[string]utils.ChecksumInfo{
 		"file.bz2": {Size: 7},
 		"file.gz":  {Size: 7},
+		"file.xz":  {Size: 7},
 		"file":     {Size: 7},
 	}
 	_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)

system/t04_mirror/UpdateMirror4Test_gold

@@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release...
 Downloading & parsing package files...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz...
+Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages...
 WARNING: ${url}dists/hardy/main/binary-amd64/Packages: sha256 hash mismatch "494414ded24da13c451b13b424928821351c78fce49f93d9e1b55f102790c206" != "8a21688ae769f2b4ffcaa366409f679d"
 ERROR: unable to update: malformed stanza syntax

system/t04_mirror/UpdateMirror6Test_gold

@@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release...
 Downloading & parsing package files...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz...
+Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz...
 Downloading ${url}dists/hardy/main/binary-amd64/Packages...
 Building download queue...
 Download queue: 1 items (30 B)