vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-04 05:10:40 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #505 from smira/rmedaer-master

Browse Source 
Added '.xz' extension in HTTP download.
This commit is contained in:
Andrey Smirnov
2017-03-17 00:47:33 +03:00
committed by GitHub
parent ce0001f94c acd8d4a6ea
commit 070347295e
7 changed files with 71 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deb/remote_test.go
+3
View File
@@ -329,6 +329,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404})
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
err := s.flat.Fetch(downloader, nil)
@@ -359,9 +360,11 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.HTTPError{Code: 404})
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.HTTPError{Code: 404})
downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.HTTPError{Code: 404})
downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
err := s.flat.Fetch(downloader, nil)
http/download.go
+18 -7
View File
@@ -4,16 +4,19 @@ import (
"compress/bzip2"
"compress/gzip"
"fmt"
"github.com/mxk/go-flowrate/flowrate"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/smira/go-ftp-protocol/protocol"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/mxk/go-flowrate/flowrate"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/smira/go-ftp-protocol/protocol"
"github.com/smira/go-xz"
)
// HTTPError is download error connected to HTTP code
@@ -58,8 +61,13 @@ type downloadTask struct {
// NewDownloader creates new instance of Downloader which specified number
// of threads and download limit in bytes/sec
func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
transport := *http.DefaultTransport.(*http.Transport)
transport := http.Transport{}
transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
transport.ResponseHeaderTimeout = 30 * time.Second
transport.TLSHandshakeTimeout = http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout
transport.ExpectContinueTimeout = http.DefaultTransport.(*http.Transport).ExpectContinueTimeout
transport.DisableCompression = true
initTransport(&transport)
transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
downloader := &downloaderImpl{
@@ -134,7 +142,7 @@ func (downloader *downloaderImpl) Download(url string, destination string, resul
// DownloadWithChecksum starts new download task with checksum verification
func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
}
// handleTask processes single download task
@@ -154,7 +162,6 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
req.URL.RawQuery = ""
}
var temppath string
for task.triesLeft > 0 {
@@ -322,6 +329,10 @@ var compressionMethods = []struct {
extenstion: ".gz",
transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
},
{
extenstion: ".xz",
transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
},
{
extenstion: "",
transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
http/download_go16.go
+11
View File
@@ -0,0 +1,11 @@
// +build !go1.7
package http
import (
"net/http"
)
func initTransport(transport *http.Transport) {
}
http/download_go17.go
+13
View File
@@ -0,0 +1,13 @@
// +build go1.7
package http
import (
"net/http"
)
func initTransport(transport *http.Transport) {
transport.DialContext = http.DefaultTransport.(*http.Transport).DialContext
transport.MaxIdleConns = http.DefaultTransport.(*http.Transport).MaxIdleConns
transport.IdleConnTimeout = http.DefaultTransport.(*http.Transport).IdleConnTimeout
}
http/download_test.go
+24 -4
View File
@@ -3,9 +3,6 @@ package http
import (
"errors"
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/console"
"github.com/smira/aptly/utils"
"io"
"io/ioutil"
"net"
@@ -14,6 +11,10 @@ import (
"runtime"
"time"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/console"
"github.com/smira/aptly/utils"
. "gopkg.in/check.v1"
)
@@ -204,6 +205,7 @@ func (s *DownloaderSuite) TestDownloadTempError(c *C) {
const (
bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
xzData = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
rawData = "test"
)
@@ -213,6 +215,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
expectedChecksums := map[string]utils.ChecksumInfo{
"file.bz2": {Size: int64(len(bzipData))},
"file.gz": {Size: int64(len(gzipData))},
"file.xz": {Size: int64(len(xzData))},
"file": {Size: int64(len(rawData))},
}
@@ -239,11 +242,25 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
c.Assert(string(buf), Equals, rawData)
c.Assert(d.Empty(), Equals, true)
// bzip2 & gzip not available, but raw is
// bzip2 & gzip not available, but xz is
buf = make([]byte, 4)
d = NewFakeDownloader()
d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
d.ExpectResponse("http://example.com/file.xz", xzData)
r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
c.Assert(err, IsNil)
defer file.Close()
io.ReadFull(r, buf)
c.Assert(string(buf), Equals, rawData)
c.Assert(d.Empty(), Equals, true)
// bzip2, gzip & xz not available, but raw is
buf = make([]byte, 4)
d = NewFakeDownloader()
d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
d.ExpectResponse("http://example.com/file", rawData)
r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
c.Assert(err, IsNil)
@@ -270,6 +287,7 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
d = NewFakeDownloader()
d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file", errors.New("403"))
_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
c.Assert(err, ErrorMatches, "403")
@@ -277,10 +295,12 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
d = NewFakeDownloader()
d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
d.ExpectError("http://example.com/file.xz", &HTTPError{Code: 404})
d.ExpectResponse("http://example.com/file", rawData)
expectedChecksums := map[string]utils.ChecksumInfo{
"file.bz2": {Size: 7},
"file.gz": {Size: 7},
"file.xz": {Size: 7},
"file": {Size: 7},
}
_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
system/t04_mirror/UpdateMirror4Test_gold
+1
View File
@@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release...
Downloading & parsing package files...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz...
Downloading ${url}dists/hardy/main/binary-amd64/Packages...
WARNING: ${url}dists/hardy/main/binary-amd64/Packages: sha256 hash mismatch "494414ded24da13c451b13b424928821351c78fce49f93d9e1b55f102790c206" != "8a21688ae769f2b4ffcaa366409f679d"
ERROR: unable to update: malformed stanza syntax
system/t04_mirror/UpdateMirror6Test_gold
+1
View File
@@ -2,6 +2,7 @@ Downloading ${url}dists/hardy/Release...
Downloading & parsing package files...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.bz2...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.gz...
Downloading ${url}dists/hardy/main/binary-amd64/Packages.xz...
Downloading ${url}dists/hardy/main/binary-amd64/Packages...
Building download queue...
Download queue: 1 items (30 B)