mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-15 07:00:52 +00:00
Imported Upstream version 1.0.1
This commit is contained in:
Sébastien Delafond
@@ -0,0 +1,389 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mxk/go-flowrate/flowrate"
|
||||
"github.com/smira/aptly/aptly"
|
||||
"github.com/smira/aptly/utils"
|
||||
"github.com/smira/go-ftp-protocol/protocol"
|
||||
"github.com/smira/go-xz"
|
||||
)
|
||||
|
||||
// Error is download error connected to HTTP code
|
||||
type Error struct {
|
||||
Code int
|
||||
URL string
|
||||
}
|
||||
|
||||
// Error
|
||||
func (e *Error) Error() string {
|
||||
return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
|
||||
}
|
||||
|
||||
// Check interface
|
||||
var (
|
||||
_ aptly.Downloader = (*downloaderImpl)(nil)
|
||||
)
|
||||
|
||||
// downloaderImpl is implementation of Downloader interface
|
||||
type downloaderImpl struct {
|
||||
queue chan *downloadTask
|
||||
stop chan struct{}
|
||||
stopped chan struct{}
|
||||
pause chan struct{}
|
||||
unpause chan struct{}
|
||||
progress aptly.Progress
|
||||
aggWriter io.Writer
|
||||
threads int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// downloadTask represents single item in queue
|
||||
type downloadTask struct {
|
||||
url string
|
||||
destination string
|
||||
result chan<- error
|
||||
expected utils.ChecksumInfo
|
||||
ignoreMismatch bool
|
||||
triesLeft int
|
||||
}
|
||||
|
||||
// NewDownloader creates new instance of Downloader which specified number
|
||||
// of threads and download limit in bytes/sec
|
||||
func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
|
||||
transport := http.Transport{}
|
||||
transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
|
||||
transport.ResponseHeaderTimeout = 30 * time.Second
|
||||
transport.TLSHandshakeTimeout = http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout
|
||||
transport.ExpectContinueTimeout = http.DefaultTransport.(*http.Transport).ExpectContinueTimeout
|
||||
transport.DisableCompression = true
|
||||
initTransport(&transport)
|
||||
transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
|
||||
|
||||
downloader := &downloaderImpl{
|
||||
queue: make(chan *downloadTask, 1000),
|
||||
stop: make(chan struct{}, threads),
|
||||
stopped: make(chan struct{}, threads),
|
||||
pause: make(chan struct{}),
|
||||
unpause: make(chan struct{}),
|
||||
threads: threads,
|
||||
progress: progress,
|
||||
client: &http.Client{
|
||||
Transport: &transport,
|
||||
},
|
||||
}
|
||||
|
||||
if downLimit > 0 {
|
||||
downloader.aggWriter = flowrate.NewWriter(progress, downLimit)
|
||||
} else {
|
||||
downloader.aggWriter = progress
|
||||
}
|
||||
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
go downloader.process()
|
||||
}
|
||||
|
||||
return downloader
|
||||
}
|
||||
|
||||
// Shutdown stops downloader after current tasks are finished,
|
||||
// but doesn't process rest of queue
|
||||
func (downloader *downloaderImpl) Shutdown() {
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
downloader.stop <- struct{}{}
|
||||
}
|
||||
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
<-downloader.stopped
|
||||
}
|
||||
}
|
||||
|
||||
// Abort stops downloader but doesn't wait for downloader to stop
|
||||
func (downloader *downloaderImpl) Abort() {
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
downloader.stop <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Pause pauses task processing
|
||||
func (downloader *downloaderImpl) Pause() {
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
downloader.pause <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Resume resumes task processing
|
||||
func (downloader *downloaderImpl) Resume() {
|
||||
for i := 0; i < downloader.threads; i++ {
|
||||
downloader.unpause <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// GetProgress returns Progress object
|
||||
func (downloader *downloaderImpl) GetProgress() aptly.Progress {
|
||||
return downloader.progress
|
||||
}
|
||||
|
||||
// Download starts new download task
|
||||
func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
|
||||
downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false, 1)
|
||||
}
|
||||
|
||||
// DownloadWithChecksum starts new download task with checksum verification
|
||||
func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
|
||||
expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
|
||||
downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
|
||||
}
|
||||
|
||||
// handleTask processes single download task
|
||||
func (downloader *downloaderImpl) handleTask(task *downloadTask) {
|
||||
downloader.progress.Printf("Downloading %s...\n", task.url)
|
||||
|
||||
req, err := http.NewRequest("GET", task.url, nil)
|
||||
if err != nil {
|
||||
task.result <- fmt.Errorf("%s: %s", task.url, err)
|
||||
return
|
||||
}
|
||||
req.Close = true
|
||||
|
||||
proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
|
||||
if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
|
||||
req.URL.Opaque = strings.Replace(req.URL.RequestURI(), "+", "%2b", -1)
|
||||
req.URL.RawQuery = ""
|
||||
}
|
||||
|
||||
var temppath string
|
||||
for task.triesLeft > 0 {
|
||||
|
||||
temppath, err = downloader.downloadTask(req, task)
|
||||
|
||||
if err != nil {
|
||||
task.triesLeft--
|
||||
} else {
|
||||
// successful download
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// still an error after retrying, giving up
|
||||
if err != nil {
|
||||
task.result <- err
|
||||
return
|
||||
}
|
||||
|
||||
err = os.Rename(temppath, task.destination)
|
||||
if err != nil {
|
||||
os.Remove(temppath)
|
||||
task.result <- fmt.Errorf("%s: %s", task.url, err)
|
||||
return
|
||||
}
|
||||
|
||||
task.result <- nil
|
||||
}
|
||||
|
||||
func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
|
||||
resp, err := downloader.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %s", task.url, err)
|
||||
}
|
||||
if resp.Body != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return "", &Error{Code: resp.StatusCode, URL: task.url}
|
||||
}
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(task.destination), 0777)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %s", task.url, err)
|
||||
}
|
||||
|
||||
temppath := task.destination + ".down"
|
||||
|
||||
outfile, err := os.Create(temppath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %s", task.url, err)
|
||||
}
|
||||
defer outfile.Close()
|
||||
|
||||
checksummer := utils.NewChecksumWriter()
|
||||
writers := []io.Writer{outfile, downloader.aggWriter}
|
||||
|
||||
if task.expected.Size != -1 {
|
||||
writers = append(writers, checksummer)
|
||||
}
|
||||
|
||||
w := io.MultiWriter(writers...)
|
||||
|
||||
_, err = io.Copy(w, resp.Body)
|
||||
if err != nil {
|
||||
os.Remove(temppath)
|
||||
return "", fmt.Errorf("%s: %s", task.url, err)
|
||||
}
|
||||
|
||||
if task.expected.Size != -1 {
|
||||
actual := checksummer.Sum()
|
||||
|
||||
if actual.Size != task.expected.Size {
|
||||
err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
|
||||
} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
|
||||
err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
|
||||
} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
|
||||
err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
|
||||
} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
|
||||
err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
|
||||
} else if task.expected.SHA512 != "" && actual.SHA512 != task.expected.SHA512 {
|
||||
err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", task.url, actual.SHA512, task.expected.SHA512)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if task.ignoreMismatch {
|
||||
downloader.progress.Printf("WARNING: %s\n", err.Error())
|
||||
} else {
|
||||
os.Remove(temppath)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return temppath, nil
|
||||
}
|
||||
|
||||
// process implements download thread in goroutine
|
||||
func (downloader *downloaderImpl) process() {
|
||||
for {
|
||||
select {
|
||||
case <-downloader.stop:
|
||||
downloader.stopped <- struct{}{}
|
||||
return
|
||||
case <-downloader.pause:
|
||||
<-downloader.unpause
|
||||
case task := <-downloader.queue:
|
||||
downloader.handleTask(task)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadTemp starts new download to temporary file and returns File
|
||||
//
|
||||
// Temporary file would be already removed, so no need to cleanup
|
||||
func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
|
||||
return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false, 1)
|
||||
}
|
||||
|
||||
// DownloadTempWithChecksum is a DownloadTemp with checksum verification
|
||||
//
|
||||
// Temporary file would be already removed, so no need to cleanup
|
||||
func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
|
||||
tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tempdir)
|
||||
|
||||
tempfile := filepath.Join(tempdir, "buffer")
|
||||
|
||||
if expected.Size != -1 && downloader.GetProgress() != nil {
|
||||
downloader.GetProgress().InitBar(expected.Size, true)
|
||||
defer downloader.GetProgress().ShutdownBar()
|
||||
}
|
||||
|
||||
ch := make(chan error, 1)
|
||||
downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch, maxTries)
|
||||
|
||||
err = <-ch
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file, err := os.Open(tempfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// List of extensions + corresponding uncompression support
|
||||
var compressionMethods = []struct {
|
||||
extenstion string
|
||||
transformation func(io.Reader) (io.Reader, error)
|
||||
}{
|
||||
{
|
||||
extenstion: ".bz2",
|
||||
transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
|
||||
},
|
||||
{
|
||||
extenstion: ".gz",
|
||||
transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
|
||||
},
|
||||
{
|
||||
extenstion: ".xz",
|
||||
transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
|
||||
},
|
||||
{
|
||||
extenstion: "",
|
||||
transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
|
||||
},
|
||||
}
|
||||
|
||||
// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
|
||||
// it finds existing file.
|
||||
func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
|
||||
var err error
|
||||
|
||||
for _, method := range compressionMethods {
|
||||
var file *os.File
|
||||
|
||||
tryURL := url + method.extenstion
|
||||
foundChecksum := false
|
||||
|
||||
for suffix, expected := range expectedChecksums {
|
||||
if strings.HasSuffix(tryURL, suffix) {
|
||||
file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch, maxTries)
|
||||
foundChecksum = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !foundChecksum {
|
||||
if !ignoreMismatch {
|
||||
continue
|
||||
}
|
||||
|
||||
file, err = DownloadTemp(downloader, tryURL)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err1, ok := err.(*Error); ok && (err1.Code == 404 || err1.Code == 403) {
|
||||
continue
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var uncompressed io.Reader
|
||||
uncompressed, err = method.transformation(file)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return uncompressed, file, err
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = fmt.Errorf("no candidates for %s found", url)
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// +build !go1.7
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func initTransport(transport *http.Transport) {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// +build go1.7
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func initTransport(transport *http.Transport) {
|
||||
transport.DialContext = http.DefaultTransport.(*http.Transport).DialContext
|
||||
transport.MaxIdleConns = http.DefaultTransport.(*http.Transport).MaxIdleConns
|
||||
transport.IdleConnTimeout = http.DefaultTransport.(*http.Transport).IdleConnTimeout
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/smira/aptly/aptly"
|
||||
"github.com/smira/aptly/console"
|
||||
"github.com/smira/aptly/utils"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type DownloaderSuite struct {
|
||||
tempfile *os.File
|
||||
l net.Listener
|
||||
url string
|
||||
ch chan bool
|
||||
progress aptly.Progress
|
||||
}
|
||||
|
||||
var _ = Suite(&DownloaderSuite{})
|
||||
|
||||
func (s *DownloaderSuite) SetUpTest(c *C) {
|
||||
s.tempfile, _ = ioutil.TempFile(os.TempDir(), "aptly-test")
|
||||
s.l, _ = net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
|
||||
s.url = fmt.Sprintf("http://localhost:%d", s.l.Addr().(*net.TCPAddr).Port)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "Hello, %s", r.URL.Path)
|
||||
})
|
||||
|
||||
s.ch = make(chan bool)
|
||||
|
||||
go func() {
|
||||
http.Serve(s.l, mux)
|
||||
s.ch <- true
|
||||
}()
|
||||
|
||||
s.progress = console.NewProgress()
|
||||
s.progress.Start()
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TearDownTest(c *C) {
|
||||
s.progress.Shutdown()
|
||||
|
||||
s.l.Close()
|
||||
<-s.ch
|
||||
|
||||
os.Remove(s.tempfile.Name())
|
||||
s.tempfile.Close()
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestStartupShutdown(c *C) {
|
||||
goroutines := runtime.NumGoroutine()
|
||||
|
||||
d := NewDownloader(10, 100, s.progress)
|
||||
d.Shutdown()
|
||||
|
||||
// wait for goroutines to shutdown
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if runtime.NumGoroutine()-goroutines > 1 {
|
||||
c.Errorf("Number of goroutines %d, expected %d", runtime.NumGoroutine(), goroutines)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestPauseResume(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
|
||||
d.Pause()
|
||||
d.Resume()
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadOK(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
ch := make(chan error)
|
||||
|
||||
d.Download(s.url+"/test", s.tempfile.Name(), ch)
|
||||
res := <-ch
|
||||
c.Assert(res, IsNil)
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
ch := make(chan error)
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{}, false, 1)
|
||||
res := <-ch
|
||||
c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0")
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, IsNil)
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, IsNil)
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
|
||||
SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, IsNil)
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
|
||||
SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
|
||||
|
||||
d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
|
||||
SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
|
||||
res = <-ch
|
||||
c.Assert(res, IsNil)
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownload404(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
ch := make(chan error)
|
||||
|
||||
d.Download(s.url+"/doesntexist", s.tempfile.Name(), ch)
|
||||
res := <-ch
|
||||
c.Assert(res, ErrorMatches, "HTTP code 404.*")
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
ch := make(chan error)
|
||||
|
||||
d.Download("http://nosuch.localhost/", s.tempfile.Name(), ch)
|
||||
res := <-ch
|
||||
c.Assert(res, ErrorMatches, ".*no such host")
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadFileError(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
ch := make(chan error)
|
||||
|
||||
d.Download(s.url+"/test", "/", ch)
|
||||
res := <-ch
|
||||
c.Assert(res, ErrorMatches, ".*permission denied")
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadTemp(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
|
||||
f, err := DownloadTemp(d, s.url+"/test")
|
||||
c.Assert(err, IsNil)
|
||||
defer f.Close()
|
||||
|
||||
buf := make([]byte, 1)
|
||||
|
||||
f.Read(buf)
|
||||
c.Assert(buf, DeepEquals, []byte("H"))
|
||||
|
||||
_, err = os.Stat(f.Name())
|
||||
c.Assert(os.IsNotExist(err), Equals, true)
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
|
||||
f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
|
||||
SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
|
||||
defer f.Close()
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
_, err = DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 13}, false, 1)
|
||||
c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadTempError(c *C) {
|
||||
d := NewDownloader(2, 0, s.progress)
|
||||
defer d.Shutdown()
|
||||
|
||||
f, err := DownloadTemp(d, s.url+"/doesntexist")
|
||||
c.Assert(err, NotNil)
|
||||
c.Assert(f, IsNil)
|
||||
c.Assert(err, ErrorMatches, "HTTP code 404.*")
|
||||
}
|
||||
|
||||
const (
|
||||
bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
|
||||
gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
|
||||
xzData = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
|
||||
rawData = "test"
|
||||
)
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
|
||||
var buf []byte
|
||||
|
||||
expectedChecksums := map[string]utils.ChecksumInfo{
|
||||
"file.bz2": {Size: int64(len(bzipData))},
|
||||
"file.gz": {Size: int64(len(gzipData))},
|
||||
"file.xz": {Size: int64(len(xzData))},
|
||||
"file": {Size: int64(len(rawData))},
|
||||
}
|
||||
|
||||
// bzip2 only available
|
||||
buf = make([]byte, 4)
|
||||
d := NewFakeDownloader()
|
||||
d.ExpectResponse("http://example.com/file.bz2", bzipData)
|
||||
r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
io.ReadFull(r, buf)
|
||||
c.Assert(string(buf), Equals, rawData)
|
||||
c.Assert(d.Empty(), Equals, true)
|
||||
|
||||
// bzip2 not available, but gz is
|
||||
buf = make([]byte, 4)
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectResponse("http://example.com/file.gz", gzipData)
|
||||
r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
io.ReadFull(r, buf)
|
||||
c.Assert(string(buf), Equals, rawData)
|
||||
c.Assert(d.Empty(), Equals, true)
|
||||
|
||||
// bzip2 & gzip not available, but xz is
|
||||
buf = make([]byte, 4)
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
|
||||
d.ExpectResponse("http://example.com/file.xz", xzData)
|
||||
r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
io.ReadFull(r, buf)
|
||||
c.Assert(string(buf), Equals, rawData)
|
||||
c.Assert(d.Empty(), Equals, true)
|
||||
|
||||
// bzip2, gzip & xz not available, but raw is
|
||||
buf = make([]byte, 4)
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
|
||||
d.ExpectResponse("http://example.com/file", rawData)
|
||||
r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
io.ReadFull(r, buf)
|
||||
c.Assert(string(buf), Equals, rawData)
|
||||
c.Assert(d.Empty(), Equals, true)
|
||||
|
||||
// gzip available, but broken
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectResponse("http://example.com/file.gz", "x")
|
||||
_, file, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
|
||||
c.Assert(err, ErrorMatches, "unexpected EOF")
|
||||
c.Assert(d.Empty(), Equals, true)
|
||||
}
|
||||
|
||||
func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
|
||||
d := NewFakeDownloader()
|
||||
_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
|
||||
c.Assert(err, ErrorMatches, "unexpected request.*")
|
||||
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file", errors.New("403"))
|
||||
_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
|
||||
c.Assert(err, ErrorMatches, "403")
|
||||
|
||||
d = NewFakeDownloader()
|
||||
d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
|
||||
d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
|
||||
d.ExpectResponse("http://example.com/file", rawData)
|
||||
expectedChecksums := map[string]utils.ChecksumInfo{
|
||||
"file.bz2": {Size: 7},
|
||||
"file.gz": {Size: 7},
|
||||
"file.xz": {Size: 7},
|
||||
"file": {Size: 7},
|
||||
}
|
||||
_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
|
||||
c.Assert(err, ErrorMatches, "checksums don't match.*")
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/smira/aptly/aptly"
|
||||
"github.com/smira/aptly/utils"
|
||||
)
|
||||
|
||||
type expectedRequest struct {
|
||||
URL string
|
||||
Err error
|
||||
Response string
|
||||
}
|
||||
|
||||
// FakeDownloader is like Downloader, but it used in tests
|
||||
// to stub out results
|
||||
type FakeDownloader struct {
|
||||
expected []expectedRequest
|
||||
anyExpected map[string]expectedRequest
|
||||
}
|
||||
|
||||
// Check interface
|
||||
var (
|
||||
_ aptly.Downloader = (*FakeDownloader)(nil)
|
||||
)
|
||||
|
||||
// NewFakeDownloader creates new expected downloader
|
||||
func NewFakeDownloader() *FakeDownloader {
|
||||
result := &FakeDownloader{}
|
||||
result.expected = make([]expectedRequest, 0)
|
||||
result.anyExpected = make(map[string]expectedRequest)
|
||||
return result
|
||||
}
|
||||
|
||||
// ExpectResponse installs expectation on upcoming download with response
|
||||
func (f *FakeDownloader) ExpectResponse(url string, response string) *FakeDownloader {
|
||||
f.expected = append(f.expected, expectedRequest{URL: url, Response: response})
|
||||
return f
|
||||
}
|
||||
|
||||
// AnyExpectResponse installs expectation on upcoming download with response in any order (url should be unique)
|
||||
func (f *FakeDownloader) AnyExpectResponse(url string, response string) *FakeDownloader {
|
||||
f.anyExpected[url] = expectedRequest{URL: url, Response: response}
|
||||
return f
|
||||
}
|
||||
|
||||
// ExpectError installs expectation on upcoming download with error
|
||||
func (f *FakeDownloader) ExpectError(url string, err error) *FakeDownloader {
|
||||
f.expected = append(f.expected, expectedRequest{URL: url, Err: err})
|
||||
return f
|
||||
}
|
||||
|
||||
// Empty verifies that are planned downloads have happened
|
||||
func (f *FakeDownloader) Empty() bool {
|
||||
return len(f.expected) == 0
|
||||
}
|
||||
|
||||
// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
|
||||
func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
|
||||
var expectation expectedRequest
|
||||
if len(f.expected) > 0 && f.expected[0].URL == url {
|
||||
expectation, f.expected = f.expected[0], f.expected[1:]
|
||||
} else if _, ok := f.anyExpected[url]; ok {
|
||||
expectation = f.anyExpected[url]
|
||||
delete(f.anyExpected, url)
|
||||
} else {
|
||||
result <- fmt.Errorf("unexpected request for %s", url)
|
||||
return
|
||||
}
|
||||
|
||||
if expectation.Err != nil {
|
||||
result <- expectation.Err
|
||||
return
|
||||
}
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(filename), 0755)
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
|
||||
outfile, err := os.Create(filename)
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
defer outfile.Close()
|
||||
|
||||
cks := utils.NewChecksumWriter()
|
||||
w := io.MultiWriter(outfile, cks)
|
||||
|
||||
_, err = w.Write([]byte(expectation.Response))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
|
||||
if expected.Size != -1 {
|
||||
if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
|
||||
expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
|
||||
if ignoreMismatch {
|
||||
fmt.Printf("WARNING: checksums don't match: %#v != %#v for %s\n", expected, cks.Sum(), url)
|
||||
} else {
|
||||
result <- fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result <- nil
|
||||
return
|
||||
}
|
||||
|
||||
// Download performs fake download by matching against first expectation in the queue
|
||||
func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
|
||||
f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false, 1)
|
||||
}
|
||||
|
||||
// Shutdown does nothing
|
||||
func (f *FakeDownloader) Shutdown() {
|
||||
}
|
||||
|
||||
// Abort does nothing
|
||||
func (f *FakeDownloader) Abort() {
|
||||
}
|
||||
|
||||
// Pause does nothing
|
||||
func (f *FakeDownloader) Pause() {
|
||||
}
|
||||
|
||||
// Resume does nothing
|
||||
func (f *FakeDownloader) Resume() {
|
||||
}
|
||||
|
||||
// GetProgress returns Progress object
|
||||
func (f *FakeDownloader) GetProgress() aptly.Progress {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package http provides all HTTP (and FTP)-related operations
|
||||
package http
|
||||
@@ -0,0 +1,12 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
// Launch gocheck tests
|
||||
func Test(t *testing.T) {
|
||||
TestingT(t)
|
||||
}
|
||||
Reference in New Issue
Block a user