diff --git a/azure/azure.go b/azure/azure.go new file mode 100644 index 00000000..5f76b744 --- /dev/null +++ b/azure/azure.go @@ -0,0 +1,2 @@ +// Package azure handles publishing to Azure Storage +package azure diff --git a/azure/azure_test.go b/azure/azure_test.go new file mode 100644 index 00000000..3e816a3a --- /dev/null +++ b/azure/azure_test.go @@ -0,0 +1,12 @@ +package azure + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +// Launch gocheck tests +func Test(t *testing.T) { + TestingT(t) +} diff --git a/azure/public.go b/azure/public.go new file mode 100644 index 00000000..038c2f96 --- /dev/null +++ b/azure/public.go @@ -0,0 +1,347 @@ +package azure + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/utils" + "github.com/pkg/errors" +) + +// PublishedStorage abstract file system with published files (actually hosted on Azure) +type PublishedStorage struct { + container azblob.ContainerURL + prefix string + pathCache map[string]string +} + +// Check interface +var ( + _ aptly.PublishedStorage = (*PublishedStorage)(nil) +) + +// NewPublishedStorage creates published storage from Azure storage credentials +func NewPublishedStorage(accountName, accountKey, container, prefix string) (*PublishedStorage, error) { + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, err + } + + containerURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, container)) + if err != nil { + return nil, err + } + + result := &PublishedStorage{ + container: azblob.NewContainerURL(*containerURL, azblob.NewPipeline(credential, azblob.PipelineOptions{})), + prefix: prefix, + } + + return result, nil +} + +// String +func (storage *PublishedStorage) String() string { + return fmt.Sprintf("Azure :%s/%s", storage.container, storage.prefix) +} + +// MkDir creates directory recursively under public path +func (storage *PublishedStorage) MkDir(path string) error { + // no op for Azure + return nil +} + +// PutFile puts file into published storage at specified path +func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error { + var ( + source *os.File + err error + ) + + sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename) + if err != nil { + return err + } + + source, err = os.Open(sourceFilename) + if err != nil { + return err + } + defer source.Close() + + err = storage.putFile(path, source, sourceMD5) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) + } + + return err +} + +// putFile uploads file-like object to +func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error { + path = filepath.Join(storage.prefix, path) + + blob := storage.container.NewBlockBlobURL(path) + + uploadOptions := azblob.UploadStreamToBlockBlobOptions{ + BufferSize: 4 * 1024 * 1024, + MaxBuffers: 8, + } + if len(sourceMD5) > 0 { + decodedMD5, err := hex.DecodeString(sourceMD5) + if err != nil { + return err + } + uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{ + ContentMD5: decodedMD5, + } + } + + _, err := azblob.UploadStreamToBlockBlob( + context.Background(), + source, + blob, + uploadOptions, + ) + + return err +} + +// RemoveDirs removes directory structure under public path +func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error { + filelist, err := storage.Filelist(path) + if err != nil { + return err + } + + for _, filename := range filelist { + blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename)) + _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err) + } + } + + return nil +} + +// Remove removes single file under public path +func (storage *PublishedStorage) Remove(path string) error { + blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path)) + _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err)) + } + return err +} + +// LinkFromPool links package file from pool to dist's pool location +// +// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// sourcePool is instance of aptly.PackagePool +// sourcePath is filepath to package file in package pool +// +// LinkFromPool returns relative path for the published file to be included in package index +func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, + sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { + + relPath := filepath.Join(publishedDirectory, fileName) + poolPath := filepath.Join(storage.prefix, relPath) + + if storage.pathCache == nil { + paths, md5s, err := storage.internalFilelist("") + if err != nil { + return fmt.Errorf("error caching paths under prefix: %s", err) + } + + storage.pathCache = make(map[string]string, len(paths)) + + for i := range paths { + storage.pathCache[paths[i]] = md5s[i] + } + } + + destinationMD5, exists := storage.pathCache[relPath] + sourceMD5 := sourceChecksums.MD5 + + if exists { + if sourceMD5 == "" { + return fmt.Errorf("unable to compare object, MD5 checksum missing") + } + + if destinationMD5 == sourceMD5 { + return nil + } + + if !force && destinationMD5 != sourceMD5 { + return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage) + } + } + + source, err := sourcePool.Open(sourcePath) + if err != nil { + return err + } + defer source.Close() + + err = storage.putFile(relPath, source, sourceMD5) + if err == nil { + storage.pathCache[relPath] = sourceMD5 + } else { + err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath)) + } + + return err +} + +func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) { + const delimiter = "/" + paths = make([]string, 0, 1024) + md5s = make([]string, 0, 1024) + prefix = filepath.Join(storage.prefix, prefix) + if prefix != "" { + prefix += delimiter + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := storage.container.ListBlobsFlatSegment( + context.Background(), marker, azblob.ListBlobsSegmentOptions{ + Prefix: prefix, + MaxResults: 1000, + Details: azblob.BlobListingDetails{Metadata: true}}) + if err != nil { + return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err) + } + + marker = listBlob.NextMarker + + for _, blob := range listBlob.Segment.BlobItems { + if prefix == "" { + paths = append(paths, blob.Name) + } else { + paths = append(paths, blob.Name[len(prefix):]) + } + md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5)) + } + } + + return paths, md5s, nil +} + +// Filelist returns list of files under prefix +func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { + paths, _, err := storage.internalFilelist(prefix) + return paths, err +} + +// Internal copy or move implementation +func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error { + const leaseDuration = 30 + + dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst)) + srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src)) + leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{}) + if err != nil || leaseResp.StatusCode() != http.StatusCreated { + return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL) + } + defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) + srcBlobLeaseID := leaseResp.LeaseID() + + copyResp, err := dstBlobURL.StartCopyFromURL( + context.Background(), + srcBlobURL.URL(), + metadata, + azblob.ModifiedAccessConditions{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil) + if err != nil { + return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err) + } + + copyStatus := copyResp.CopyStatus() + for { + if copyStatus == azblob.CopyStatusSuccess { + if move { + _, err = srcBlobURL.Delete( + context.Background(), + azblob.DeleteSnapshotsOptionNone, + azblob.BlobAccessConditions{ + LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}, + }) + return err + } + return nil + } else if copyStatus == azblob.CopyStatusPending { + time.Sleep(1 * time.Second) + blobPropsResp, err := dstBlobURL.GetProperties( + context.Background(), + azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}}, + azblob.ClientProvidedKeyOptions{}) + if err != nil { + return fmt.Errorf("error getting destination blob properties %s", dstBlobURL) + } + copyStatus = blobPropsResp.CopyStatus() + + _, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{}) + if err != nil { + return fmt.Errorf("error renewing source blob lease %s", srcBlobURL) + } + } else { + return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus) + } + } +} + +// RenameFile renames (moves) file +func (storage *PublishedStorage) RenameFile(oldName, newName string) error { + return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */) +} + +// SymLink creates a copy of src file and adds link information as meta data +func (storage *PublishedStorage) SymLink(src string, dst string) error { + return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */) +} + +// HardLink using symlink functionality as hard links do not exist +func (storage *PublishedStorage) HardLink(src string, dst string) error { + return storage.SymLink(src, dst) +} + +// FileExists returns true if path exists +func (storage *PublishedStorage) FileExists(path string) (bool, error) { + blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path)) + resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + if err != nil { + storageError, ok := err.(azblob.StorageError) + if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) { + return false, nil + } + return false, err + } else if resp.StatusCode() == http.StatusOK { + return true, nil + } + return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) +} + +// ReadLink returns the symbolic link pointed to by path. +// This simply reads text file created with SymLink +func (storage *PublishedStorage) ReadLink(path string) (string, error) { + blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path)) + resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + if err != nil { + return "", err + } else if resp.StatusCode() != http.StatusOK { + return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) + } + return resp.NewMetadata()["SymLink"], nil +} diff --git a/azure/public_test.go b/azure/public_test.go new file mode 100644 index 00000000..fcd28363 --- /dev/null +++ b/azure/public_test.go @@ -0,0 +1,370 @@ +package azure + +import ( + "context" + "crypto/md5" + "crypto/rand" + "io/ioutil" + "os" + "path/filepath" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/aptly-dev/aptly/files" + "github.com/aptly-dev/aptly/utils" + . "gopkg.in/check.v1" +) + +type PublishedStorageSuite struct { + accountName, accountKey string + storage, prefixedStorage *PublishedStorage +} + +var _ = Suite(&PublishedStorageSuite{}) + +const testContainerPrefix = "aptlytest-" + +func randContainer() string { + return testContainerPrefix + randString(32-len(testContainerPrefix)) +} + +func randString(n int) string { + if n <= 0 { + panic("negative number") + } + const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz" + var bytes = make([]byte, n) + rand.Read(bytes) + for i, b := range bytes { + bytes[i] = alphanum[b%byte(len(alphanum))] + } + return string(bytes) +} + +func (s *PublishedStorageSuite) SetUpSuite(c *C) { + s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT") + if s.accountName == "" { + println("Please set the the following two environment variables to run the Azure storage tests.") + println(" 1. AZURE_STORAGE_ACCOUNT") + println(" 2. AZURE_STORAGE_ACCESS_KEY") + c.Skip("AZURE_STORAGE_ACCOUNT not set.") + } + s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") + if s.accountKey == "" { + println("Please set the the following two environment variables to run the Azure storage tests.") + println(" 1. AZURE_STORAGE_ACCOUNT") + println(" 2. AZURE_STORAGE_ACCESS_KEY") + c.Skip("AZURE_STORAGE_ACCESS_KEY not set.") + } +} + +func (s *PublishedStorageSuite) SetUpTest(c *C) { + container := randContainer() + prefix := "lala" + + var err error + + s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "") + c.Assert(err, IsNil) + cnt := s.storage.container + _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) + c.Assert(err, IsNil) + + s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix) + c.Assert(err, IsNil) +} + +func (s *PublishedStorageSuite) TearDownTest(c *C) { + cnt := s.storage.container + _, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{}) + c.Assert(err, IsNil) +} + +func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { + blob := s.storage.container.NewBlobURL(path) + resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + c.Assert(err, IsNil) + body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}) + data, err := ioutil.ReadAll(body) + c.Assert(err, IsNil) + return data +} + +func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { + _, err := s.storage.container.NewBlobURL(path).GetProperties( + context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + c.Assert(err, NotNil) + storageError, ok := err.(azblob.StorageError) + c.Assert(ok, Equals, true) + c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound))) +} + +func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { + hash := md5.Sum(data) + _, err := azblob.UploadBufferToBlockBlob( + context.Background(), + data, + s.storage.container.NewBlockBlobURL(path), + azblob.UploadToBlockBlobOptions{ + BlobHTTPHeaders: azblob.BlobHTTPHeaders{ + ContentMD5: hash[:], + }, + }) + c.Assert(err, IsNil) +} + +func (s *PublishedStorageSuite) TestPutFile(c *C) { + content := []byte("Welcome to Azure!") + filename := "a/b.txt" + + dir := c.MkDir() + err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644) + c.Assert(err, IsNil) + + err = s.storage.PutFile(filename, filepath.Join(dir, "a")) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, filename), DeepEquals, content) + + err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a")) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content) +} + +func (s *PublishedStorageSuite) TestPutFilePlus(c *C) { + content := []byte("Welcome to Azure!") + filename := "a/b+c.txt" + + dir := c.MkDir() + err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644) + c.Assert(err, IsNil) + + err = s.storage.PutFile(filename, filepath.Join(dir, "a")) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, filename), DeepEquals, content) + s.AssertNoFile(c, "a/b c.txt") +} + +func (s *PublishedStorageSuite) TestFilelist(c *C) { + paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"} + for _, path := range paths { + s.PutFile(c, path, []byte("test")) + } + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"}) + + list, err = s.storage.Filelist("test") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b"}) + + list, err = s.storage.Filelist("test2") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{}) + + list, err = s.prefixedStorage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c"}) +} + +func (s *PublishedStorageSuite) TestFilelistPlus(c *C) { + paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} + for _, path := range paths { + s.PutFile(c, path, []byte("test")) + } + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"}) + + list, err = s.storage.Filelist("test") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a 1", "a+1"}) + + list, err = s.storage.Filelist("test2") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{}) + + list, err = s.prefixedStorage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a b", "a+b", "c"}) +} + +func (s *PublishedStorageSuite) TestRemove(c *C) { + s.PutFile(c, "a/b", []byte("test")) + + err := s.storage.Remove("a/b") + c.Check(err, IsNil) + + s.AssertNoFile(c, "a/b") + + s.PutFile(c, "lala/xyz", []byte("test")) + + err = s.prefixedStorage.Remove("xyz") + c.Check(err, IsNil) + + s.AssertNoFile(c, "lala/xyz") +} + +func (s *PublishedStorageSuite) TestRemovePlus(c *C) { + s.PutFile(c, "a/b+c", []byte("test")) + s.PutFile(c, "a/b", []byte("test")) + + err := s.storage.Remove("a/b+c") + c.Check(err, IsNil) + + s.AssertNoFile(c, "a/b+c") + s.AssertNoFile(c, "a/b c") + + err = s.storage.Remove("a/b") + c.Check(err, IsNil) + + s.AssertNoFile(c, "a/b") +} + +func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { + paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"} + for _, path := range paths { + s.PutFile(c, path, []byte("test")) + } + + err := s.storage.RemoveDirs("test", nil) + c.Check(err, IsNil) + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"}) +} + +func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) { + paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} + for _, path := range paths { + s.PutFile(c, path, []byte("test")) + } + + err := s.storage.RemoveDirs("test", nil) + c.Check(err, IsNil) + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"}) +} + +func (s *PublishedStorageSuite) TestRenameFile(c *C) { + dir := c.MkDir() + err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644) + c.Assert(err, IsNil) + + err = s.storage.PutFile("source.txt", filepath.Join(dir, "a")) + c.Check(err, IsNil) + + err = s.storage.RenameFile("source.txt", "dest.txt") + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!")) + + exists, err := s.storage.FileExists("source.txt") + c.Check(err, IsNil) + c.Check(exists, Equals, false) +} + +func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { + root := c.MkDir() + pool := files.NewPackagePool(root, false) + cs := files.NewMockChecksumStorage() + + tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb") + err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644) + c.Assert(err, IsNil) + cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"} + + tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb") + err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644) + c.Assert(err, IsNil) + cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"} + + tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz") + os.MkdirAll(filepath.Dir(tmpFile3), 0777) + err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644) + c.Assert(err, IsNil) + cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"} + + src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs) + c.Assert(err, IsNil) + src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs) + c.Assert(err, IsNil) + src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs) + c.Assert(err, IsNil) + + // first link from pool + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) + + // duplicate link from pool + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) + + // link from pool with conflict + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) + c.Check(err, ErrorMatches, ".*file already exists and is different.*") + + c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) + + // link from pool with conflict and force + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam")) + + // for prefixed storage: + // first link from pool + err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + // 2nd link from pool, providing wrong path for source file + // + // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) + s.prefixedStorage.pathCache = nil + err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) + + // link from pool with nested file name + err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) + c.Check(err, IsNil) + + c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents")) +} + +func (s *PublishedStorageSuite) TestSymLink(c *C) { + s.PutFile(c, "a/b", []byte("test")) + + err := s.storage.SymLink("a/b", "a/b.link") + c.Check(err, IsNil) + + var link string + link, err = s.storage.ReadLink("a/b.link") + c.Check(err, IsNil) + c.Check(link, Equals, "a/b") + + c.Skip("copy not available in s3test") +} + +func (s *PublishedStorageSuite) TestFileExists(c *C) { + s.PutFile(c, "a/b", []byte("test")) + + exists, err := s.storage.FileExists("a/b") + c.Check(err, IsNil) + c.Check(exists, Equals, true) + + exists, _ = s.storage.FileExists("a/b.invalid") + c.Check(err, IsNil) + c.Check(exists, Equals, false) +} diff --git a/context/context.go b/context/context.go index 183b7060..88c41124 100644 --- a/context/context.go +++ b/context/context.go @@ -15,6 +15,7 @@ import ( "time" "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/azure" "github.com/aptly-dev/aptly/console" "github.com/aptly-dev/aptly/database" "github.com/aptly-dev/aptly/database/goleveldb" @@ -379,6 +380,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto if err != nil { Fatal(err) } + } else if strings.HasPrefix(name, "azure:") { + params, ok := context.config().AzurePublishRoots[name[6:]] + if !ok { + Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:])) + } + + var err error + publishedStorage, err = azure.NewPublishedStorage( + params.AccountName, params.AccountKey, params.Container, params.Prefix) + if err != nil { + Fatal(err) + } } else { Fatal(fmt.Errorf("unknown published storage format: %v", name)) } diff --git a/go.mod b/go.mod index a9b0b975..e5a06664 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.15 require ( github.com/AlekSi/pointer v1.0.0 + github.com/Azure/azure-storage-blob-go v0.13.0 github.com/DisposaBoy/JsonConfigReader v0.0.0-20130112093355-33a99fdf1d5e github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e github.com/aws/aws-sdk-go v1.25.0 @@ -21,7 +22,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f github.com/ncw/swift v1.0.30 github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/smartystreets/gunit v1.0.4 // indirect github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5 github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76 @@ -32,9 +33,9 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/ugorji/go v1.1.4 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 - golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc - golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 - gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/sys v0.0.0-20200828194041-157a740278f4 + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v8 v8.18.2 // indirect diff --git a/go.sum b/go.sum index a5fd4a1c..40d3800d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,19 @@ github.com/AlekSi/pointer v1.0.0 h1:KWCWzsvFxNLcmM5XmiqHsGTTsuwZMsLFwWF9Y+//bNE= github.com/AlekSi/pointer v1.0.0/go.mod h1:1kjywbfcPFCmncIxtk6fIEub6LKrfMz3gc5QKVOSOA8= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc= +github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.2 h1:Aze/GQeAN1RRbGmnUJvUj+tFGBzFdIg3293/A9rbxC4= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/DisposaBoy/JsonConfigReader v0.0.0-20130112093355-33a99fdf1d5e h1:rv5qJCfIzQhhefHp8MO98hoGRI3mdps2iiGA3o4nm8A= github.com/DisposaBoy/JsonConfigReader v0.0.0-20130112093355-33a99fdf1d5e/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI= github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e h1:24jix3mqu421BBMWbVBOl5pDw0f9ncazW10kaMywzHQ= @@ -10,6 +24,8 @@ github.com/cheggaaa/pb v1.0.10 h1:CNg2511WECXZ7Ja6jjyz9CMBpQOrMuP5+H5zfjgVi/Q= github.com/cheggaaa/pb v1.0.10/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -22,6 +38,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.0.5 h1:Esu2EFM5vrzNynnGQpj0nxhCkzVQh2HRY7AXUh/dyJM= github.com/h2non/filetype v1.0.5/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -32,8 +50,13 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKMYMj0FJRCF6tQubUuQ02GM= github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= @@ -46,6 +69,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.30 h1:CrRYmUc+mFGIvBiS5JIA4sIdURfDpJ4CGmpmR9mQAZ0= github.com/ncw/swift v1.0.30/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -53,8 +78,8 @@ github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 h1:9J0mOv1rXIBlRjQCiAGyx9C3dZZh5uIa3HU0oTV8v1E= github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -80,21 +105,29 @@ github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= -golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc h1:Kx1Ke+iCR1aDjbWXgmEQGFxoHtNL49aRZGV7/+jJ41Y= -golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405 h1:829vOVxxusYHC+IqBtkX5mbKtsY9fheQiQn0MZRVLfQ= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -107,7 +140,6 @@ gopkg.in/h2non/filetype.v1 v1.0.1 h1:JMZLYHwIsvWGh+6UcU//eA1aiM8g4SaZm3lJweIR5Ew gopkg.in/h2non/filetype.v1 v1.0.1/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index 63e5e22d..4157c96a 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -20,5 +20,6 @@ "skipContentsPublishing": false, "FileSystemPublishEndpoints": {}, "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {} + "SwiftPublishEndpoints": {}, + "AzurePublishEndpoints": {} } diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 099e31ed..013cd4db 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -20,5 +20,6 @@ "skipContentsPublishing": false, "FileSystemPublishEndpoints": {}, "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {} + "SwiftPublishEndpoints": {}, + "AzurePublishEndpoints": {} } \ No newline at end of file diff --git a/utils/config.go b/utils/config.go index a281cc50..9856a397 100644 --- a/utils/config.go +++ b/utils/config.go @@ -30,6 +30,7 @@ type ConfigStructure struct { // nolint: maligned FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints"` S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints"` SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"` + AzurePublishRoots map[string]AzurePublishRoot `json:"AzurePublishEndpoints"` } // FileSystemPublishRoot describes single filesystem publishing entry point @@ -72,6 +73,14 @@ type SwiftPublishRoot struct { Container string `json:"container"` } +// AzurePublishRoot describes single Azure publishing entry point +type AzurePublishRoot struct { + AccountName string `json:"accountName"` + AccountKey string `json:"accountKey"` + Container string `json:"container"` + Prefix string `json:"prefix"` +} + // Config is configuration for aptly, shared by all modules var Config = ConfigStructure{ RootDir: filepath.Join(os.Getenv("HOME"), ".aptly"), @@ -93,6 +102,7 @@ var Config = ConfigStructure{ FileSystemPublishRoots: map[string]FileSystemPublishRoot{}, S3PublishRoots: map[string]S3PublishRoot{}, SwiftPublishRoots: map[string]SwiftPublishRoot{}, + AzurePublishRoots: map[string]AzurePublishRoot{}, } // LoadConfig loads configuration from json file diff --git a/utils/config_test.go b/utils/config_test.go index 1e55ed44..7e858442 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -44,6 +44,9 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { s.config.SwiftPublishRoots = map[string]SwiftPublishRoot{"test": { Container: "repo"}} + s.config.AzurePublishRoots = map[string]AzurePublishRoot{"test": { + Container: "repo"}} + err := SaveConfig(configname, &s.config) c.Assert(err, IsNil) @@ -114,6 +117,14 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { " \"prefix\": \"\",\n"+ " \"container\": \"repo\"\n"+ " }\n"+ + " },\n"+ + " \"AzurePublishEndpoints\": {\n"+ + " \"test\": {\n"+ + " \"accountName\": \"\",\n"+ + " \"accountKey\": \"\",\n"+ + " \"container\": \"repo\",\n"+ + " \"prefix\": \"\"\n"+ + " }\n"+ " }\n"+ "}") }