package swift import ( "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "time" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" "github.com/ncw/swift" "github.com/pkg/errors" ) // PublishedStorage abstract file system with published files (actually hosted on Swift) type PublishedStorage struct { conn *swift.Connection container string prefix string supportBulkDelete bool } type swiftInfo map[string]interface{} // Check interface var ( _ aptly.PublishedStorage = (*PublishedStorage)(nil) ) // NewPublishedStorage creates new instance of PublishedStorage with specified Swift access // keys, tenant and tenantId func NewPublishedStorage(username string, password string, authURL string, tenant string, tenantID string, domain string, domainID string, tenantDomain string, tenantDomainID string, container string, prefix string) (*PublishedStorage, error) { if username == "" { if username = os.Getenv("OS_USERNAME"); username == "" { username = os.Getenv("ST_USER") } } if password == "" { if password = os.Getenv("OS_PASSWORD"); password == "" { password = os.Getenv("ST_KEY") } } if authURL == "" { if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" { authURL = os.Getenv("ST_AUTH") } } if tenant == "" { tenant = os.Getenv("OS_TENANT_NAME") } if tenantID == "" { tenantID = os.Getenv("OS_TENANT_ID") } if domain == "" { domain = os.Getenv("OS_USER_DOMAIN_NAME") } if domainID == "" { domainID = os.Getenv("OS_USER_DOMAIN_ID") } if tenantDomain == "" { tenantDomain = os.Getenv("OS_PROJECT_DOMAIN") } if tenantDomainID == "" { tenantDomainID = os.Getenv("OS_PROJECT_DOMAIN_ID") } ct := &swift.Connection{ UserName: username, ApiKey: password, AuthUrl: authURL, UserAgent: "aptly/" + aptly.Version, Tenant: tenant, TenantId: tenantID, Domain: domain, DomainId: domainID, TenantDomain: tenantDomain, TenantDomainId: tenantDomainID, ConnectTimeout: 60 * time.Second, Timeout: 60 * time.Second, } err := ct.Authenticate() if err != nil { return nil, fmt.Errorf("swift authentication failed: %s", err) } var bulkDelete bool resp, err := http.Get(filepath.Join(ct.StorageUrl, "..", "..") + "/info") if err == nil { defer func() { _ = resp.Body.Close() }() decoder := json.NewDecoder(resp.Body) var infos swiftInfo if decoder.Decode(&infos) == nil { _, bulkDelete = infos["bulk_delete"] } } result := &PublishedStorage{ conn: ct, container: container, prefix: prefix, supportBulkDelete: bulkDelete, } return result, nil } // String represents the Storage as string func (storage *PublishedStorage) String() string { return fmt.Sprintf("Swift: %s/%s/%s", storage.conn.StorageUrl, storage.container, storage.prefix) } // MkDir creates directory recursively under public path func (storage *PublishedStorage) MkDir(_ string) error { // no op for Swift return nil } // PutFile puts file into published storage at specified path func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error { var ( source *os.File err error ) source, err = os.Open(sourceFilename) if err != nil { return err } defer func() { _ = source.Close() }() err = storage.putFile(path, source) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) } return err } func (storage *PublishedStorage) putFile(path string, source io.Reader) error { _, err := storage.conn.ObjectPut(storage.container, filepath.Join(storage.prefix, path), source, false, "", "", nil) return err } // Remove removes single file under public path func (storage *PublishedStorage) Remove(path string) error { err := storage.conn.ObjectDelete(storage.container, filepath.Join(storage.prefix, path)) if err != nil { return fmt.Errorf("error deleting %s from %s: %s", path, storage, err) } return nil } // RemoveDirs removes directory structure under public path func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error { path = filepath.Join(storage.prefix, path) opts := swift.ObjectsOpts{ Prefix: path, } objects, err := storage.conn.ObjectNamesAll(storage.container, &opts) if err != nil { return fmt.Errorf("error removing dir %s from %s: %s", path, storage, err) } for index, name := range objects { objects[index] = name[len(storage.prefix):] } multiDelete := true if storage.supportBulkDelete { _, err := storage.conn.BulkDelete(storage.container, objects) multiDelete = err != nil } if multiDelete { for _, name := range objects { if err := storage.conn.ObjectDelete(storage.container, name); err != nil { return err } } } return nil } // LinkFromPool links package file from pool to dist's pool location // // publishedPrefix is desired prefix for the location in the pool. // publishedRelPath is desired location in pool (like pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { relPath := filepath.Join(publishedPrefix, publishedRelPath, fileName) poolPath := filepath.Join(storage.prefix, relPath) var ( info swift.Object err error ) info, _, err = storage.conn.Object(storage.container, poolPath) if err != nil { if err != swift.ObjectNotFound { return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err) } } else { if sourceChecksums.MD5 == "" { return fmt.Errorf("unable to compare object, MD5 checksum missing") } if !force && info.Hash != sourceChecksums.MD5 { return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage) } } source, err := sourcePool.Open(sourcePath) if err != nil { return err } defer func() { _ = source.Close() }() err = storage.putFile(relPath, source) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath)) } return err } // Filelist returns list of files under prefix func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { prefix = filepath.Join(storage.prefix, prefix) if prefix != "" { prefix += "/" } opts := swift.ObjectsOpts{ Prefix: prefix, } contents, err := storage.conn.ObjectNamesAll(storage.container, &opts) if err != nil { return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err) } for index, name := range contents { contents[index] = name[len(prefix):] } return contents, nil } // RenameFile renames (moves) file func (storage *PublishedStorage) RenameFile(oldName, newName string) error { err := storage.conn.ObjectMove(storage.container, filepath.Join(storage.prefix, oldName), storage.container, filepath.Join(storage.prefix, newName)) if err != nil { return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err) } return nil } // SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { srcObjectName := filepath.Join(storage.prefix, src) dstObjectName := filepath.Join(storage.prefix, dst) headers := map[string]string{ "SymLink": srcObjectName, } _, err := storage.conn.ObjectCopy(storage.container, srcObjectName, storage.container, dstObjectName, headers) if err != nil { return fmt.Errorf("error symlinking %s -> %s in %s: %s", srcObjectName, dstObjectName, storage, err) } return err } // HardLink using symlink functionality as hard links do not exist func (storage *PublishedStorage) HardLink(src string, dst string) error { return storage.SymLink(src, dst) } // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) (bool, error) { _, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path)) if err != nil { if err == swift.ObjectNotFound { return false, nil } return false, err } return true, nil } // ReadLink returns the symbolic link pointed to by path func (storage *PublishedStorage) ReadLink(path string) (string, error) { srcObjectName := filepath.Join(storage.prefix, path) _, headers, err := storage.conn.Object(storage.container, srcObjectName) if err != nil { return "", fmt.Errorf("error reading symlink %s in %s: %s", srcObjectName, storage, err) } return headers["SymLink"], nil }