package files import ( "fmt" "io" "os" "path/filepath" "sort" "sync" "syscall" "github.com/google/uuid" "github.com/saracen/walker" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" ) // PackagePool is deduplicated storage of package files on filesystem type PackagePool struct { sync.Mutex rootPath string supportLegacyPaths bool } // Check interface var ( _ aptly.PackagePool = (*PackagePool)(nil) _ aptly.LocalPackagePool = (*PackagePool)(nil) ) // NewPackagePool creates new instance of PackagePool which specified root func NewPackagePool(root string, supportLegacyPaths bool) *PackagePool { rootPath, err := filepath.Abs(root) if err != nil { panic(err) } return &PackagePool{ rootPath: rootPath, supportLegacyPaths: supportLegacyPaths, } } // LegacyPath returns path relative to pool's root for pre-1.1 aptly (based on MD5) func (pool *PackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) { filename = filepath.Base(filename) if filename == "." || filename == "/" { return "", fmt.Errorf("filename %s is invalid", filename) } hashMD5 := checksums.MD5 if len(hashMD5) < 4 { return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename) } return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil } // buildPoolPath generates pool path based on file checksum func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) (string, error) { filename = filepath.Base(filename) if filename == "." || filename == "/" { return "", fmt.Errorf("filename %s is invalid", filename) } hash := checksums.SHA256 if len(hash) < 4 { // this should never happen in real life return "", fmt.Errorf("unable to compute pool location for filename %v, SHA256 is missing", filename) } return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename), nil } // FilepathList returns file paths of all the files in the pool func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) { pool.Lock() defer pool.Unlock() dirs, err := os.ReadDir(pool.rootPath) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } if len(dirs) == 0 { return nil, nil } if progress != nil { progress.InitBar(int64(len(dirs)), false, aptly.BarGeneralBuildFileList) defer progress.ShutdownBar() } result := []string{} resultLock := &sync.Mutex{} for _, dir := range dirs { err = walker.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo) error { if !info.IsDir() { resultLock.Lock() defer resultLock.Unlock() result = append(result, path[len(pool.rootPath)+1:]) } return nil }) if err != nil { return nil, err } if progress != nil { progress.AddBar(1) } } sort.Strings(result) return result, nil } // Remove deletes file in package pool returns its size func (pool *PackagePool) Remove(path string) (size int64, err error) { pool.Lock() defer pool.Unlock() path = filepath.Join(pool.rootPath, path) info, err := os.Stat(path) if err != nil { return 0, err } err = os.Remove(path) return info.Size(), err } func (pool *PackagePool) ensureChecksums(poolPath, fullPoolPath string, checksumStorage aptly.ChecksumStorage) (targetChecksums *utils.ChecksumInfo, err error) { targetChecksums, err = checksumStorage.Get(poolPath) if err != nil { return } if targetChecksums == nil { // we don't have checksums stored yet for this file targetChecksums = &utils.ChecksumInfo{} *targetChecksums, err = utils.ChecksumsForFile(fullPoolPath) if err != nil { return } err = checksumStorage.Update(poolPath, targetChecksums) } return } // Verify checks whether file exists in the pool and fills back checksum info // // if poolPath is empty, poolPath is generated automatically based on checksum info (if available) // in any case, if function returns true, it also fills back checksums with complete information about the file in the pool func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) { possiblePoolPaths := []string{} if poolPath != "" { possiblePoolPaths = append(possiblePoolPaths, poolPath) } else { // try to guess if checksums.SHA256 != "" { modernPath, err := pool.buildPoolPath(basename, checksums) if err != nil { return "", false, err } possiblePoolPaths = append(possiblePoolPaths, modernPath) } if pool.supportLegacyPaths && checksums.MD5 != "" { legacyPath, err := pool.LegacyPath(basename, checksums) if err != nil { return "", false, err } possiblePoolPaths = append(possiblePoolPaths, legacyPath) } } for _, path := range possiblePoolPaths { fullPoolPath := filepath.Join(pool.rootPath, path) targetInfo, err := os.Stat(fullPoolPath) if err != nil { if !os.IsNotExist(err) { // unable to stat target location? return "", false, err } // doesn't exist, skip it continue } if targetInfo.Size() != checksums.Size { // oops, wrong file? continue } var targetChecksums *utils.ChecksumInfo targetChecksums, err = pool.ensureChecksums(path, fullPoolPath, checksumStorage) if err != nil { return "", false, err } if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 || checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 { // wrong file? return "", false, nil } // fill back checksums *checksums = *targetChecksums return path, true, nil } return "", false, nil } // Import copies file into package pool // // - srcPath is full path to source file as it is now // - basename is desired human-readable name (canonical filename) // - checksums are used to calculate file placement // - move indicates whether srcPath can be removed func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, checksumStorage aptly.ChecksumStorage) (string, error) { pool.Lock() defer pool.Unlock() source, err := os.Open(srcPath) if err != nil { return "", err } defer func() { _ = source.Close() }() sourceInfo, err := source.Stat() if err != nil { return "", err } if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.Size != sourceInfo.Size() { // need to update checksums, MD5 and SHA256 should be always defined *checksums, err = utils.ChecksumsForFile(srcPath) if err != nil { return "", err } } // build target path poolPath, err := pool.buildPoolPath(basename, checksums) if err != nil { return "", err } fullPoolPath := filepath.Join(pool.rootPath, poolPath) targetInfo, err := os.Stat(fullPoolPath) if err != nil { if !os.IsNotExist(err) { // unable to stat target location? return "", err } } else { // target already exists and same size if targetInfo.Size() == sourceInfo.Size() { var targetChecksums *utils.ChecksumInfo targetChecksums, err = pool.ensureChecksums(poolPath, fullPoolPath, checksumStorage) if err != nil { return "", err } *checksums = *targetChecksums return poolPath, nil } // trying to overwrite file? return "", fmt.Errorf("unable to import into pool: file %s already exists", fullPoolPath) } if pool.supportLegacyPaths { // file doesn't exist at new location, check legacy location var ( legacyTargetInfo os.FileInfo legacyPath, legacyFullPath string ) legacyPath, err = pool.LegacyPath(basename, checksums) if err != nil { return "", err } legacyFullPath = filepath.Join(pool.rootPath, legacyPath) legacyTargetInfo, err = os.Stat(legacyFullPath) if err != nil { if !os.IsNotExist(err) { return "", err } } else { // legacy file exists if legacyTargetInfo.Size() == sourceInfo.Size() { // file exists at legacy path and it's same size, consider it's already in the pool var targetChecksums *utils.ChecksumInfo targetChecksums, err = pool.ensureChecksums(legacyPath, legacyFullPath, checksumStorage) if err != nil { return "", err } *checksums = *targetChecksums return legacyPath, nil } // size is different, import at new path } } // create subdirs as necessary poolDir := filepath.Dir(fullPoolPath) err = os.MkdirAll(poolDir, 0777) if err != nil { return "", err } // check if we can use hardlinks instead of copying/moving poolDirInfo, err := os.Stat(poolDir) if err != nil { return "", err } if poolDirInfo.Sys().(*syscall.Stat_t).Dev == sourceInfo.Sys().(*syscall.Stat_t).Dev { // same filesystem, try to use hardlink err = os.Link(srcPath, fullPoolPath) } else { err = os.ErrInvalid } if err != nil { // different filesystems or failed hardlink, fallback to copy var target *os.File target, err = os.Create(fullPoolPath) if err != nil { return "", err } defer func() { _ = target.Close() }() _, err = io.Copy(target, source) if err == nil { err = target.Close() } } if err == nil { if !checksums.Complete() { // need full checksums here *checksums, err = utils.ChecksumsForFile(srcPath) if err != nil { return "", err } } err = checksumStorage.Update(poolPath, checksums) } if err == nil && move { err = os.Remove(srcPath) } return poolPath, err } func (pool *PackagePool) Size(path string) (size int64, err error) { stat, err := pool.Stat(path) if err != nil { return 0, err } return stat.Size(), nil } // Open returns io.ReadCloser to access the file func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { return os.Open(filepath.Join(pool.rootPath, path)) } // Stat returns Unix stat(2) info func (pool *PackagePool) Stat(path string) (os.FileInfo, error) { return os.Stat(filepath.Join(pool.rootPath, path)) } // Link generates hardlink to destination path func (pool *PackagePool) Link(path, dstPath string) error { return os.Link(filepath.Join(pool.rootPath, path), dstPath) } // Symlink generates symlink to destination path func (pool *PackagePool) Symlink(path, dstPath string) error { return os.Symlink(filepath.Join(pool.rootPath, path), dstPath) } // FullPath generates full path to the file in pool // // Please use with care: it's not supposed to be used to access files func (pool *PackagePool) FullPath(path string) string { return filepath.Join(pool.rootPath, path) } // GenerateTempPath generates temporary path for download (which is fast to import into package pool later on) func (pool *PackagePool) GenerateTempPath(filename string) (string, error) { random := uuid.NewString() return filepath.Join(pool.rootPath, random[0:2], random[2:4], random[4:]+filename), nil }