From 8ab8398c50d758fea5f01b9d7ec24585c4a18102 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez Date: Tue, 5 Sep 2023 15:22:24 -0500 Subject: [PATCH] Use github.com/saracen/walker for file walk operations In some local tests w/ a slowed down filesystem, this massively cut down on the time to clean up a repository by ~3x, bringing a total 'publish update' time from ~16s to ~13s. Signed-off-by: Ryan Gonzalez --- api/files.go | 14 +++++++++----- deb/changes.go | 11 +++++++---- deb/import.go | 14 ++++++++++---- files/package_pool.go | 11 +++++++---- files/public.go | 12 ++++++++---- go.mod | 3 +++ go.sum | 4 ++++ 7 files changed, 48 insertions(+), 21 deletions(-) diff --git a/api/files.go b/api/files.go index 3e7ad149..745d41c8 100644 --- a/api/files.go +++ b/api/files.go @@ -6,8 +6,10 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/gin-gonic/gin" + "github.com/saracen/walker" ) func verifyPath(path string) bool { @@ -34,17 +36,16 @@ func verifyDir(c *gin.Context) bool { // GET /files func apiFilesListDirs(c *gin.Context) { list := []string{} + listLock := &sync.Mutex{} - err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - + err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error { if path == context.UploadPath() { return nil } if info.IsDir() { + listLock.Lock() + defer listLock.Unlock() list = append(list, filepath.Base(path)) return filepath.SkipDir } @@ -121,6 +122,7 @@ func apiFilesListFiles(c *gin.Context) { } list := []string{} + listLock := &sync.Mutex{} root := filepath.Join(context.UploadPath(), c.Params.ByName("dir")) err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error { @@ -132,6 +134,8 @@ func apiFilesListFiles(c *gin.Context) { return nil } + listLock.Lock() + defer listLock.Unlock() list = append(list, filepath.Base(path)) return nil diff --git a/deb/changes.go b/deb/changes.go index 22c1fd86..c264986a 100644 --- a/deb/changes.go +++ b/deb/changes.go @@ -8,11 +8,13 @@ import ( "path/filepath" "sort" "strings" + "sync" "text/template" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // Changes is a result of .changes file parsing @@ -247,6 +249,8 @@ func (c *Changes) GetArchitecture() string { // CollectChangesFiles walks filesystem collecting all .changes files func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) { + changesFilesLock := &sync.Mutex{} + for _, location := range locations { info, err2 := os.Stat(location) if err2 != nil { @@ -255,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha continue } if info.IsDir() { - err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error { - if err3 != nil { - return err3 - } + err2 = walker.Walk(location, func(path string, info os.FileInfo) error { if info.IsDir() { return nil } if strings.HasSuffix(info.Name(), ".changes") { + changesFilesLock.Lock() + defer changesFilesLock.Unlock() changesFiles = append(changesFiles, path) } diff --git a/deb/import.go b/deb/import.go index 2a2336f4..d0f65479 100644 --- a/deb/import.go +++ b/deb/import.go @@ -5,14 +5,19 @@ import ( "path/filepath" "sort" "strings" + "sync" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // CollectPackageFiles walks filesystem collecting all candidates for package files func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) { + packageFilesLock := &sync.Mutex{} + otherFilesLock := &sync.Mutex{} + for _, location := range locations { info, err2 := os.Stat(location) if err2 != nil { @@ -21,18 +26,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac continue } if info.IsDir() { - err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error { - if err3 != nil { - return err3 - } + err2 = walker.Walk(location, func(path string, info os.FileInfo) error { if info.IsDir() { return nil } if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") || strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") { + packageFilesLock.Lock() + defer packageFilesLock.Unlock() packageFiles = append(packageFiles, path) } else if strings.HasSuffix(info.Name(), ".buildinfo") { + otherFilesLock.Lock() + defer otherFilesLock.Unlock() otherFiles = append(otherFiles, path) } diff --git a/files/package_pool.go b/files/package_pool.go index 65a016d5..b56b58eb 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -5,10 +5,12 @@ import ( "io" "os" "path/filepath" + "sort" "sync" "syscall" "github.com/pborman/uuid" + "github.com/saracen/walker" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" @@ -98,13 +100,13 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) } result := []string{} + resultLock := &sync.Mutex{} for _, dir := range dirs { - err = filepath.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + err = walker.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo) error { if !info.IsDir() { + resultLock.Lock() + defer resultLock.Unlock() result = append(result, path[len(pool.rootPath)+1:]) } return nil @@ -118,6 +120,7 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) } } + sort.Strings(result) return result, nil } diff --git a/files/public.go b/files/public.go index cf7e8ac4..cab54c97 100644 --- a/files/public.go +++ b/files/public.go @@ -5,11 +5,14 @@ import ( "io" "os" "path/filepath" + "sort" "strings" + "sync" "syscall" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // PublishedStorage abstract file system with public dirs (published repos) @@ -232,12 +235,12 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { root := filepath.Join(storage.rootPath, prefix) result := []string{} + resultLock := &sync.Mutex{} - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + err := walker.Walk(root, func(path string, info os.FileInfo) error { if !info.IsDir() { + resultLock.Lock() + defer resultLock.Unlock() result = append(result, path[len(root)+1:]) } return nil @@ -248,6 +251,7 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { return []string{}, nil } + sort.Strings(result) return result, err } diff --git a/go.mod b/go.mod index 7e8e29db..91eeec1e 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/rs/zerolog v1.30.0 + github.com/saracen/walker v0.1.3 github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5 github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76 github.com/smira/go-ftp-protocol v0.0.0-20140829150050-066b75c2b70d @@ -92,6 +93,8 @@ require ( golang.org/x/arch v0.5.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e925eaa8..46dc68f8 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g= +github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk= github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5 h1:jLFwP6SDEUHmb6QSu5n2FHseWzMio1ou1FV9p7W6p7I= github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5/go.mod h1:XTQy55hw5s3pxmC42m7X0/b+9naXQ1rGN9Of6BGIZmU= github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76 h1:OM075OkN4x9IB1mbzkzaKaJjFxx8Mfss8Z3E1LHwawQ= @@ -276,6 +278,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=