Use github.com/saracen/walker for file walk operations

In some local tests w/ a slowed down filesystem, this massively cut down
on the time to clean up a repository by ~3x, bringing a total 'publish
update' time from ~16s to ~13s.

Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com>

api/files.go

@@ -6,8 +6,10 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
 func verifyPath(path string) bool {
@@ -34,17 +36,16 @@ func verifyDir(c *gin.Context) bool {
 // GET /files
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -121,6 +122,7 @@ func apiFilesListFiles(c *gin.Context) {
 	}
 
 	list := []string{}
+	listLock := &sync.Mutex{}
 	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
 
 	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
@@ -132,6 +134,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil

deb/changes.go

@@ -8,11 +8,13 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/saracen/walker"
 )
 
 // Changes is a result of .changes file parsing
@@ -247,6 +249,8 @@ func (c *Changes) GetArchitecture() string {
 
 // CollectChangesFiles walks filesystem collecting all .changes files
 func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	changesFilesLock := &sync.Mutex{}
+
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -255,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 			continue
 		}
 		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
+			err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
 				if info.IsDir() {
 					return nil
 				}
 
 				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFilesLock.Lock()
+					defer changesFilesLock.Unlock()
 					changesFiles = append(changesFiles, path)
 				}
 

deb/import.go

@@ -5,14 +5,19 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/saracen/walker"
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
 func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
+	packageFilesLock := &sync.Mutex{}
+	otherFilesLock := &sync.Mutex{}
+
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -21,18 +26,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			continue
 		}
 		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
+			err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
 				if info.IsDir() {
 					return nil
 				}
 
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
+					packageFilesLock.Lock()
+					defer packageFilesLock.Unlock()
 					packageFiles = append(packageFiles, path)
 				} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+					otherFilesLock.Lock()
+					defer otherFilesLock.Unlock()
 					otherFiles = append(otherFiles, path)
 				}
 

files/package_pool.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 	"syscall"
 
 	"github.com/pborman/uuid"
+	"github.com/saracen/walker"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
@@ -98,13 +100,13 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error)
 	}
 
 	result := []string{}
+	resultLock := &sync.Mutex{}
 
 	for _, dir := range dirs {
-		err = filepath.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
+		err = walker.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo) error {
 			if !info.IsDir() {
+				resultLock.Lock()
+				defer resultLock.Unlock()
 				result = append(result, path[len(pool.rootPath)+1:])
 			}
 			return nil
@@ -118,6 +120,7 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error)
 		}
 	}
 
+	sort.Strings(result)
 	return result, nil
 }
 

files/public.go

@@ -5,11 +5,14 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"sync"
 	"syscall"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/saracen/walker"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
@@ -232,12 +235,12 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 	root := filepath.Join(storage.rootPath, prefix)
 	result := []string{}
+	resultLock := &sync.Mutex{}
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
+	err := walker.Walk(root, func(path string, info os.FileInfo) error {
 		if !info.IsDir() {
+			resultLock.Lock()
+			defer resultLock.Unlock()
 			result = append(result, path[len(root)+1:])
 		}
 		return nil
@@ -248,6 +251,7 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 		return []string{}, nil
 	}
 
+	sort.Strings(result)
 	return result, err
 }
 

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rs/zerolog v1.30.0
+	github.com/saracen/walker v0.1.3
 	github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5
 	github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76
 	github.com/smira/go-ftp-protocol v0.0.0-20140829150050-066b75c2b70d
@@ -92,6 +93,8 @@ require (
 	golang.org/x/arch v0.5.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

go.sum

@@ -218,6 +218,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g=
+github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk=
 github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5 h1:jLFwP6SDEUHmb6QSu5n2FHseWzMio1ou1FV9p7W6p7I=
 github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5/go.mod h1:XTQy55hw5s3pxmC42m7X0/b+9naXQ1rGN9Of6BGIZmU=
 github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76 h1:OM075OkN4x9IB1mbzkzaKaJjFxx8Mfss8Z3E1LHwawQ=
@@ -276,6 +278,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=