New upstream version 1.5.0+ds1

api/repos.go

@@ -2,13 +2,17 @@ package api
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
@@ -17,11 +21,9 @@ import (
 func apiReposList(c *gin.Context) {
 	result := []*deb.LocalRepo{}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
-	defer collection.RUnlock()
-
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	collection.ForEach(func(r *deb.LocalRepo) error {
 		result = append(result, r)
 		return nil
 	})
@@ -46,11 +48,9 @@ func apiReposCreate(c *gin.Context) {
 	repo.DefaultComponent = b.DefaultComponent
 	repo.DefaultDistribution = b.DefaultDistribution
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
-
-	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	err := collection.Add(repo)
 	if err != nil {
 		c.AbortWithError(400, err)
 		return
@@ -62,6 +62,7 @@ func apiReposCreate(c *gin.Context) {
 // PUT /api/repos/:name
 func apiReposEdit(c *gin.Context) {
 	var b struct {
+		Name                *string
 		Comment             *string
 		DefaultDistribution *string
 		DefaultComponent    *string
@@ -71,9 +72,8 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -81,6 +81,15 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
+	if b.Name != nil {
+		_, err := collection.ByName(*b.Name)
+		if err == nil {
+			// already exists
+			c.AbortWithError(404, err)
+			return
+		}
+		repo.Name = *b.Name
+	}
 	if b.Comment != nil {
 		repo.Comment = *b.Comment
 	}
@@ -102,9 +111,8 @@ func apiReposEdit(c *gin.Context) {
 
 // GET /api/repos/:name
 func apiReposShow(c *gin.Context) {
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
-	defer collection.RUnlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -118,53 +126,42 @@ func apiReposShow(c *gin.Context) {
 // DELETE /api/repos/:name
 func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	name := c.Params.ByName("name")
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.RLock()
-	defer snapshotCollection.RUnlock()
-
-	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
-	publishedCollection.RLock()
-	defer publishedCollection.RUnlock()
-
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	repo, err := collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
 	}
 
-	published := publishedCollection.ByLocalRepo(repo)
-	if len(published) > 0 {
-		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
-		return
-	}
-
-	if !force {
-		snapshots := snapshotCollection.ByLocalRepoSource(repo)
-		if len(snapshots) > 0 {
-			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
-			return
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.ByLocalRepo(repo)
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
 		}
-	}
 
-	err = collection.Drop(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
+		if !force {
+			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
+			}
+		}
 
-	c.JSON(200, gin.H{})
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+	})
 }
 
 // GET /api/repos/:name/packages
 func apiReposPackagesShow(c *gin.Context) {
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -178,11 +175,11 @@ func apiReposPackagesShow(c *gin.Context) {
 		return
 	}
 
-	showPackages(c, repo.RefList())
+	showPackages(c, repo.RefList(), collectionFactory)
 }
 
 // Handler for both add and delete
-func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
 	var b struct {
 		PackageRefs []string
 	}
@@ -191,9 +188,8 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -207,54 +203,54 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		return
 	}
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
-
-	// verify package refs and build package list
-	for _, ref := range b.PackageRefs {
-		var p *deb.Package
-
-		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+	resources := []string{string(repo.Key())}
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Loading packages...\n")
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
 		if err != nil {
-			if err == database.ErrNotFound {
-				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
-			} else {
-				c.AbortWithError(500, err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			var p *deb.Package
+
+			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
+				}
+
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = cb(list, p, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
-			return
 		}
-		err = cb(list, p)
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
-			c.AbortWithError(400, err)
-			return
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
-	}
-
-	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
-		return
-	}
-
-	c.JSON(200, repo)
-
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
 }
 
 // POST /repos/:name/packages
 func apiReposPackagesAdd(c *gin.Context) {
-	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Adding package %s\n", p.Name)
 		return list.Add(p)
 	})
 }
 
 // DELETE /repos/:name/packages
 func apiReposPackagesDelete(c *gin.Context) {
-	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Removing package %s\n", p.Name)
 		list.Remove(p)
 		return nil
 	})
@@ -275,17 +271,18 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
+	dirParam := c.Params.ByName("dir")
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
 		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
@@ -297,75 +294,91 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	verifier := context.GetVerifier()
-
-	var (
-		sources                      []string
-		packageFiles, failedFiles    []string
-		otherFiles                   []string
-		processedFiles, failedFiles2 []string
-		reporter                     = &aptly.RecordingResultReporter{
-			Warnings:     []string{},
-			AddedLines:   []string{},
-			RemovedLines: []string{},
-		}
-		list *deb.PackageList
-	)
-
+	var taskName string
+	var sources []string
 	if fileParam == "" {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+		taskName = fmt.Sprintf("Add packages from dir %s to repo %s", dirParam, name)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
 	} else {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+		taskName = fmt.Sprintf("Add package %s from dir %s to repo %s", fileParam, dirParam, name)
 	}
 
-	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+	resources := []string{string(repo.Key())}
+	resources = append(resources, sources...)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		verifier := context.GetVerifier()
 
-	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
-		return
-	}
-
-	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
-	failedFiles = append(failedFiles, failedFiles2...)
-
-	processedFiles = append(processedFiles, otherFiles...)
-
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
-		return
-	}
-
-	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
-		return
-	}
-
-	if !noRemove {
-		processedFiles = utils.StrSliceDeduplicate(processedFiles)
-
-		for _, file := range processedFiles {
-			err := os.Remove(file)
-			if err != nil {
-				reporter.Warning("unable to remove file %s: %s", file, err)
+		var (
+			packageFiles, failedFiles    []string
+			otherFiles                   []string
+			processedFiles, failedFiles2 []string
+			reporter                     = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
 			}
+			list *deb.PackageList
+		)
+
+		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
 		}
 
-		// atempt to remove dir, if it fails, that's fine: probably it's not empty
-		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
-	}
+		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+		failedFiles = append(failedFiles, failedFiles2...)
+		processedFiles = append(processedFiles, otherFiles...)
 
-	if failedFiles == nil {
-		failedFiles = []string{}
-	}
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
+		}
 
-	c.JSON(200, gin.H{
-		"Report":      reporter,
-		"FailedFiles": failedFiles,
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+
+		if !noRemove {
+			processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+			for _, file := range processedFiles {
+				err := os.Remove(file)
+				if err != nil {
+					reporter.Warning("unable to remove file %s: %s", file, err)
+				}
+			}
+
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
 	})
 }
 
@@ -383,62 +396,100 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
 
 	repoTemplateString := c.Params.ByName("name")
+	collectionFactory := context.NewCollectionFactory()
 
 	if !verifyDir(c) {
 		return
 	}
 
+	var sources []string
+	var taskName string
+	dirParam := c.Params.ByName("dir")
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
 		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	var (
-		err                       error
-		verifier                  = context.GetVerifier()
-		sources, changesFiles     []string
-		failedFiles, failedFiles2 []string
-		reporter                  = &aptly.RecordingResultReporter{
-			Warnings:     []string{},
-			AddedLines:   []string{},
-			RemovedLines: []string{},
-		}
-	)
-
 	if fileParam == "" {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+		taskName = fmt.Sprintf("Include packages from changes files in dir %s to repo matching template %s", dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
 	} else {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+		taskName = fmt.Sprintf("Include packages from changes file %s from dir %s to repo matching template %s", fileParam, dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
 	}
 
-	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.Lock()
-	defer localRepoCollection.Unlock()
-
-	changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
-	_, failedFiles2, err = deb.ImportChangesFiles(
-		changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
-		repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
-		context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
-	failedFiles = append(failedFiles, failedFiles2...)
-
+	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
+		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
 		return
 	}
 
-	if !noRemoveFiles {
-		// atempt to remove dir, if it fails, that's fine: probably it's not empty
-		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
-	}
+	var resources []string
+	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+		resources = append(resources, task.AllLocalReposResourcesKey)
+	} else {
+		// repo template string is simple text so only use resource key of specific repository
+		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
 
-	if failedFiles == nil {
-		failedFiles = []string{}
+		resources = append(resources, string(repo.Key()))
 	}
+	resources = append(resources, sources...)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var (
+			err                       error
+			verifier                  = context.GetVerifier()
+			changesFiles              []string
+			failedFiles, failedFiles2 []string
+			reporter                  = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+		)
+
+		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+		_, failedFiles2, err = deb.ImportChangesFiles(
+			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+		failedFiles = append(failedFiles, failedFiles2...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import changes files: %s", err)
+		}
+
+		if !noRemoveFiles {
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
 
-	c.JSON(200, gin.H{
-		"Report":      reporter,
-		"FailedFiles": failedFiles,
 	})
 }