publish: lock resources from all SourceKinds

Fix/launchpad test dependency

Makefile

@@ -131,6 +131,7 @@ serve: prepare swagger-install  ## Run development server (auto recompiling)
 	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf ~/.aptly.conf
 	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
 	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
 dpkg: prepare swagger  ## Build debian packages

api/publish.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"errors"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
@@ -255,13 +256,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	if b.SourceKind == deb.SourceSnapshot {
 		var snapshot *deb.Snapshot
 
-		snapshotCollection := collectionFactory.SnapshotCollection()
+		tmpCollection := collectionFactory.SnapshotCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 			names = append(names, source.Name)
 
-			snapshot, err = snapshotCollection.ByName(source.Name)
+			snapshot, err = tmpCollection.ByName(source.Name)
 			if err != nil {
 				AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
 				return
@@ -273,13 +274,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	} else if b.SourceKind == deb.SourceLocalRepo {
 		var localRepo *deb.LocalRepo
 
-		localCollection := collectionFactory.LocalRepoCollection()
+		tmpCollection := collectionFactory.LocalRepoCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 			names = append(names, source.Name)
 
-			localRepo, err = localCollection.ByName(source.Name)
+			localRepo, err = tmpCollection.ByName(source.Name)
 			if err != nil {
 				AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
 				return
@@ -332,8 +333,6 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		resources = append(resources, string(published.Key()))
-
 		if b.Origin != "" {
 			published.Origin = b.Origin
 		}
@@ -465,18 +464,60 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
+	resources := []string{string(published.Key())}
+
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
+		fmt.Printf("RACE DEBUG: deb.SourceLocalRepo\n")
+
+		// FIXME: lock repo ?
+		// localCollection := collectionFactory.LocalRepoCollection()
+		// for _, source := range b.Sources {
+		// 	components = append(components, source.Component)
+		// 	names = append(names, source.Name)
+
+		// 	localRepo, err = localCollection.ByName(source.Name)
+		// 	if err != nil {
+		// 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
+		// 		return
+		// 	}
+
+		// 	resources = append(resources, string(localRepo.Key()))
+		// }
 	} else if published.SourceKind == deb.SourceSnapshot {
+		fmt.Printf("RACE DEBUG: deb.SourceSnapshot: %s\n", b.Snapshots)
 		for _, snapshotInfo := range b.Snapshots {
-			_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err2 != nil {
 				AbortWithJSONError(c, http.StatusNotFound, err2)
 				return
 			}
+			resources = append(resources, string(snapshot.ResourceKey()))
+
+			for _, sourceID := range snapshot.SourceIDs {
+				if snapshot.SourceKind == deb.SourceSnapshot {
+					// FIXME: implement
+					err := errors.New("not implemented deb.SourceSnapshot")
+					AbortWithJSONError(c, http.StatusNotFound, err)
+					return
+				} else if snapshot.SourceKind == deb.SourceLocalRepo {
+					var repo *deb.LocalRepo
+					repo, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+					if err != nil {
+						AbortWithJSONError(c, http.StatusNotFound, err)
+						return
+					}
+					resources = append(resources, string(repo.Key()))
+				} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+					// FIXME: implement
+					err := errors.New("not implemented: deb.SourceRemoteRepo")
+					AbortWithJSONError(c, http.StatusNotFound, err)
+					return
+				}
+			}
 		}
 	} else {
 		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
@@ -515,7 +556,6 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.Version = *b.Version
 	}
 
-	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.LoadComplete(published, collectionFactory)

api/repos.go

@@ -420,6 +420,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
 // @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
@@ -623,11 +624,11 @@ type reposCopyPackageParams struct {
 // @Summary Copy Package
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
-// @Produce json
 // @Param name path string true "Destination repo"
 // @Param src path string true "Source repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"

api/router.go

@@ -18,6 +18,12 @@ import (
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		countPackagesByRepos()

deb/ppa.go

@@ -28,6 +28,11 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 		}
 	}
 
+	baseurl := config.PpaBaseURL
+	if baseurl == "" {
+		baseurl = "http://ppa.launchpad.net"
+	}
+
 	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
@@ -39,7 +44,7 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 
 	distribution = codename
 	components = []string{"main"}
-	url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID)
+	url = fmt.Sprintf("%s/%s/%s/%s", baseurl, matches[1], matches[2], distributorID)
 
 	return
 }

debian/aptly.conf

@@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu
 # Codename for short PPA url expansion
 ppa_codename: ""
 
+# PPA Base URL (default: launchpad)
+# # ppa_baseurl: http://ppa.launchpad.net
+
 
 # Aptly Server
 ###############

system/api_lib.py

@@ -34,6 +34,7 @@ class APITest(BaseTest):
                 "linkMethod": "symlink"
             }
         },
+        "enableMetricsEndpoint": True,
         "enableSwaggerEndpoint": True
     }
 

system/t02_config/ConfigShowTest_gold

@@ -12,6 +12,7 @@
     "dependencyVerboseResolve": false,
     "ppaDistributorID": "ubuntu",
     "ppaCodename": "",
+    "ppaBaseURL": "http://ppa.launchpad.net",
     "serveInAPIMode": true,
     "enableMetricsEndpoint": true,
     "enableSwaggerEndpoint": false,

system/t02_config/ConfigShowYAMLTest_gold

@@ -11,6 +11,7 @@ dep_follow_source: false
 dep_verboseresolve: false
 ppa_distributor_id: ubuntu
 ppa_codename: ""
+ppa_baseurl: http://ppa.launchpad.net
 serve_in_api_mode: true
 enable_metrics_endpoint: true
 enable_swagger_endpoint: false

system/t02_config/CreateConfigTest_gold

@@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu
 # Codename for short PPA url expansion
 ppa_codename: ""
 
+# PPA Base URL (default: launchpad)
+# # ppa_baseurl: http://ppa.launchpad.net
+
 
 # Aptly Server
 ###############

system/t04_mirror/CreateMirror18Test_gold

@@ -1,4 +1,4 @@
-Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
+Downloading: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
 gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC
 gpgv:                using RSA key 5BFCD481D86D5824470E469F9000B1C3A01F726C
 gpgv: Good signature from "Launchpad PPA for Anton Gladky"
@@ -6,5 +6,5 @@ gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC
 gpgv:                using RSA key 02219381E9161C78A46CB2BFA5279A973B1F56C0
 gpgv: Good signature from "Launchpad sim"
 
-Mirror [mirror18]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick successfully added.
+Mirror [mirror18]: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/ maverick successfully added.
 You can run 'aptly mirror update mirror18' to download repository contents.

system/t04_mirror/CreateMirror18Test_mirror_show

@@ -1,5 +1,5 @@
 Name: mirror18
-Archive Root URL: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/
+Archive Root URL: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/
 Distribution: maverick
 Components: main
 Architectures: amd64, armel, i386, powerpc

system/t04_mirror/create.py

@@ -221,6 +221,7 @@ class CreateMirror18Test(BaseTest):
         "max-tries": 1,
         "ppaDistributorID": "ubuntu",
         "ppaCodename": "maverick",
+        "ppaBaseURL": "http://repo.aptly.info/system-tests/ppa",
     }
 
     fixtureCmds = [

task/list.go

@@ -65,6 +65,7 @@ func (list *List) consumer() {
 						task.State = SUCCEEDED
 					}
 
+					fmt.Printf("RACE DEBUG: Task %s done, freeing %s\n", task.Name, task.resources)
 					list.usedResources.Free(task.resources)
 
 					task.wgTask.Done()
@@ -77,6 +78,8 @@ func (list *List) consumer() {
 							blockingTasks := list.usedResources.UsedBy(t.resources)
 							if len(blockingTasks) == 0 {
 								list.usedResources.MarkInUse(t.resources, t)
+
+								fmt.Printf("RACE DEBUG: Starting queued task %s, using %s\n", t.Name, t.resources)
 								// unlock list since queueing may block
 								list.Unlock()
 								unlocked = true
@@ -209,10 +212,12 @@ func (list *List) RunTaskInBackground(name string, resources []string, process P
 	tasks := list.usedResources.UsedBy(resources)
 	if len(tasks) == 0 {
 		list.usedResources.MarkInUse(task.resources, task)
+		fmt.Printf("RACE DEBUG: Starting task %s, using %s\n", name, resources)
 		// queueing task might block if channel not ready, unlock list before queueing
 		list.Unlock()
 		list.queue <- task
 	} else {
+		fmt.Printf("RACE DEBUG: Queued task %s, locked %s\n", name, resources)
 		list.Unlock()
 	}
 

utils/config.go

@@ -31,6 +31,7 @@ type ConfigStructure struct { // nolint: maligned
 	// PPA
 	PpaDistributorID string `json:"ppaDistributorID"              yaml:"ppa_distributor_id"`
 	PpaCodename      string `json:"ppaCodename"                   yaml:"ppa_codename"`
+	PpaBaseURL       string `json:"ppaBaseURL"                    yaml:"ppa_baseurl"`
 
 	// Server
 	ServeInAPIMode        bool `json:"serveInAPIMode"                yaml:"serve_in_api_mode"`
@@ -235,6 +236,7 @@ var Config = ConfigStructure{
 	SkipLegacyPool:         false,
 	PpaDistributorID:       "ubuntu",
 	PpaCodename:            "",
+	PpaBaseURL:             "http://ppa.launchpad.net",
 	FileSystemPublishRoots: map[string]FileSystemPublishRoot{},
 	S3PublishRoots:         map[string]S3PublishRoot{},
 	SwiftPublishRoots:      map[string]SwiftPublishRoot{},

utils/config_test.go

@@ -85,6 +85,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"  \"dependencyVerboseResolve\": false,\n" +
 		"  \"ppaDistributorID\": \"\",\n" +
 		"  \"ppaCodename\": \"\",\n" +
+		"  \"ppaBaseURL\": \"\",\n" +
 		"  \"serveInAPIMode\": false,\n" +
 		"  \"enableMetricsEndpoint\": false,\n" +
 		"  \"enableSwaggerEndpoint\": false,\n" +
@@ -252,6 +253,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) {
 		"dep_verboseresolve: false\n" +
 		"ppa_distributor_id: \"\"\n" +
 		"ppa_codename: \"\"\n" +
+		"ppa_baseurl: \"\"\n" +
 		"serve_in_api_mode: false\n" +
 		"enable_metrics_endpoint: false\n" +
 		"enable_swagger_endpoint: false\n" +
@@ -308,6 +310,7 @@ dep_follow_source: true
 dep_verboseresolve: true
 ppa_distributor_id: Ubuntu
 ppa_codename: code
+ppa_baseurl: http://ppa.launchpad.net
 serve_in_api_mode: true
 enable_metrics_endpoint: true
 enable_swagger_endpoint: true