publish: lock resources from all SourceKinds

.github/workflows/ci.yml

@@ -1,4 +1,3 @@
----
 name: CI
 
 on:
@@ -11,6 +10,7 @@ on:
 
 defaults:
   run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
     shell: bash --noprofile --norc -eo pipefail {0}
 
 env:
@@ -33,7 +33,7 @@ jobs:
           make docker-image
       - name: "Unit Tests"
         run: |
-          make docker-unit-test
+          make docker-unit-tests
           mkdir -p out/coverage
           mv unit.out out/coverage/
       - uses: actions/upload-artifact@v4
@@ -140,7 +140,7 @@ jobs:
 
       - name: "Upload Code Coverage"
         if: github.actor != 'dependabot[bot]'
-        uses: codecov/codecov-action@v7.0.0
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.txt
@@ -155,7 +155,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 26.04", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
         arch: ["amd64", "i386" , "arm64" , "armhf"]
         include:
           - name: "Debian 13/trixie"
@@ -167,9 +167,6 @@ jobs:
           - name: "Debian 11/bullseye"
             suite: bullseye
             image: debian:bullseye-slim
-          - name: "Ubuntu 26.04"
-            suite: resolute
-            image: ubuntu:26.04
           - name: "Ubuntu 24.04"
             suite: noble
             image: ubuntu:24.04

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu
 ### List of Repositories
 
 * [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
-* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
 
@@ -130,14 +130,14 @@ aptly version: 1.5.0+189+g0fc90dff
 
 In order to run aptly unit tests, enter the following:
 ```
-make docker-unit-test
+make docker-unit-tests
 ```
 
 #### Running system tests
 
 In order to run aptly system tests, enter the following:
 ```
-make docker-system-test
+make docker-system-tests
 ```
 
 #### Running golangci-lint

Makefile

@@ -194,7 +194,7 @@ docker-shell:  ## Run aptly and other commands in docker container
 docker-deb:  ## Build debian packages in docker container
 	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
 
-docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
+docker-unit-tests:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
 	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
@@ -241,4 +241,4 @@ clean:  ## remove local build and module cache
 	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
 	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

api/gpg.go

@@ -54,7 +54,7 @@ type gpgDeleteKeyParams struct {
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
-// @Description Add GPG public keys for verifying remote repositories for mirroring.
+// @Description Add GPG public keys for veryfing remote repositories for mirroring.
 // @Description
 // @Description Keys can be added in two ways:
 // @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)

api/mirror.go

@@ -497,7 +497,7 @@ func apiMirrorsEdit(c *gin.Context) {
 type mirrorUpdateParams struct {
 	// Change mirror name to `Name`
 	Name string `                 json:"Name"                   example:"mirror1"`
-	// Gpg keyring(s) for verifying Release file
+	// Gpg keyring(s) for verifing Release file
 	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
 	// Set "true" to ignore checksum errors
 	IgnoreChecksums bool `        json:"IgnoreChecksums"`

api/publish.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"errors"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
@@ -124,7 +125,7 @@ func apiPublishList(c *gin.Context) {
 // @Description See also: `aptly publish show`
 // @Tags Publish
 // @Produce json
-// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs"
+// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs"
 // @Param distribution path string true "distribution name"
 // @Success 200 {object} deb.PublishedRepo
 // @Failure 404 {object} Error "Published repository not found"
@@ -255,13 +256,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	if b.SourceKind == deb.SourceSnapshot {
 		var snapshot *deb.Snapshot
 
-		snapshotCollection := collectionFactory.SnapshotCollection()
+		tmpCollection := collectionFactory.SnapshotCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 			names = append(names, source.Name)
 
-			snapshot, err = snapshotCollection.ByName(source.Name)
+			snapshot, err = tmpCollection.ByName(source.Name)
 			if err != nil {
 				AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
 				return
@@ -273,13 +274,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	} else if b.SourceKind == deb.SourceLocalRepo {
 		var localRepo *deb.LocalRepo
 
-		localCollection := collectionFactory.LocalRepoCollection()
+		tmpCollection := collectionFactory.LocalRepoCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 			names = append(names, source.Name)
 
-			localRepo, err = localCollection.ByName(source.Name)
+			localRepo, err = tmpCollection.ByName(source.Name)
 			if err != nil {
 				AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
 				return
@@ -332,8 +333,6 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		resources = append(resources, string(published.Key()))
-
 		if b.Origin != "" {
 			published.Origin = b.Origin
 		}
@@ -465,18 +464,60 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
+	resources := []string{string(published.Key())}
+
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
+		fmt.Printf("RACE DEBUG: deb.SourceLocalRepo\n")
+
+		// FIXME: lock repo ?
+		// localCollection := collectionFactory.LocalRepoCollection()
+		// for _, source := range b.Sources {
+		// 	components = append(components, source.Component)
+		// 	names = append(names, source.Name)
+
+		// 	localRepo, err = localCollection.ByName(source.Name)
+		// 	if err != nil {
+		// 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err))
+		// 		return
+		// 	}
+
+		// 	resources = append(resources, string(localRepo.Key()))
+		// }
 	} else if published.SourceKind == deb.SourceSnapshot {
+		fmt.Printf("RACE DEBUG: deb.SourceSnapshot: %s\n", b.Snapshots)
 		for _, snapshotInfo := range b.Snapshots {
-			_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err2 != nil {
 				AbortWithJSONError(c, http.StatusNotFound, err2)
 				return
 			}
+			resources = append(resources, string(snapshot.ResourceKey()))
+
+			for _, sourceID := range snapshot.SourceIDs {
+				if snapshot.SourceKind == deb.SourceSnapshot {
+					// FIXME: implement
+					err := errors.New("not implemented deb.SourceSnapshot")
+					AbortWithJSONError(c, http.StatusNotFound, err)
+					return
+				} else if snapshot.SourceKind == deb.SourceLocalRepo {
+					var repo *deb.LocalRepo
+					repo, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+					if err != nil {
+						AbortWithJSONError(c, http.StatusNotFound, err)
+						return
+					}
+					resources = append(resources, string(repo.Key()))
+				} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+					// FIXME: implement
+					err := errors.New("not implemented: deb.SourceRemoteRepo")
+					AbortWithJSONError(c, http.StatusNotFound, err)
+					return
+				}
+			}
 		}
 	} else {
 		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
@@ -515,7 +556,6 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.Version = *b.Version
 	}
 
-	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.LoadComplete(published, collectionFactory)

api/repos.go

@@ -3,7 +3,6 @@ package api
 import (
 	"fmt"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -103,7 +102,7 @@ type repoCreateParams struct {
 	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
 	// Default component when publishing from this local repo
 	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
-	// Snapshot name to create repository from (optional)
+	// Snapshot name to create repoitory from (optional)
 	FromSnapshot string `            json:"FromSnapshot"         example:""`
 }
 
@@ -176,19 +175,19 @@ func apiReposCreate(c *gin.Context) {
 
 type reposEditParams struct {
 	// Name of repository to modify
-	Name *string `                       json:"Name"                 example:"new-repo-name"`
+	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
 	// Change Comment of repository
 	Comment *string `                    json:"Comment"              example:"example repo"`
 	// Change Default Distribution for publishing
 	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
-	// Change Default Component for publishing
+	// Change Devault Component for publishing
 	DefaultComponent *string `        json:"DefaultComponent"     example:""`
 }
 
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Param name path string true "Repository name to modify"
+// @Param name path string true "Repository name"
 // @Consume  json
 // @Param request body reposEditParams true "Parameters"
 // @Produce json
@@ -201,20 +200,11 @@ func apiReposEdit(c *gin.Context) {
 	if c.Bind(&b) != nil {
 		return
 	}
-	if b.Name != nil {
-		new_name, err := url.PathUnescape(*b.Name)
-		if err != nil {
-			AbortWithJSONError(c, 400, err)
-			return
-		}
-		*b.Name = new_name
-	}
-
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	name := c.Params.ByName("name") // repo to modify
+	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)

deb/local.go

@@ -168,8 +168,6 @@ func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
 
 // LoadComplete loads additional information for local repo
 func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
-	repo.packageRefs = &PackageRefList{}
-
 	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
@@ -178,6 +176,7 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 		return err
 	}
 
+	repo.packageRefs = &PackageRefList{}
 	return repo.packageRefs.Decode(encoded)
 }
 

deb/local_test.go

@@ -133,18 +133,6 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(r.String(), Equals, repo.String())
 }
 
-func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) {
-	repo := NewLocalRepo("local1", "Comment 1")
-	c.Assert(s.collection.Update(repo), IsNil)
-
-	r, err := s.collection.ByName("local1")
-	c.Assert(err, IsNil)
-
-	c.Assert(s.collection.LoadComplete(r), IsNil)
-	c.Assert(r.packageRefs, NotNil)
-	c.Assert(r.NumPackages(), Equals, 0)
-}
-
 func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Update(repo), IsNil)

deb/reflist.go

@@ -79,9 +79,6 @@ func (l *PackageRefList) Decode(input []byte) error {
 
 // ForEach calls handler for each package ref in list
 func (l *PackageRefList) ForEach(handler func([]byte) error) error {
-	if l == nil {
-		return nil
-	}
 	var err error
 	for _, p := range l.Refs {
 		err = handler(p)

deb/reflist_test.go

@@ -130,17 +130,6 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
-func (s *PackageRefListSuite) TestForEachNilList(c *C) {
-	var l *PackageRefList
-	called := false
-	err := l.ForEach(func([]byte) error {
-		called = true
-		return nil
-	})
-	c.Assert(err, IsNil)
-	c.Assert(called, Equals, false)
-}
-
 func (s *PackageRefListSuite) TestHas(c *C) {
 	_ = s.list.Add(s.p1)
 	_ = s.list.Add(s.p3)

docs/Files.md

@@ -2,7 +2,7 @@
 <div>
 
 In order to add debian package files to a local repository, files are first uploaded to a temporary directory.
-Then the directory (or a specific file within) is added to a repository. After adding to a repository, the directory resp. files are removed bt default.
+Then the directory (or a specific file within) is added to a repository. After adding to a repositorty, the directory resp. files are removed bt default.
 
 All uploaded files are stored under `<rootDir>/upload/<tempdir>` directory.
 

docs/Packages.md

@@ -1,5 +1,5 @@
 # Search Package Collection
 <div>
-Perform operations on the whole collection of packages in aptly database.
+Perform operations on the whole collection of packages in apty database.
 </div>
 

docs/Publish.md

@@ -35,6 +35,6 @@ aptly publish repo my-repo --gpg-key=KEY_ID_a --gpg-key=KEY_ID_b
 #### Parameters
 
 Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`.  `:distribution` is distribution name, while `:prefix` is `[<storage>:]<prefix>` (storage is optional, it defaults to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores
-should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambiguous in URLs.
+should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambigious in URLs.
 
 </div>

docs/Repos.md

@@ -1,6 +1,6 @@
 # Manage Local Repositories
 <div>
-A local repository is a collection of versioned packages (usually custom packages created internally).
+A local repository is a collection of versionned packages (usually custom packages created internally).
 
 Packages can be added, removed, moved or copied between repos.
 

system/t12_api/repos.py

@@ -461,34 +461,3 @@ class ReposAPITestCopyPackage(APITest):
 
         self.check_equal(self.get(f"/api/repos/{repo2_name}/packages").json(),
                          ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378'])
-
-
-class ReposAPITestCreateEdit(APITest):
-    """
-    POST /api/repos,
-    """
-    def check(self):
-        repo_name = self.random_name() + ' with space'
-        repo_desc = {'Comment': 'fun repo',
-                     'DefaultComponent': 'contrib',
-                     'DefaultDistribution': 'bookworm',
-                     'Name': repo_name}
-
-        resp = self.post("/api/repos", json=repo_desc)
-        self.check_equal(resp.json(), repo_desc)
-        self.check_equal(resp.status_code, 201)
-
-        repo_desc = {'Comment': 'modified repo',
-                     'DefaultComponent': 'main',
-                     'DefaultDistribution': 'trixie',
-                     'Name': repo_name + '@renamed'}
-        resp = self.put(f"/api/repos/{repo_name}", json=repo_desc)
-        self.check_equal(resp.json(), repo_desc)
-        self.check_equal(resp.status_code, 200)
-
-        resp = self.get("/api/repos/" + repo_name + '@renamed')
-        self.check_equal(resp.json(), repo_desc)
-        self.check_equal(resp.status_code, 200)
-
-        resp = self.delete("/api/repos/" + repo_name + '@renamed')
-        self.check_equal(resp.status_code, 200)

system/testout.py

@@ -3,7 +3,7 @@ import tempfile
 
 class TestOut:
     def __init__(self):
-        self.tmp_file = tempfile.NamedTemporaryFile(delete=True)
+        self.tmp_file = tempfile.NamedTemporaryFile(delete=False)
         self.read_pos = 0
 
     def fileno(self):

task/list.go

@@ -65,6 +65,7 @@ func (list *List) consumer() {
 						task.State = SUCCEEDED
 					}
 
+					fmt.Printf("RACE DEBUG: Task %s done, freeing %s\n", task.Name, task.resources)
 					list.usedResources.Free(task.resources)
 
 					task.wgTask.Done()
@@ -77,6 +78,8 @@ func (list *List) consumer() {
 							blockingTasks := list.usedResources.UsedBy(t.resources)
 							if len(blockingTasks) == 0 {
 								list.usedResources.MarkInUse(t.resources, t)
+
+								fmt.Printf("RACE DEBUG: Starting queued task %s, using %s\n", t.Name, t.resources)
 								// unlock list since queueing may block
 								list.Unlock()
 								unlocked = true
@@ -209,10 +212,12 @@ func (list *List) RunTaskInBackground(name string, resources []string, process P
 	tasks := list.usedResources.UsedBy(resources)
 	if len(tasks) == 0 {
 		list.usedResources.MarkInUse(task.resources, task)
+		fmt.Printf("RACE DEBUG: Starting task %s, using %s\n", name, resources)
 		// queueing task might block if channel not ready, unlock list before queueing
 		list.Unlock()
 		list.queue <- task
 	} else {
+		fmt.Printf("RACE DEBUG: Queued task %s, locked %s\n", name, resources)
 		list.Unlock()
 	}