From 67bd15487d5760fccae32516061d57d1ccbb0ebf Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Mon, 14 Apr 2025 13:20:08 +0200 Subject: [PATCH 1/7] Fixes Issue#1435. Signed-off-by: Christoph Fiehe --- s3/public.go | 4 ++-- s3/public_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/s3/public.go b/s3/public.go index 47623a00..234a4b22 100644 --- a/s3/public.go +++ b/s3/public.go @@ -338,7 +338,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, poolPath := filepath.Join(storage.prefix, relPath) if storage.pathCache == nil { - paths, md5s, err := storage.internalFilelist(filepath.Join(storage.prefix, publishedPrefix, "pool"), true) + paths, md5s, err := storage.internalFilelist(filepath.Join(publishedPrefix, "pool"), true) if err != nil { return errors.Wrap(err, "error caching paths under prefix") } @@ -346,7 +346,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, storage.pathCache = make(map[string]string, len(paths)) for i := range paths { - storage.pathCache[paths[i]] = md5s[i] + storage.pathCache[filepath.Join("pool", paths[i])] = md5s[i] } } diff --git a/s3/public_test.go b/s3/public_test.go index 83605e3e..a6631dad 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -370,7 +370,7 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) { // Check only one listing request was done to the server s.checkGetRequestsEqual(c, "/test?", []string{ - "/test?list-type=2&max-keys=1000&prefix=lala%2Flala%2Fpool%2F", + "/test?list-type=2&max-keys=1000&prefix=lala%2Fpool%2F", }) // Publish two packages at a prefixed storage plus a publish prefix. From 9ef217b351e27da8aabbfb8f7b4f47dc8c5c23cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 19 Apr 2025 12:58:48 +0200 Subject: [PATCH 2/7] ci: use go 1.23 compatible with gocovmerge --- .github/workflows/ci.yml | 2 +- go.mod | 16 ++++++++-------- go.sum | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e82307bb..c65b90a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: - name: "Merge Code Coverage" run: | - go install github.com/wadey/gocovmerge@latest + go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt - name: "Upload Code Coverage" diff --git a/go.mod b/go.mod index 2f1de401..2e0703aa 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aptly-dev/aptly -go 1.22.7 +go 1.23 require ( github.com/AlekSi/pointer v1.1.0 @@ -32,9 +32,9 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/ugorji/go/codec v1.2.11 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 - golang.org/x/term v0.27.0 + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/sys v0.30.0 + golang.org/x/term v0.29.0 golang.org/x/time v0.5.0 google.golang.org/protobuf v1.34.2 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -103,10 +103,10 @@ require ( go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/tools v0.30.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/grpc v1.64.1 // indirect diff --git a/go.sum b/go.sum index 1e03143e..e4f3b575 100644 --- a/go.sum +++ b/go.sum @@ -324,12 +324,15 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -348,6 +351,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -357,6 +362,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -385,6 +392,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -392,6 +401,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -401,6 +412,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -412,6 +425,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= +golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 664a5cd67538df60b410094eb31ac6b96f877ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 12 Apr 2025 22:05:36 +0200 Subject: [PATCH 3/7] go1.23: fix system test --- system/t07_serve/Serve1Test_http | 2 ++ 1 file changed, 2 insertions(+) diff --git a/system/t07_serve/Serve1Test_http b/system/t07_serve/Serve1Test_http index 0b6d4468..3d380ca5 100644 --- a/system/t07_serve/Serve1Test_http +++ b/system/t07_serve/Serve1Test_http @@ -1,6 +1,8 @@ + 
 debian/
 dists/
 multi/
 pool/
+
From e062df68c5b32bed3de1fd5ea44e160419b314a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 20 Apr 2025 20:15:26 +0200 Subject: [PATCH 4/7] go1.23: update golangci-lint version and fix warnings. --- .github/workflows/golangci-lint.yml | 2 +- Makefile | 4 ++-- api/publish.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 8da4335a..dad2bdc5 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -44,7 +44,7 @@ jobs: # Require: The version of golangci-lint to use. # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. - version: v1.54.1 + version: v1.64.5 # Optional: working directory, useful for monorepos # working-directory: somedir diff --git a/Makefile b/Makefile index 680c738c..4682f774 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ GOPATH=$(shell go env GOPATH) VERSION=$(shell make -s version) PYTHON?=python3 BINPATH?=$(GOPATH)/bin -GOLANGCI_LINT_VERSION=v1.54.1 # version supporting go 1.19 +GOLANGCI_LINT_VERSION=v1.64.5 # version supporting go 1.23 COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) @@ -71,7 +71,7 @@ lint: prepare # Install golangci-lint @test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) # Running lint - @PATH=$(BINPATH)/:$(PATH) golangci-lint run + @PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter 0 --max-same-issues 0 build: prepare swagger ## Build aptly diff --git a/api/publish.go b/api/publish.go index e19afb17..1245790c 100644 --- a/api/publish.go +++ b/api/publish.go @@ -625,7 +625,7 @@ func apiPublishAddSource(c *gin.Context) { resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) @@ -739,7 +739,7 @@ func apiPublishSetSources(c *gin.Context) { resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) @@ -789,7 +789,7 @@ func apiPublishDropChanges(c *gin.Context) { resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) @@ -869,7 +869,7 @@ func apiPublishUpdateSource(c *gin.Context) { resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) @@ -932,7 +932,7 @@ func apiPublishRemoveSource(c *gin.Context) { resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) From e447fc0f1e3b875a9b8830c03f97e8059c6f7316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 20 Apr 2025 23:54:10 +0200 Subject: [PATCH 5/7] ci: keep CI artifacts for 7 days --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c65b90a4..d8b61e9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,6 +207,23 @@ jobs: run: | .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }} + - name: "Get aptly version" + env: + FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }} + run: | + aptlyver=$(make -s version) + echo "Aptly Version: $aptlyver" + echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT + id: releaseversion + + - name: "Upload CI Artifacts" + if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags') + uses: actions/upload-artifact@v4 + with: + name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }} + path: build/ + retention-days: 7 + ci-binary-build: name: "Build" needs: test @@ -267,6 +284,15 @@ jobs: path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip compression-level: 0 # no compression + - name: "Upload CI Artifacts" + uses: actions/upload-artifact@v4 + if: "!startsWith(github.event.ref, 'refs/tags')" + with: + name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }} + path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + compression-level: 0 # no compression + retention-days: 7 + gh-release: name: "Github Release" runs-on: ubuntu-latest From c07bf2b10886619e6e97d6c245cdd972bb27c35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 21 Apr 2025 23:19:32 +0200 Subject: [PATCH 6/7] s3: add debug logs for commands * initialize zerolog for commands * Change default log format: remote colors and timestamp --- api/router.go | 5 --- console/progress.go | 5 +-- console/terminal.go | 12 ------- context/context.go | 8 +++++ s3/public.go | 8 ++++- system/t06_publish/S3Publish1Test_gold | 18 ++++++++++ system/t06_publish/S3Publish2Test_gold | 43 ++++++++++++++++++++++ system/t06_publish/S3Publish3Test_gold | 49 ++++++++++++++++++++++++++ system/t06_publish/S3Publish5Test_gold | 6 ++++ system/t06_publish/S3Publish6Test_gold | 43 ++++++++++++++++++++++ utils/logging.go | 16 ++++++--- 11 files changed, 188 insertions(+), 25 deletions(-) delete mode 100644 console/terminal.go diff --git a/api/router.go b/api/router.go index 3536ab09..e537e6c0 100644 --- a/api/router.go +++ b/api/router.go @@ -2,7 +2,6 @@ package api import ( "net/http" - "os" "sync/atomic" "github.com/aptly-dev/aptly/aptly" @@ -56,13 +55,9 @@ func Router(c *ctx.AptlyContext) http.Handler { router.UseRawPath = true if c.Config().LogFormat == "json" { - c.StructuredLogging(true) - utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout) gin.DefaultWriter = utils.LogWriter{Logger: log.Logger} router.Use(JSONLogger()) } else { - c.StructuredLogging(false) - utils.SetupDefaultLogger(c.Config().LogLevel) router.Use(gin.Logger()) } diff --git a/console/progress.go b/console/progress.go index 16191857..d86434cc 100644 --- a/console/progress.go +++ b/console/progress.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/utils" "github.com/cheggaaa/pb" "github.com/rs/zerolog/log" "github.com/wsxiaoys/terminal/color" @@ -78,7 +79,7 @@ func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) { if p.bar != nil { panic("bar already initialized") } - if RunningOnTerminal() { + if utils.RunningOnTerminal() { p.bar = pb.New(0) p.bar.Total = count p.bar.NotPrint = true @@ -141,7 +142,7 @@ func (p *Progress) PrintfStdErr(msg string, a ...interface{}) { // ColoredPrintf does printf in colored way + newline func (p *Progress) ColoredPrintf(msg string, a ...interface{}) { - if RunningOnTerminal() { + if utils.RunningOnTerminal() { p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"} } else { // stip color marks diff --git a/console/terminal.go b/console/terminal.go deleted file mode 100644 index ba88af70..00000000 --- a/console/terminal.go +++ /dev/null @@ -1,12 +0,0 @@ -package console - -import ( - "syscall" - - "golang.org/x/term" -) - -// RunningOnTerminal checks whether stdout is terminal -func RunningOnTerminal() bool { - return term.IsTerminal(syscall.Stdout) -} diff --git a/context/context.go b/context/context.go index 5df5bbfd..545a1aa9 100644 --- a/context/context.go +++ b/context/context.go @@ -123,6 +123,14 @@ func (context *AptlyContext) config() *utils.ConfigStructure { } } + if utils.Config.LogFormat == "json" { + context.StructuredLogging(true) + utils.SetupJSONLogger(utils.Config.LogLevel, os.Stdout) + } else { + context.StructuredLogging(false) + utils.SetupDefaultLogger(utils.Config.LogLevel) + } + context.configLoaded = true } diff --git a/s3/public.go b/s3/public.go index 234a4b22..1f5340ce 100644 --- a/s3/public.go +++ b/s3/public.go @@ -172,6 +172,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err } defer source.Close() + log.Debug().Msgf("S3: PutFile '%s'", path) err = storage.putFile(path, source, "") if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) @@ -196,7 +197,6 @@ func (storage *PublishedStorage) getMD5(path string) (string, error) { // putFile uploads file-like object to func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker, sourceMD5 string) error { - params := &s3.PutObjectInput{ Bucket: aws.String(storage.bucket), Key: aws.String(filepath.Join(storage.prefix, path)), @@ -238,6 +238,7 @@ func (storage *PublishedStorage) Remove(path string) error { Key: aws.String(filepath.Join(storage.prefix, path)), } + log.Debug().Msgf("S3: Remove '%s'", path) if _, err := storage.s3.DeleteObject(context.TODO(), params); err != nil { var notFoundErr *smithy.GenericAPIError if errors.As(err, ¬FoundErr) && notFoundErr.Code == "NoSuchBucket" { @@ -270,6 +271,7 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error return err } + log.Debug().Msgf("S3: RemoveDirs '%s'", path) if storage.disableMultiDel { for i := range filelist { params := &s3.DeleteObjectInput{ @@ -385,6 +387,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } defer source.Close() + log.Debug().Msgf("S3: LinkFromPool '%s'", relPath) err = storage.putFile(relPath, source, sourceMD5) if err == nil { storage.pathCache[relPath] = sourceMD5 @@ -465,6 +468,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { params.ServerSideEncryption = storage.encryptionMethod } + log.Debug().Msgf("S3: RenameFile %s -> %s", oldName, newName) _, err := storage.s3.CopyObject(context.TODO(), params) if err != nil { return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err) @@ -494,6 +498,7 @@ func (storage *PublishedStorage) SymLink(src string, dst string) error { params.ServerSideEncryption = types.ServerSideEncryption(storage.encryptionMethod) } + log.Debug().Msgf("S3: SymLink %s -> %s", src, dst) _, err := storage.s3.CopyObject(context.TODO(), params) if err != nil { return fmt.Errorf("error symlinking %s -> %s in %s: %s", src, dst, storage, err) @@ -504,6 +509,7 @@ func (storage *PublishedStorage) SymLink(src string, dst string) error { // HardLink using symlink functionality as hard links do not exist func (storage *PublishedStorage) HardLink(src string, dst string) error { + log.Debug().Msgf("S3: HardLink %s -> %s", src, dst) return storage.SymLink(src, dst) } diff --git a/system/t06_publish/S3Publish1Test_gold b/system/t06_publish/S3Publish1Test_gold index b65701eb..33690b03 100644 --- a/system/t06_publish/S3Publish1Test_gold +++ b/system/t06_publish/S3Publish1Test_gold @@ -1,8 +1,26 @@ Loading packages... Generating metadata files and linking package files... +DBG S3: LinkFromPool 'pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' +DBG S3: LinkFromPool 'pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc' +DBG S3: LinkFromPool 'pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz' +DBG S3: LinkFromPool 'pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz' +DBG S3: LinkFromPool 'pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc' Finalizing metadata files... +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.bz2' +DBG S3: PutFile 'dists/maverick/main/source/Sources' +DBG S3: PutFile 'dists/maverick/main/source/Sources.gz' +DBG S3: PutFile 'dists/maverick/main/source/Sources.bz2' +DBG S3: PutFile 'dists/maverick/main/Contents-i386.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Release' +DBG S3: PutFile 'dists/maverick/main/source/Release' +DBG S3: PutFile 'dists/maverick/Contents-i386.gz' +DBG S3: PutFile 'dists/maverick/Release' Signing file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/Release.gpg' Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/InRelease' Local repo local-repo has been successfully published. Now you can add following line to apt sources: diff --git a/system/t06_publish/S3Publish2Test_gold b/system/t06_publish/S3Publish2Test_gold index cd68115c..e4fcd5cc 100644 --- a/system/t06_publish/S3Publish2Test_gold +++ b/system/t06_publish/S3Publish2Test_gold @@ -1,9 +1,52 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp.bz2' +DBG S3: PutFile 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: PutFile 'dists/maverick/main/source/Release.tmp' +DBG S3: PutFile 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/Release.tmp' Signing file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/Release.tmp.gpg' Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp -> dists/maverick/main/binary-i386/Packages +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.gz -> dists/maverick/main/binary-i386/Packages.gz +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.bz2 -> dists/maverick/main/binary-i386/Packages.bz2 +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: RenameFile dists/maverick/main/Contents-i386.tmp.gz -> dists/maverick/main/Contents-i386.gz +DBG S3: Remove 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-i386/Release.tmp -> dists/maverick/main/binary-i386/Release +DBG S3: Remove 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: RenameFile dists/maverick/InRelease.tmp -> dists/maverick/InRelease +DBG S3: Remove 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp -> dists/maverick/main/source/Sources +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp.gz -> dists/maverick/main/source/Sources.gz +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp.gz' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp.bz2 -> dists/maverick/main/source/Sources.bz2 +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp.bz2' +DBG S3: RenameFile dists/maverick/main/source/Release.tmp -> dists/maverick/main/source/Release +DBG S3: Remove 'dists/maverick/main/source/Release.tmp' +DBG S3: RenameFile dists/maverick/Contents-i386.tmp.gz -> dists/maverick/Contents-i386.gz +DBG S3: Remove 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/Release.tmp -> dists/maverick/Release +DBG S3: Remove 'dists/maverick/Release.tmp' +DBG S3: RenameFile dists/maverick/Release.tmp.gpg -> dists/maverick/Release.gpg +DBG S3: Remove 'dists/maverick/Release.tmp.gpg' Cleaning up published repository s3:test1:./maverick... Cleaning up component 'main'... +DBG S3: Remove 'pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz' Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/S3Publish3Test_gold b/system/t06_publish/S3Publish3Test_gold index 348fbe81..b8e67b68 100644 --- a/system/t06_publish/S3Publish3Test_gold +++ b/system/t06_publish/S3Publish3Test_gold @@ -1,9 +1,58 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... +DBG S3: PutFile 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: PutFile 'dists/maverick/Contents-amd64.tmp.gz' +DBG S3: PutFile 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-amd64/Packages.tmp' +DBG S3: PutFile 'dists/maverick/main/binary-amd64/Packages.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-amd64/Packages.tmp.bz2' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: PutFile 'dists/maverick/main/Contents-amd64.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-amd64/Release.tmp' +DBG S3: PutFile 'dists/maverick/Release.tmp' Signing file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/Release.tmp.gpg' Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/main/Contents-amd64.tmp.gz -> dists/maverick/main/Contents-amd64.gz +DBG S3: Remove 'dists/maverick/main/Contents-amd64.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-i386/Release.tmp -> dists/maverick/main/binary-i386/Release +DBG S3: Remove 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: RenameFile dists/maverick/main/Contents-i386.tmp.gz -> dists/maverick/main/Contents-i386.gz +DBG S3: Remove 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-amd64/Release.tmp -> dists/maverick/main/binary-amd64/Release +DBG S3: Remove 'dists/maverick/main/binary-amd64/Release.tmp' +DBG S3: RenameFile dists/maverick/Release.tmp -> dists/maverick/Release +DBG S3: Remove 'dists/maverick/Release.tmp' +DBG S3: RenameFile dists/maverick/InRelease.tmp -> dists/maverick/InRelease +DBG S3: Remove 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/main/binary-amd64/Packages.tmp -> dists/maverick/main/binary-amd64/Packages +DBG S3: Remove 'dists/maverick/main/binary-amd64/Packages.tmp' +DBG S3: RenameFile dists/maverick/main/binary-amd64/Packages.tmp.bz2 -> dists/maverick/main/binary-amd64/Packages.bz2 +DBG S3: Remove 'dists/maverick/main/binary-amd64/Packages.tmp.bz2' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.bz2 -> dists/maverick/main/binary-i386/Packages.bz2 +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp -> dists/maverick/main/binary-i386/Packages +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.gz -> dists/maverick/main/binary-i386/Packages.gz +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: RenameFile dists/maverick/Release.tmp.gpg -> dists/maverick/Release.gpg +DBG S3: Remove 'dists/maverick/Release.tmp.gpg' +DBG S3: RenameFile dists/maverick/Contents-amd64.tmp.gz -> dists/maverick/Contents-amd64.gz +DBG S3: Remove 'dists/maverick/Contents-amd64.tmp.gz' +DBG S3: RenameFile dists/maverick/Contents-i386.tmp.gz -> dists/maverick/Contents-i386.gz +DBG S3: Remove 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-amd64/Packages.tmp.gz -> dists/maverick/main/binary-amd64/Packages.gz +DBG S3: Remove 'dists/maverick/main/binary-amd64/Packages.tmp.gz' Cleaning up published repository s3:test1:./maverick... Cleaning up component 'main'... +DBG S3: Remove 'pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb' +DBG S3: Remove 'pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_amd64.deb' +DBG S3: Remove 'pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_i386.deb' +DBG S3: Remove 'pool/main/g/gnuplot/gnuplot_4.6.1-1~maverick2_all.deb' Published snapshot repository s3:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/S3Publish5Test_gold b/system/t06_publish/S3Publish5Test_gold index 0b8f635f..a1b531e5 100644 --- a/system/t06_publish/S3Publish5Test_gold +++ b/system/t06_publish/S3Publish5Test_gold @@ -1,4 +1,10 @@ +DBG S3: RemoveDirs 'dists/sq2' Cleaning up published repository s3:test1:./sq2... Cleaning up component 'main'... +DBG S3: Remove 'pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb' +DBG S3: Remove 'pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz' Published repository has been removed successfully. diff --git a/system/t06_publish/S3Publish6Test_gold b/system/t06_publish/S3Publish6Test_gold index cd68115c..cfa321d4 100644 --- a/system/t06_publish/S3Publish6Test_gold +++ b/system/t06_publish/S3Publish6Test_gold @@ -1,9 +1,52 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... +DBG S3: PutFile 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: PutFile 'dists/maverick/main/source/Release.tmp' +DBG S3: PutFile 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp.gz' +DBG S3: PutFile 'dists/maverick/main/source/Sources.tmp.bz2' +DBG S3: PutFile 'dists/maverick/Release.tmp' Signing file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/Release.tmp.gpg' Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +DBG S3: PutFile 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/InRelease.tmp -> dists/maverick/InRelease +DBG S3: Remove 'dists/maverick/InRelease.tmp' +DBG S3: RenameFile dists/maverick/main/source/Release.tmp -> dists/maverick/main/source/Release +DBG S3: Remove 'dists/maverick/main/source/Release.tmp' +DBG S3: RenameFile dists/maverick/Contents-i386.tmp.gz -> dists/maverick/Contents-i386.gz +DBG S3: Remove 'dists/maverick/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp -> dists/maverick/main/binary-i386/Packages +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.bz2 -> dists/maverick/main/binary-i386/Packages.bz2 +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.bz2' +DBG S3: RenameFile dists/maverick/Release.tmp -> dists/maverick/Release +DBG S3: Remove 'dists/maverick/Release.tmp' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp.bz2 -> dists/maverick/main/source/Sources.bz2 +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp.bz2' +DBG S3: RenameFile dists/maverick/Release.tmp.gpg -> dists/maverick/Release.gpg +DBG S3: Remove 'dists/maverick/Release.tmp.gpg' +DBG S3: RenameFile dists/maverick/main/Contents-i386.tmp.gz -> dists/maverick/main/Contents-i386.gz +DBG S3: Remove 'dists/maverick/main/Contents-i386.tmp.gz' +DBG S3: RenameFile dists/maverick/main/binary-i386/Release.tmp -> dists/maverick/main/binary-i386/Release +DBG S3: Remove 'dists/maverick/main/binary-i386/Release.tmp' +DBG S3: RenameFile dists/maverick/main/binary-i386/Packages.tmp.gz -> dists/maverick/main/binary-i386/Packages.gz +DBG S3: Remove 'dists/maverick/main/binary-i386/Packages.tmp.gz' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp -> dists/maverick/main/source/Sources +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp' +DBG S3: RenameFile dists/maverick/main/source/Sources.tmp.gz -> dists/maverick/main/source/Sources.gz +DBG S3: Remove 'dists/maverick/main/source/Sources.tmp.gz' Cleaning up published repository s3:test1:./maverick... Cleaning up component 'main'... +DBG S3: Remove 'pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc' +DBG S3: Remove 'pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz' Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/utils/logging.go b/utils/logging.go index 592e8f30..6eceae14 100644 --- a/utils/logging.go +++ b/utils/logging.go @@ -4,12 +4,19 @@ import ( "io" "os" "strings" + "syscall" "time" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "golang.org/x/term" ) +// RunningOnTerminal checks whether stdout is terminal +func RunningOnTerminal() bool { + return term.IsTerminal(syscall.Stdout) +} + type LogWriter struct { Logger zerolog.Logger } @@ -32,11 +39,10 @@ func SetupDefaultLogger(levelStr string) { zerolog.MessageFieldName = "message" zerolog.LevelFieldName = "level" - log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}). - Level(GetLogLevelOrDebug(levelStr)). - With(). - Timestamp(). - Logger() + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, + FormatTimestamp: func(_ interface{}) string { return "" }, + NoColor: !RunningOnTerminal(), + }).Level(GetLogLevelOrDebug(levelStr)) } func GetLogLevelOrDebug(levelStr string) zerolog.Level { From 22bc2f9d0f14da2d43fcb3d5eaa46ef1c3d8e71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 21 Apr 2025 14:13:37 +0200 Subject: [PATCH 7/7] system-tests: improve sorted compare sort both aptly output and gold file. output original output for debugging on failure. * Makefile: enable CAPTURE=1 env variable for capturing gold files * docker-system-test: use AWS env vars for S3 tests * fix system tests timing issue with order of gpg logs in publish tests --- Makefile | 12 ++++--- system/lib.py | 29 ++++++++------- system/t06_publish/PublishSnapshot38Test_gold | 1 + system/t06_publish/s3.py | 6 ++++ system/t06_publish/snapshot.py | 35 +++++++++++++++++++ 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 4682f774..fa5d6255 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,10 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) -# Uncomment to update system test gold files -# CAPTURE := "--capture" +# export CAPUTRE=1 for regenrating test gold files +ifeq ($(CAPTURE),1) +CAPTURE_ARG := --capture +endif help: ## Print this help @grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -50,7 +52,7 @@ swagger-install: echo "// @version $(VERSION)" >> docs/swagger.conf azurite-start: - azurite -l /tmp/aptly-azurite & \ + azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \ echo $$! > ~/.azurite.pid azurite-stop: @@ -102,7 +104,7 @@ system-test: prepare swagger etcd-install ## Run system tests if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) # Run system tests - PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) + PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) bench: @echo "\e[33m\e[1mRunning benchmark ...\e[0m" @@ -186,6 +188,8 @@ docker-system-test: ## Run system tests in docker container (add TEST=t04_mirro AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ + AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ system-test TEST=$(TEST) \ azurite-stop diff --git a/system/lib.py b/system/lib.py index 1287f81d..3f95b102 100644 --- a/system/lib.py +++ b/system/lib.py @@ -289,8 +289,6 @@ class BaseTest(object): def run(self): output = self.run_cmd(self.runCmd, self.expectedCode) - if self.sortOutput: - output = self.sort_lines(output) self.output = self.output_processor(output) def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None): @@ -493,24 +491,29 @@ class BaseTest(object): return a.decode('utf-8') return a - def verify_match(self, a, b, match_prepare=None, ensure_utf8=True): + def verify_match(self, gold, orig, match_prepare=None, ensure_utf8=True): + output = orig if ensure_utf8: - a = self.ensure_utf8(a) - b = self.ensure_utf8(b) + gold = self.ensure_utf8(gold) + output = self.ensure_utf8(output) if match_prepare is not None: - a = match_prepare(a) - b = match_prepare(b) + gold = match_prepare(gold) + output = match_prepare(output) + + # sort if requested + if self.sortOutput: + gold = self.sort_lines(gold) + output = self.sort_lines(output) # strip trailing whitespace and newlines - a = a.strip() - b = b.strip() + gold = gold.strip() + output = output.strip() - if a != b: + if gold != output: diff = "".join(difflib.unified_diff( - [l + "\n" for l in a.split("\n")], [l + "\n" for l in b.split("\n")])) - - raise Exception("content doesn't match:\n" + diff + "\n") + [l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")])) + raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n") check = check_output diff --git a/system/t06_publish/PublishSnapshot38Test_gold b/system/t06_publish/PublishSnapshot38Test_gold index 8b9bb3cb..64d9fe57 100644 --- a/system/t06_publish/PublishSnapshot38Test_gold +++ b/system/t06_publish/PublishSnapshot38Test_gold @@ -12,3 +12,4 @@ Now you can add following line to apt sources: Don't forget to add your GPG key to apt with apt-key. You can also use `aptly serve` to publish your repositories over HTTP quickly. + diff --git a/system/t06_publish/s3.py b/system/t06_publish/s3.py index c818023a..f57adb06 100644 --- a/system/t06_publish/s3.py +++ b/system/t06_publish/s3.py @@ -14,6 +14,7 @@ class S3Publish1Test(S3Test): "aptly repo add local-repo ${files}", "aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386", ] + sortOutput = True runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:" def check(self): @@ -53,6 +54,7 @@ class S3Publish2Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:", "aptly repo remove local-repo pyspi" ] + sortOutput = True runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:" def check(self): @@ -93,6 +95,7 @@ class S3Publish3Test(S3Test): "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1 s3:test1:", ] + sortOutput = True runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1: snap3" def check(self): @@ -129,6 +132,7 @@ class S3Publish4Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=xyz local-repo s3:test1:", "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:prefix", ] + sortOutput = True runCmd = "aptly publish list" @@ -144,6 +148,7 @@ class S3Publish5Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 s3:test1:", "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 s3:test1:", ] + sortOutput = True runCmd = "aptly publish drop sq2 s3:test1:" def check(self): @@ -172,6 +177,7 @@ class S3Publish6Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:", "aptly repo remove local-repo pyspi" ] + sortOutput = True runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:" def check(self): diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 2ba245b4..fb877bb7 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -22,6 +22,7 @@ class PublishSnapshot1Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1" gold_processor = BaseTest.expand_environ @@ -131,6 +132,7 @@ class PublishSnapshot2Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap2 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" gold_processor = BaseTest.expand_environ @@ -168,6 +170,7 @@ class PublishSnapshot3Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap3 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -component=contrib snap3" gold_processor = BaseTest.expand_environ @@ -208,6 +211,7 @@ class PublishSnapshot4Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap4 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly -architectures=i386 publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap4" gold_processor = BaseTest.expand_environ @@ -247,6 +251,7 @@ class PublishSnapshot5Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap5 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira" gold_processor = BaseTest.expand_environ @@ -294,6 +299,7 @@ class PublishSnapshot6Test(BaseTest): "aptly snapshot create snap2 from mirror wheezy-main", "aptly snapshot merge snap6 snap2 snap" ] + sortOutput = True runCmd = "aptly publish snapshot snap6" expectedCode = 1 @@ -308,6 +314,7 @@ class PublishSnapshot7Test(BaseTest): "aptly snapshot create snap7 from mirror gnuplot-maverick", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap7", ] + sortOutput = True runCmd = "aptly publish snapshot snap7" expectedCode = 1 @@ -322,6 +329,7 @@ class PublishSnapshot8Test(BaseTest): "aptly snapshot create snap8 from mirror gnuplot-maverick", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap8 ./ppa", ] + sortOutput = True runCmd = "aptly publish snapshot snap8 ppa" expectedCode = 1 @@ -334,6 +342,7 @@ class PublishSnapshot9Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap9 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap9 ppa/dists/la" expectedCode = 1 @@ -346,6 +355,7 @@ class PublishSnapshot10Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap10 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap10 ppa/pool/la" expectedCode = 1 @@ -358,6 +368,7 @@ class PublishSnapshot11Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap11 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap11 ../la" expectedCode = 1 @@ -367,6 +378,7 @@ class PublishSnapshot12Test(BaseTest): publish snapshot: no snapshot """ fixtureDB = True + sortOutput = True runCmd = "aptly publish snapshot snap12" expectedCode = 1 @@ -465,6 +477,7 @@ class PublishSnapshot16Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap16 from mirror gnuplot-maverick-src", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap16" gold_processor = BaseTest.expand_environ @@ -524,6 +537,7 @@ class PublishSnapshot17Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap17 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap17" gold_processor = BaseTest.expand_environ @@ -578,6 +592,7 @@ class PublishSnapshot18Test(BaseTest): "aptly repo add repo1 ${files}", "aptly snapshot create snap1 from repo repo1", ] + sortOutput = True runCmd = "aptly publish snapshot snap1" expectedCode = 1 @@ -682,6 +697,7 @@ class PublishSnapshot24Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap24 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -origin=aptly24 -notautomatic=yes -butautomaticupgrades=yes snap24" gold_processor = BaseTest.expand_environ @@ -701,6 +717,7 @@ class PublishSnapshot25Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap25 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -architectures=amd64 --distribution=maverick -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap25" gold_processor = BaseTest.expand_environ @@ -736,6 +753,7 @@ class PublishSnapshot26Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap26.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=main,contrib snap26.1 snap26.2" gold_processor = BaseTest.expand_environ @@ -864,6 +882,7 @@ class PublishSnapshot27Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap27.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=, snap27.1 snap27.2" gold_processor = BaseTest.expand_environ @@ -879,6 +898,7 @@ class PublishSnapshot28Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap28.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -component=, snap28.1 snap28.2" expectedCode = 1 @@ -891,6 +911,7 @@ class PublishSnapshot29Test(BaseTest): "aptly snapshot create snap29.1 empty", "aptly snapshot create snap29.2 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=b,b snap29.1 snap29.2" expectedCode = 1 @@ -906,6 +927,7 @@ class PublishSnapshot30Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap30.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap30.1 snap30.2" expectedCode = 1 @@ -917,6 +939,7 @@ class PublishSnapshot31Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap31.1 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap31.1 snap31.2" expectedCode = 1 @@ -928,6 +951,7 @@ class PublishSnapshot32Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap32.1 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap32.1" expectedCode = 2 @@ -948,6 +972,7 @@ class PublishSnapshot33Test(BaseTest): "aptly snapshot create snap2 from repo local-repo2", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" expectedCode = 1 gold_processor = BaseTest.expand_environ @@ -966,6 +991,7 @@ class PublishSnapshot34Test(BaseTest): "aptly snapshot create snap2 from repo local-repo2", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1", ] + sortOutput = True runCmd = "aptly publish snapshot -force-overwrite -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" gold_processor = BaseTest.expand_environ @@ -987,6 +1013,7 @@ class PublishSnapshot35Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ @@ -1118,6 +1145,7 @@ class PublishSnapshot36Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap36 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-contents snap36" gold_processor = BaseTest.expand_environ @@ -1145,6 +1173,7 @@ class PublishSnapshot37Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ @@ -1160,8 +1189,10 @@ class PublishSnapshot38Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ + sortOutput = True def check(self): super(PublishSnapshot38Test, self).check() @@ -1187,6 +1218,7 @@ class PublishSnapshot39Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -suite=stable snap1" gold_processor = BaseTest.expand_environ @@ -1222,6 +1254,7 @@ class PublishSnapshot40Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap40 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-bz2 snap40" gold_processor = BaseTest.expand_environ @@ -1254,6 +1287,7 @@ class PublishSnapshot41Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg ps41", "aptly snapshot create snap41 from mirror ps41", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap41" gold_processor = BaseTest.expand_environ @@ -1375,6 +1409,7 @@ class PublishSnapshot42Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -multi-dist snap1" gold_processor = BaseTest.expand_environ