From 36121e56430551ca37dbaa109ca8f9a6ad2c48ae Mon Sep 17 00:00:00 2001 From: Leigh London Date: Tue, 19 Nov 2024 14:59:26 +1100 Subject: [PATCH 01/64] Bulk patch aws/aws-sdk-go-v2 dependencies --- go.mod | 36 ++++++++++++++--------------- go.sum | 72 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index b1e030c0..91f7e565 100644 --- a/go.mod +++ b/go.mod @@ -46,19 +46,19 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.8.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -120,11 +120,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 github.com/ProtonMail/go-crypto v1.0.0 - github.com/aws/aws-sdk-go-v2 v1.30.3 - github.com/aws/aws-sdk-go-v2/config v1.25.1 - github.com/aws/aws-sdk-go-v2/credentials v1.16.1 - github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 - github.com/aws/smithy-go v1.20.3 + github.com/aws/aws-sdk-go-v2 v1.32.5 + github.com/aws/aws-sdk-go-v2/config v1.28.5 + github.com/aws/aws-sdk-go-v2/credentials v1.17.46 + github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1 + github.com/aws/smithy-go v1.22.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 diff --git a/go.sum b/go.sum index 43c4b545..4425614a 100644 --- a/go.sum +++ b/go.sum @@ -24,42 +24,42 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/awalterschulze/gographviz v2.0.1+incompatible h1:XIECBRq9VPEQqkQL5pw2OtjCAdrtIgFKoJU8eT98AS8= github.com/awalterschulze/gographviz v2.0.1+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= -github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= -github.com/aws/aws-sdk-go-v2/config v1.25.1 h1:YsjngBOl2mx4l3egkVWndr6/6TqtkdsWJFZIsQ924Ek= -github.com/aws/aws-sdk-go-v2/config v1.25.1/go.mod h1:yV6h7TRVzhdIFmUk9WWDRpWwYGg1woEzKr0k1IYz2Tk= -github.com/aws/aws-sdk-go-v2/credentials v1.16.1 h1:WessyrdgyFN5TB+eLQdrFSlN/3oMnqukIFhDxK6z8h0= -github.com/aws/aws-sdk-go-v2/credentials v1.16.1/go.mod h1:RQJyPxKcr+m4ArlIG1LUhMOrjposVfzbX6H8oR6oCgE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 h1:9wKDWEjwSnXZre0/O3+ZwbBl1SmlgWYBbrTV10X/H1s= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4/go.mod h1:t4i+yGHMCcUNIX1x7YVYa6bH/Do7civ5I6cG/6PMfyA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 h1:V47N5eKgVZoRSvx2+RQ0EpAEit/pqOhqeSQFiS4OFEQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.2/go.mod h1:/pE21vno3q1h4bbhUOEi+6Zu/aT26UK2WKkDXd+TssQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 h1:sMAcO7VHVw28HTAdZpTULDzFirHOsVm/x25CxhUH0jA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 h1:vwyiRTnXLqsak/6WAQ+uTRhVqKI6vxUQ0HJXjKij0zM= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.2/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= +github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24 h1:JX70yGKLj25+lMC5Yyh8wBtvB01GDilyRuJvXJ4piD0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24/go.mod h1:+Ln60j9SUTD0LEwnhEB0Xhg61DHqplBrbZpLgyjoEHg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5 h1:gvZOjQKPxFXy1ft3QnEyXmT+IqneM9QAUWlM3r0mfqw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5/go.mod h1:DLWnfvIcm9IET/mmjdxeXbBKmTCm0ZB8p1za9BVteM8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5 h1:P1doBzv5VEg1ONxnJss1Kh5ZG/ewoIE4MQtKKc6Crgg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5/go.mod h1:NOP+euMW7W3Ukt28tAxPuoWao4rhhqJD3QEBk7oCg7w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1 h1:LXLnDfjT/P6SPIaCE86xCOjJROPn4FNB2EdN68vMK5c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1/go.mod h1:ralv4XawHjEMaHOWnTFushl0WRqim/gQWesAMF6hTow= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= From 247f5e7c607455727077934ec0442e4b4a4b912c Mon Sep 17 00:00:00 2001 From: Leigh London Date: Tue, 19 Nov 2024 15:06:58 +1100 Subject: [PATCH 02/64] update AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 1fd68974..0da15b8b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -66,3 +66,4 @@ List of contributors, in chronological order: * Andrey Loukhnov (https://github.com/aol-nnov) * Christoph Fiehe (https://github.com/cfiehe) * Blake Kostner (https://github.com/btkostner) +* Leigh London (https://github.com/leighlondon) From b2d05828a5847cae9cdf575b079825f3a2bf7e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 30 Oct 2024 17:36:05 +0100 Subject: [PATCH 03/64] set systemd service file limit to 32768 --- debian/aptly-api.default | 8 +++++++- debian/aptly-api.service | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/debian/aptly-api.default b/debian/aptly-api.default index a407ec52..5950dde4 100644 --- a/debian/aptly-api.default +++ b/debian/aptly-api.default @@ -1,2 +1,8 @@ -# Default settings for aptly-api +# Default settings for aptly-api systemd service + +# bind to host:port LISTEN_ADDRESS='localhost:8080' + +# aptly options: +# -no-lock allow aptly commands in parallel with api service (no global database lock, but lock per request) +APTLY_OPTIONS="-no-lock" diff --git a/debian/aptly-api.service b/debian/aptly-api.service index 5e447ace..16d6383c 100644 --- a/debian/aptly-api.service +++ b/debian/aptly-api.service @@ -9,7 +9,8 @@ Group=aptly-api Environment=TERM=dumb WorkingDirectory=~ EnvironmentFile=/etc/default/aptly-api -ExecStart=/usr/bin/aptly api serve -config=/etc/aptly.conf -no-lock -listen=${LISTEN_ADDRESS} +ExecStart=/usr/bin/aptly api serve -config=/etc/aptly.conf ${APTLY_OPTIONS} -listen=${LISTEN_ADDRESS} +LimitNOFILE=32768 [Install] WantedBy=multi-user.target From 2f540a8026e2bb0901f8aa266ce9e21448c88c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 20 Nov 2024 13:49:47 +0100 Subject: [PATCH 04/64] debian: do not conflict with gnupg1 --- debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/control b/debian/control index cf1723e0..f7ed7f09 100644 --- a/debian/control +++ b/debian/control @@ -89,7 +89,6 @@ Package: aptly Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, bzip2, xz-utils, gpgv, gpg Suggests: graphviz -Conflicts: gnupg1, gpgv1 Built-Using: ${misc:Static-Built-Using}, ${misc:Built-Using} Description: Swiss army knife for Debian repository management - main package It offers several features making it easy to manage Debian package From 7d9f020ae86e56d9fb36ed6e88d776f9e2ab06dc Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Mon, 2 Dec 2024 15:09:31 +0100 Subject: [PATCH 05/64] Fix null pointer when dropping a multi dist published repo. Signed-off-by: Christoph Fiehe --- deb/publish.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deb/publish.go b/deb/publish.go index df8ad10f..3bf397af 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1660,6 +1660,11 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly return err } + err = collection.LoadComplete(repo, collectionFactory) + if err != nil { + return err + } + removePrefix := true removePoolComponents := repo.Components() cleanComponents := []string{} From 320307f5046a7942a612dcbc434887d8db361bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 14:23:36 +0100 Subject: [PATCH 06/64] graph: do not remove tempfile when opening in viewer --- cmd/graph.go | 18 ++--------- system/t14_graph/CreateGraphOutputTest_gold | 2 ++ system/t14_graph/CreateGraphTest_gold | 2 ++ system/t14_graph/graph.py | 33 +++++++++++++++++++++ 4 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 system/t14_graph/CreateGraphOutputTest_gold create mode 100644 system/t14_graph/CreateGraphTest_gold create mode 100644 system/t14_graph/graph.py diff --git a/cmd/graph.go b/cmd/graph.go index 4425d087..64b12d9e 100644 --- a/cmd/graph.go +++ b/cmd/graph.go @@ -9,7 +9,6 @@ import ( "path/filepath" "runtime" "strings" - "time" "github.com/aptly-dev/aptly/deb" "github.com/aptly-dev/aptly/utils" @@ -79,10 +78,6 @@ func aptlyGraph(cmd *commander.Command, args []string) error { return err } - defer func() { - _ = os.Remove(tempfilename) - }() - if output != "" { err = utils.CopyFile(tempfilename, output) if err != nil { @@ -90,23 +85,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error { } fmt.Printf("Output saved to %s\n", output) + _ = os.Remove(tempfilename) } else { command := getOpenCommand() - fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename) + fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename) args := strings.Split(command, " ") viewer := exec.Command(args[0], append(args[1:], tempfilename)...) viewer.Stderr = os.Stderr - if err = viewer.Start(); err == nil { - // Wait for a second so that the visualizer has a chance to - // open the input file. This needs to be done even if we're - // waiting for the visualizer as it can be just a wrapper that - // spawns a browser tab and returns right away. - defer func(t <-chan time.Time) { - <-t - }(time.After(time.Second)) - } + err = viewer.Start() } return err diff --git a/system/t14_graph/CreateGraphOutputTest_gold b/system/t14_graph/CreateGraphOutputTest_gold new file mode 100644 index 00000000..e9ca9ebe --- /dev/null +++ b/system/t14_graph/CreateGraphOutputTest_gold @@ -0,0 +1,2 @@ +Generating graph... +Output saved to /tmp/aptly-graph.png diff --git a/system/t14_graph/CreateGraphTest_gold b/system/t14_graph/CreateGraphTest_gold new file mode 100644 index 00000000..12928d29 --- /dev/null +++ b/system/t14_graph/CreateGraphTest_gold @@ -0,0 +1,2 @@ +Generating graph... +Displaying png file: xdg-open /tmp/aptly-graph1173098610.png diff --git a/system/t14_graph/graph.py b/system/t14_graph/graph.py new file mode 100644 index 00000000..ba864275 --- /dev/null +++ b/system/t14_graph/graph.py @@ -0,0 +1,33 @@ +""" +Test aptly graph +""" + +import os +import re + +from lib import BaseTest + + +class CreateGraphTest(BaseTest): + """ + open graph in viewer + """ + fixtureCmds = ["mkdir -p ../build", "ln -fs /bin/true ../build/xdg-open"] + environmentOverride = {"PATH": os.environ["PATH"] + ":../build"} + runCmd = "aptly graph" + + def outputMatchPrepare(self, s): + return re.sub(r"[0-9]", "", s) + + def teardown(self): + self.run_cmd(["rm", "-f", "../build/xdg-open"]) + + +class CreateGraphOutputTest(BaseTest): + """ + open graph in viewer + """ + runCmd = "aptly graph -output /tmp/aptly-graph.png" + + def teardown(self): + self.run_cmd(["rm", "-f", "/tmp/aptly-graph.png"]) From 3b785e41657335caae66faaec5f362cec168ec62 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:19:23 +0900 Subject: [PATCH 07/64] Refactor Filter options into a struct It was already a lot of options for one method and I am going to add another one in the next commit. --- api/api.go | 14 ++++-- api/mirror.go | 8 +++- api/repos.go | 9 +++- api/snapshot.go | 9 +++- cmd/repo_move.go | 9 +++- cmd/repo_remove.go | 2 +- cmd/snapshot_filter.go | 9 +++- cmd/snapshot_pull.go | 9 +++- cmd/snapshot_search.go | 9 +++- deb/list.go | 49 +++++++++++--------- deb/list_test.go | 102 +++++++++++++++++++++++++++-------------- deb/remote.go | 9 +++- 12 files changed, 167 insertions(+), 71 deletions(-) diff --git a/api/api.go b/api/api.go index a52406d8..6c9ed7e6 100644 --- a/api/api.go +++ b/api/api.go @@ -227,8 +227,13 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory list.PrepareIndex() - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) + list, err = list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + Source: nil, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + }) if err != nil { AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err)) return @@ -244,8 +249,9 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory fmt.Println("filter packages by version, query string parse err: ", err) c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err)) } else { - tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false, - nil, 0, []string{}) + tmpList, err := list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{versionQ}, + }) if err == nil { if tmpList.Len() > 0 { diff --git a/api/mirror.go b/api/mirror.go index 0052f2a5..803551dd 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -306,8 +306,12 @@ func apiMirrorsPackages(c *gin.Context) { list.PrepareIndex() - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) + list, err = list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + }) if err != nil { AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err)) } diff --git a/api/repos.go b/api/repos.go index 005c2aaa..3e3ddd3b 100644 --- a/api/repos.go +++ b/api/repos.go @@ -586,7 +586,14 @@ func apiReposCopyPackage(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err) } - toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress()) + toProcess, err := srcList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: jsonBody.WithDeps, + Source: dstList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err) } diff --git a/api/snapshot.go b/api/snapshot.go index 1a9c2278..df51adcd 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -636,7 +636,14 @@ func apiSnapshotsPull(c *gin.Context) { } // Filter with dependencies as requested - destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress()) + destinationPackageList, err := sourcePackageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: !noDeps, + Source: toPackageList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } diff --git a/cmd/repo_move.go b/cmd/repo_move.go index 8be6698b..e53c3a7c 100644 --- a/cmd/repo_move.go +++ b/cmd/repo_move.go @@ -116,7 +116,14 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error { } } - toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress()) + toProcess, err := srcList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: withDeps, + Source: dstList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to %s: %s", command, err) } diff --git a/cmd/repo_remove.go b/cmd/repo_remove.go index 93e8535c..287a42d5 100644 --- a/cmd/repo_remove.go +++ b/cmd/repo_remove.go @@ -45,7 +45,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error { } list.PrepareIndex() - toRemove, err := list.Filter(queries, false, nil, 0, nil) + toRemove, err := list.Filter(deb.FilterOptions{Queries: queries}) if err != nil { return fmt.Errorf("unable to remove: %s", err) } diff --git a/cmd/snapshot_filter.go b/cmd/snapshot_filter.go index b81a9cfc..c712ae67 100644 --- a/cmd/snapshot_filter.go +++ b/cmd/snapshot_filter.go @@ -67,7 +67,14 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error { } // Filter with dependencies as requested - result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := packageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: withDeps, + Source: nil, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to filter: %s", err) } diff --git a/cmd/snapshot_pull.go b/cmd/snapshot_pull.go index 9c2559f5..27593b77 100644 --- a/cmd/snapshot_pull.go +++ b/cmd/snapshot_pull.go @@ -97,7 +97,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error { } // Filter with dependencies as requested - result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := sourcePackageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: !noDeps, + Source: packageList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to pull: %s", err) } diff --git a/cmd/snapshot_search.go b/cmd/snapshot_search.go index d771af7c..7af78e1e 100644 --- a/cmd/snapshot_search.go +++ b/cmd/snapshot_search.go @@ -103,8 +103,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error } } - result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to search: %s", err) } diff --git a/deb/list.go b/deb/list.go index 4f863ccf..9f3f67a9 100644 --- a/deb/list.go +++ b/deb/list.go @@ -503,32 +503,37 @@ func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided boo return } -// Filter filters package index by specified queries (ORed together), possibly pulling dependencies -func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) { - return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil) +// FilterOptions specifies options for Filter() +type FilterOptions struct { + Queries []PackageQuery + WithDependencies bool + Source *PackageList + DependencyOptions int + Architectures []string + Progress aptly.Progress // set to non-nil to report progress } -// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress -func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) { +// Filter filters package index by specified queries (ORed together), possibly pulling dependencies +func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { if !l.indexed { panic("list not indexed, can't filter") } result := NewPackageList() - for _, query := range queries { - result.Append(query.Query(l)) + for _, query := range options.Queries { + _ = result.Append(query.Query(l)) } - if withDependencies { + if options.WithDependencies { added := result.Len() result.PrepareIndex() dependencySource := NewPackageList() - if source != nil { - dependencySource.Append(source) + if options.Source != nil { + _ = dependencySource.Append(options.Source) } - dependencySource.Append(result) + _ = dependencySource.Append(result) dependencySource.PrepareIndex() // while some new dependencies were discovered @@ -536,22 +541,22 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie added = 0 // find missing dependencies - missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress) + missing, err := result.VerifyDependencies(options.DependencyOptions, options.Architectures, dependencySource, options.Progress) if err != nil { return nil, err } // try to satisfy dependencies for _, dep := range missing { - if dependencyOptions&DepFollowAllVariants == 0 { + if options.DependencyOptions&DepFollowAllVariants == 0 { // dependency might have already been satisfied // with packages already been added // // when follow-all-variants is enabled, we need to try to expand anyway, // as even if dependency is satisfied now, there might be other ways to satisfy dependency if result.Search(dep, false, true) != nil { - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) } continue } @@ -564,19 +569,19 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie continue } - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{g}Injecting package@|: %s", p) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{g}Injecting package@|: %s", p) } - result.Add(p) - dependencySource.Add(p) + _ = result.Add(p) + _ = dependencySource.Add(p) added++ - if dependencyOptions&DepFollowAllVariants == 0 { + if options.DependencyOptions&DepFollowAllVariants == 0 { break } } } else { - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String()) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String()) } } diff --git a/deb/list_test.go b/deb/list_test.go index 4209f125..e13cd4f2 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -131,7 +131,7 @@ func (s *PackageListSuite) TestAddLen(c *C) { c.Check(s.list.Len(), Equals, 1) c.Check(s.list.Add(s.p3), IsNil) c.Check(s.list.Len(), Equals, 2) - c.Check(s.list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") + c.Check(s.list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") } func (s *PackageListSuite) TestRemove(c *C) { @@ -311,7 +311,11 @@ func (s *PackageListSuite) TestSearch(c *C) { } func (s *PackageListSuite) TestFilter(c *C) { - c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter") + c.Check(func() { + s.list.Filter(FilterOptions{ + Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, + }) + }, Panics, "list not indexed, can't filter") plString := func(l *PackageList) string { list := make([]string, 0, l.Len()) @@ -324,81 +328,111 @@ func (s *PackageListSuite) TestFilter(c *C) { return strings.Join(list, " ") } - result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil) + result, err := s.il.Filter(FilterOptions{Queries: []PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"}, - &PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{Queries: []PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"}, + &PkgQuery{"dpkg", "1.8", "amd64"}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{ + result, err = s.il.Filter(FilterOptions{Queries: []PackageQuery{ &DependencyQuery{Dep: Dependency{Pkg: "app"}}, &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}, &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}}, &DependencyQuery{Dep: Dependency{Pkg: "xyz"}}, - &DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil) + &DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"}) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, + WithDependencies: true, + Source: NewPackageList(), + Architectures: []string{"i386"}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{ - &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}}, - &DependencyQuery{Dep: Dependency{Pkg: "lib"}}, - &DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"}) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{ + &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}}, + &DependencyQuery{Dep: Dependency{Pkg: "lib"}}, + &DependencyQuery{Dep: Dependency{Pkg: "data"}}}, + WithDependencies: true, + Source: NewPackageList(), + Architectures: []string{"i386", "amd64"}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "") - result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + //result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + // &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all") - result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"}, - &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"}, + &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&NotQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&NotQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, - &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, - }}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, + &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, + }}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, - &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, - }}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, + &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, + }}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") } diff --git a/deb/remote.go b/deb/remote.go index b12f0dd3..6845e0d8 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -596,7 +596,14 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu emptyList.PrepareIndex() oldLen = repo.packageList.Len() - repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress) + repo.packageList, err = repo.packageList.Filter(FilterOptions{ + Queries: []PackageQuery{filterQuery}, + WithDependencies: repo.FilterWithDeps, + Source: emptyList, + DependencyOptions: dependencyOptions, + Architectures: repo.Architectures, + Progress: progress, + }) if repo.packageList != nil { newLen = repo.packageList.Len() } From 0c76677b16aabdbdc3b109c19bd2bc0e5fb4aa26 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:19:46 +0900 Subject: [PATCH 08/64] Fix -with-sources not downloading differently named sources Such as e.g. downloading 'glibc' when the sources for 'libc6' are requested. --- deb/list.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ deb/package.go | 2 +- deb/remote.go | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/deb/list.go b/deb/list.go index 9f3f67a9..daca4372 100644 --- a/deb/list.go +++ b/deb/list.go @@ -2,6 +2,7 @@ package deb import ( "fmt" + "regexp" "sort" "strings" @@ -507,12 +508,24 @@ func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided boo type FilterOptions struct { Queries []PackageQuery WithDependencies bool + WithSources bool // Source packages correspond to binary packages are included Source *PackageList DependencyOptions int Architectures []string Progress aptly.Progress // set to non-nil to report progress } +// SourceRegex is a regular expression to match source package names. +// > In a binary package control file [...], the source package name may be followed by a version number in +// > parentheses. This version number may be omitted [...] if it has the same value as the Version field of +// > the binary package in question. +// > [...] +// > Package names (both source and binary, see Package) must consist only of lower case letters (a-z), +// > digits (0-9), plus (+) and minus (-) signs, and periods (.). +// > They must be at least two characters long and must start with an alphanumeric character. +// -- https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-source +var SourceRegex = regexp.MustCompile(`^([a-z0-9][-+.a-z0-9]+)(?:\s+\(([^)]+)\))?$`) + // Filter filters package index by specified queries (ORed together), possibly pulling dependencies func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { if !l.indexed { @@ -524,6 +537,37 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { for _, query := range options.Queries { _ = result.Append(query.Query(l)) } + // The above loop already finds source packages that are named equal to their binary package, but we still need + // to account for those that are named differently. + if options.WithSources { + sourceQueries := make([]PackageQuery, 0) + for _, pkg := range result.packages { + if pkg.Source == "" { + continue + } + matches := SourceRegex.FindStringSubmatch(pkg.Source) + if matches == nil { + return nil, fmt.Errorf("invalid Source field: %s", pkg.Source) + } + sourceName := matches[1] + if sourceName == pkg.Name { + continue + } + sourceVersion := pkg.Version + if matches[2] != "" { + sourceVersion = matches[2] + } + sourceQueries = append(sourceQueries, &DependencyQuery{Dependency{ + Pkg: sourceName, + Version: sourceVersion, + Relation: VersionEqual, + Architecture: ArchitectureSource, + }}) + } + for _, query := range sourceQueries { + _ = result.Append(query.Query(l)) + } + } if options.WithDependencies { added := result.Len() diff --git a/deb/package.go b/deb/package.go index cf6a98ca..fc4c9b68 100644 --- a/deb/package.go +++ b/deb/package.go @@ -450,7 +450,7 @@ func (p *Package) GetArchitecture() string { return p.Architecture } -// GetDependencies compiles list of dependncies by flags from options +// GetDependencies compiles list of dependencies by flags from options func (p *Package) GetDependencies(options int) (dependencies []string) { deps := p.Deps() diff --git a/deb/remote.go b/deb/remote.go index 6845e0d8..9f89f905 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -600,6 +600,7 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu Queries: []PackageQuery{filterQuery}, WithDependencies: repo.FilterWithDeps, Source: emptyList, + WithSources: repo.DownloadSources, DependencyOptions: dependencyOptions, Architectures: repo.Architectures, Progress: progress, From ef6815222cfd05498d65622fa980c64c4cb6641b Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:43:51 +0900 Subject: [PATCH 09/64] Add unit tests for filtering with source packages --- deb/list_test.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/deb/list_test.go b/deb/list_test.go index e13cd4f2..efff77a3 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -381,8 +381,6 @@ func (s *PackageListSuite) TestFilter(c *C) { c.Check(err, IsNil) c.Check(plString(result), Equals, "") - //result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - // &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) result, err = s.il.Filter(FilterOptions{ Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, @@ -435,6 +433,31 @@ func (s *PackageListSuite) TestFilter(c *C) { }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") + + // Different version for the source package + for _, p := range s.sourcePackages { + c.Check(s.il.Add(p), IsNil) + } + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "lib"}}}, + Architectures: []string{"i386", "amd64"}, + WithSources: true, + }) + c.Check(err, IsNil) + c.Check(plString(result), Equals, "lib_0.9_source lib_1.0_i386") + + // Different name for the source package + err = s.il.Add(&Package{Name: "glibc", Version: "1.0", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}) + c.Check(err, IsNil) + err = s.il.Add(&Package{Name: "libc1", Version: "1.0", Architecture: "i386", Source: "glibc", deps: &PackageDependencies{}}) + c.Check(err, IsNil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "libc1"}}}, + Architectures: []string{"i386"}, + WithSources: true, + }) + c.Check(err, IsNil) + c.Check(plString(result), Equals, "glibc_1.0_source libc1_1.0_i386") } func (s *PackageListSuite) TestVerifyDependencies(c *C) { From 8c3fe8dabbb5ad4ce5b3a7521da6429cfc2b1591 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Mon, 9 Dec 2024 14:16:42 +0900 Subject: [PATCH 10/64] Fix failing system test The fix of the -with-filter flag causes the following previously missing source files to be downloaded, so I updated the test file. ``` rkward_0.7.5-1~bullseyecran.0.debian.tar.xz rkward_0.7.5-1~bullseyecran.0.dsc rkward_0.7.5.orig.tar.gz rpy2_3.5.12-1~bullseyecran.0.debian.tar.xz rpy2_3.5.12-1~bullseyecran.0.dsc rpy2_3.5.12.orig.tar.gz ``` --- deb/list.go | 4 ++-- system/t04_mirror/UpdateMirror10Test_gold | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deb/list.go b/deb/list.go index daca4372..28644e40 100644 --- a/deb/list.go +++ b/deb/list.go @@ -508,11 +508,11 @@ func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided boo type FilterOptions struct { Queries []PackageQuery WithDependencies bool - WithSources bool // Source packages correspond to binary packages are included + WithSources bool // Source packages corresponding to binary packages are included Source *PackageList DependencyOptions int Architectures []string - Progress aptly.Progress // set to non-nil to report progress + Progress aptly.Progress // set to non-nil value to report progress } // SourceRegex is a regular expression to match source package names. diff --git a/system/t04_mirror/UpdateMirror10Test_gold b/system/t04_mirror/UpdateMirror10Test_gold index 54251a64..3b4ab6f2 100644 --- a/system/t04_mirror/UpdateMirror10Test_gold +++ b/system/t04_mirror/UpdateMirror10Test_gold @@ -2,7 +2,7 @@ Applying filter... Building download queue... -Download queue: 11 items (39.66 MiB) +Download queue: 17 items (47.22 MiB) Downloading & parsing package files... Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/InRelease Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/Packages.bz2 @@ -16,10 +16,16 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-data_0.7.5-1~bullseyecran.0_all.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_i386.deb +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0.debian.tar.xz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0.dsc Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_i386.deb +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5.orig.tar.gz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12-1~bullseyecran.0.debian.tar.xz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12-1~bullseyecran.0.dsc +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12.orig.tar.gz Mirror `flat-src` has been updated successfully. -Packages filtered: 110 -> 11. +Packages filtered: 110 -> 13. gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41 gpgv: Good signature from "Johannes Ranke " gpgv: Signature made Thu Nov 2 07:43:52 2023 UTC \ No newline at end of file From 568345c3967b824373942004f95395633548314c Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 10 Dec 2024 12:12:23 +0900 Subject: [PATCH 11/64] Add name to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 0da15b8b..69d479c3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -67,3 +67,4 @@ List of contributors, in chronological order: * Christoph Fiehe (https://github.com/cfiehe) * Blake Kostner (https://github.com/btkostner) * Leigh London (https://github.com/leighlondon) +* Gordian Schönherr (https://github.com/schoenherrg) From 5e91b10c8c21a54527d70e36dd647e70a292b707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 11 Dec 2024 05:32:50 +0100 Subject: [PATCH 12/64] improve test to check for source pkgs with different name --- system/t06_publish/snapshot.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index aa3fb54e..2ba245b4 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1344,6 +1344,26 @@ class PublishSnapshot41Test(BaseTest): self.check_exists('public/pool/main/libx/libxslt/libxslt1.1_1.1.32-2.2~deb10u2_i386.deb') self.check_exists('public/pool/main/libz/libzstd/libzstd1_1.3.8+dfsg-3+deb10u2_i386.deb') self.check_exists('public/pool/main/z/zlib/zlib1g_1.2.11.dfsg-1+deb10u2_i386.deb') + # check source packages with different names + self.check_exists('public/pool/main/u/util-linux/util-linux_2.33.1-0.1+deb10u1.dsc') + self.check_exists('public/pool/main/u/util-linux/util-linux_2.33.1-0.1+deb10u1.debian.tar.xz') + self.check_exists('public/pool/main/u/util-linux/util-linux_2.33.1.orig.tar.xz') + self.check_exists('public/pool/main/g/glibc/glibc_2.28-10+deb10u2.debian.tar.xz') + self.check_exists('public/pool/main/g/glibc/glibc_2.28-10+deb10u2.dsc') + self.check_exists('public/pool/main/g/glibc/glibc_2.28.orig.tar.xz') + self.check_exists('public/pool/main/n/ncurses/ncurses_6.1+20181013-2+deb10u5.debian.tar.xz') + self.check_exists('public/pool/main/n/ncurses/ncurses_6.1+20181013-2+deb10u5.dsc') + self.check_exists('public/pool/main/n/ncurses/ncurses_6.1+20181013.orig.tar.gz') + self.check_exists('public/pool/main/z/zlib/zlib_1.2.11.dfsg-1+deb10u2.debian.tar.xz') + self.check_exists('public/pool/main/z/zlib/zlib_1.2.11.dfsg-1+deb10u2.dsc') + self.check_exists('public/pool/main/z/zlib/zlib_1.2.11.dfsg.orig.tar.gz') + self.check_exists('public/pool/main/x/xz-utils/xz-utils_5.2.4-1+deb10u1.debian.tar.xz') + self.check_exists('public/pool/main/x/xz-utils/xz-utils_5.2.4-1+deb10u1.dsc') + self.check_exists('public/pool/main/x/xz-utils/xz-utils_5.2.4.orig.tar.xz') + self.check_exists('public/pool/main/e/e2fsprogs/e2fsprogs_1.44.5-1+deb10u2.debian.tar.xz') + self.check_exists('public/pool/main/e/e2fsprogs/e2fsprogs_1.44.5-1+deb10u2.dsc') + self.check_exists('public/pool/main/e/e2fsprogs/e2fsprogs_1.44.5.orig.tar.gz') + self.check_exists('public/pool/main/e/e2fsprogs/e2fsprogs_1.44.5.orig.tar.gz.asc') class PublishSnapshot42Test(BaseTest): From ba86851d0739ec1e318cc2bf3b0de6d4c9f16dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 01:06:25 +0200 Subject: [PATCH 13/64] add api documentation stubs --- api/api.go | 40 +++++++++++++++++-- api/db.go | 9 ++++- api/files.go | 42 +++++++++++++++---- api/gpg.go | 9 ++++- api/graph.go | 8 ++++ api/packages.go | 9 ++++- api/publish.go | 12 ++++++ api/repos.go | 104 ++++++++++++++++++++++++++++++++++++++++++------ api/s3.go | 4 +- api/snapshot.go | 72 +++++++++++++++++++++++++++++---- api/task.go | 82 +++++++++++++++++++++++++++++++++----- 11 files changed, 347 insertions(+), 44 deletions(-) diff --git a/api/api.go b/api/api.go index a52406d8..c66a44b1 100644 --- a/api/api.go +++ b/api/api.go @@ -23,12 +23,38 @@ import ( // 3. SnapshotCollection // 4. PublishedRepoCollection -// GET /api/version +type aptlyVersion struct { + // Aptly Version + Version string `json:"Version"` +} + +// @Summary Aptly version +// @Description **Get aptly version** +// @Description Returns the aptly version +// @Tags Status +// @Produce json +// @Success 200 {object} aptlyVersion +// @Router /api/version [get] func apiVersion(c *gin.Context) { c.JSON(200, gin.H{"Version": aptly.Version}) } -// GET /api/ready +type aptlyStatus struct { + // Aptly Status + Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"` +} + +// @Summary Ready State +// @Description **Get aptly ready state** +// @Description +// @Description Return aptly ready state: +// @Description - `Aptly is ready` (HTTP 200) +// @Description - `Aptly is unavailable` (HTTP 503) +// @Tags Status +// @Produce json +// @Success 200 {object} aptlyStatus "Aptly is ready" +// @Failure 503 {object} aptlyStatus "Aptly is unavailable" +// @Router /api/ready [get] func apiReady(isReady *atomic.Value) func(*gin.Context) { return func(c *gin.Context) { if isReady == nil || !isReady.Load().(bool) { @@ -40,7 +66,15 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) { } } -// GET /api/healthy +// @Summary Health State +// @Description **Get aptly health state** +// @Description +// @Description Return aptly health state: +// @Description - `Aptly is healthy` (HTTP 200) +// @Tags Status +// @Produce json +// @Success 200 {object} aptlyStatus +// @Router /api/healthy [get] func apiHealthy(c *gin.Context) { c.JSON(200, gin.H{"Status": "Aptly is healthy"}) } diff --git a/api/db.go b/api/db.go index 3f8b826d..5f7727ff 100644 --- a/api/db.go +++ b/api/db.go @@ -11,7 +11,14 @@ import ( "github.com/gin-gonic/gin" ) -// POST /api/db/cleanup +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Database +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/db/cleanup [post] func apiDbCleanup(c *gin.Context) { resources := []string{string(task.AllResourcesKey)} diff --git a/api/files.go b/api/files.go index 7ae682f8..64e3a527 100644 --- a/api/files.go +++ b/api/files.go @@ -34,10 +34,10 @@ func verifyDir(c *gin.Context) bool { return true } -// @Summary Get files -// @Description Get list of uploaded files. +// @Summary List Files +// @Description **Get list of uploaded files** // @Tags Files -// @Produce json +// @Produce json // @Success 200 {array} string "List of files" // @Router /api/files [get] func apiFilesListDirs(c *gin.Context) { @@ -67,7 +67,14 @@ func apiFilesListDirs(c *gin.Context) { c.JSON(200, list) } -// POST /files/:dir/ +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Files +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/files/{dir} [post] func apiFilesUpload(c *gin.Context) { if !verifyDir(c) { return @@ -121,7 +128,14 @@ func apiFilesUpload(c *gin.Context) { } -// GET /files/:dir +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Files +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/files/{dir} [get] func apiFilesListFiles(c *gin.Context) { if !verifyDir(c) { return @@ -159,7 +173,14 @@ func apiFilesListFiles(c *gin.Context) { c.JSON(200, list) } -// DELETE /files/:dir +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Files +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/files/{dir} [delete] func apiFilesDeleteDir(c *gin.Context) { if !verifyDir(c) { return @@ -174,7 +195,14 @@ func apiFilesDeleteDir(c *gin.Context) { c.JSON(200, gin.H{}) } -// DELETE /files/:dir/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Files +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/files/{dir}/{name} [delete] func apiFilesDeleteFile(c *gin.Context) { if !verifyDir(c) { return diff --git a/api/gpg.go b/api/gpg.go index 72e938d2..835bbb71 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -12,7 +12,14 @@ import ( "github.com/gin-gonic/gin" ) -// POST /api/gpg +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags GPG +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/gpg [post] func apiGPGAddKey(c *gin.Context) { var b struct { Keyserver string diff --git a/api/graph.go b/api/graph.go index d04e687f..cd04063b 100644 --- a/api/graph.go +++ b/api/graph.go @@ -12,6 +12,14 @@ import ( "github.com/gin-gonic/gin" ) +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Graph +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/graph [get] // GET /api/graph.:ext?layout=[vertical|horizontal(default)] func apiGraph(c *gin.Context) { var ( diff --git a/api/packages.go b/api/packages.go index 09ae785a..e649a80e 100644 --- a/api/packages.go +++ b/api/packages.go @@ -4,7 +4,14 @@ import ( "github.com/gin-gonic/gin" ) -// GET /api/packages/:key +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Packages +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/packages/{key} [get] func apiPackagesShow(c *gin.Context) { collectionFactory := context.NewCollectionFactory() p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key"))) diff --git a/api/publish.go b/api/publish.go index 9b10e65f..ae9bec52 100644 --- a/api/publish.go +++ b/api/publish.go @@ -102,6 +102,7 @@ func apiPublishList(c *gin.Context) { c.JSON(http.StatusOK, repos) } +<<<<<<< HEAD // @Summary Show Published Repository // @Description **Get published repository information** // @Description @@ -117,6 +118,17 @@ func apiPublishList(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution} [get] func apiPublishShow(c *gin.Context) { +======= +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Publish +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/publish/{prefix} [post] +func apiPublishRepoOrSnapshot(c *gin.Context) { +>>>>>>> d237283f (add api documentation stubs) param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) diff --git a/api/repos.go b/api/repos.go index 005c2aaa..bf540c52 100644 --- a/api/repos.go +++ b/api/repos.go @@ -18,7 +18,14 @@ import ( "github.com/gin-gonic/gin" ) -// GET /repos +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos [get] func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc { return func(c *gin.Context) { c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -35,7 +42,16 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H } } -// GET /repos/:storage/*pkgPath +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Param storage path string true "Storage" +// @Param pkgPath path string true "Package Path" allowReserved=true +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/{storage}/{pkgPath} [get] func reposServeInAPIMode(c *gin.Context) { pkgpath := c.Param("pkgPath") @@ -51,7 +67,8 @@ func reposServeInAPIMode(c *gin.Context) { } // @Summary Get repos -// @Description Get list of available repos. Each repo is returned as in “show” API. +// @Description **Get list of available repos** +// @Description Each repo is returned as in “show” API. // @Tags Repos // @Produce json // @Success 200 {array} deb.LocalRepo @@ -142,7 +159,14 @@ func apiReposCreate(c *gin.Context) { c.JSON(http.StatusCreated, repo) } -// PUT /api/repos/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name} [put] func apiReposEdit(c *gin.Context) { var b struct { Name *string @@ -214,7 +238,14 @@ func apiReposShow(c *gin.Context) { c.JSON(200, repo) } -// DELETE /api/repos/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name} [delete] func apiReposDrop(c *gin.Context) { force := c.Request.URL.Query().Get("force") == "1" name := c.Params.ByName("name") @@ -249,7 +280,14 @@ func apiReposDrop(c *gin.Context) { }) } -// GET /api/repos/:name/packages +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/packages [get] func apiReposPackagesShow(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.LocalRepoCollection() @@ -330,7 +368,14 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li }) } -// POST /repos/:name/packages +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/packages [post] func apiReposPackagesAdd(c *gin.Context) { apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error { out.Printf("Adding package %s\n", p.Name) @@ -338,7 +383,14 @@ func apiReposPackagesAdd(c *gin.Context) { }) } -// DELETE /repos/:name/packages +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/packages [delete] func apiReposPackagesDelete(c *gin.Context) { apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error { out.Printf("Removing package %s\n", p.Name) @@ -347,7 +399,14 @@ func apiReposPackagesDelete(c *gin.Context) { }) } -// POST /repos/:name/file/:dir/:file +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/file/{dir}/{file} [post] func apiReposPackageFromFile(c *gin.Context) { // redirect all work to dir method apiReposPackageFromDir(c) @@ -487,7 +546,14 @@ func apiReposPackageFromDir(c *gin.Context) { }) } -// POST /repos/:name/copy/:src/:file +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/copy/{src}/{file} [post] func apiReposCopyPackage(c *gin.Context) { dstRepoName := c.Params.ByName("name") srcRepoName := c.Params.ByName("src") @@ -626,13 +692,27 @@ func apiReposCopyPackage(c *gin.Context) { }) } -// POST /repos/:name/include/:dir/:file +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/include/{dir}/{file} [post] func apiReposIncludePackageFromFile(c *gin.Context) { // redirect all work to dir method apiReposIncludePackageFromDir(c) } -// POST /repos/:name/include/:dir +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Repos +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/include/{dir} [post] func apiReposIncludePackageFromDir(c *gin.Context) { forceReplace := c.Request.URL.Query().Get("forceReplace") == "1" noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1" diff --git a/api/s3.go b/api/s3.go index 33ed0c83..19be7537 100644 --- a/api/s3.go +++ b/api/s3.go @@ -4,8 +4,8 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary Get S3 buckets -// @Description Get list of S3 buckets. +// @Summary S3 buckets +// @Description **Get list of S3 buckets** // @Tags S3 // @Produce json // @Success 200 {array} string "List of S3 buckets" diff --git a/api/snapshot.go b/api/snapshot.go index 1a9c2278..75110f71 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -39,7 +39,14 @@ func apiSnapshotsList(c *gin.Context) { c.JSON(200, result) } -// POST /api/mirrors/:name/snapshots/ +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/mirrors/{name}/snapshots [post] func apiSnapshotsCreateFromMirror(c *gin.Context) { var ( err error @@ -98,7 +105,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { }) } -// POST /api/snapshots +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots [post] func apiSnapshotsCreate(c *gin.Context) { var ( err error @@ -173,7 +187,14 @@ func apiSnapshotsCreate(c *gin.Context) { }) } -// POST /api/repos/:name/snapshots +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/repos/{name}/snapshots [post] func apiSnapshotsCreateFromRepository(c *gin.Context) { var ( err error @@ -227,7 +248,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { }) } -// PUT /api/snapshots/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots/{name} [put] func apiSnapshotsUpdate(c *gin.Context) { var ( err error @@ -277,7 +305,14 @@ func apiSnapshotsUpdate(c *gin.Context) { }) } -// GET /api/snapshots/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots/{name} [get] func apiSnapshotsShow(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.SnapshotCollection() @@ -297,7 +332,14 @@ func apiSnapshotsShow(c *gin.Context) { c.JSON(200, snapshot) } -// DELETE /api/snapshots/:name +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots/{name} [delete] func apiSnapshotsDrop(c *gin.Context) { name := c.Params.ByName("name") force := c.Request.URL.Query().Get("force") == "1" @@ -336,7 +378,14 @@ func apiSnapshotsDrop(c *gin.Context) { }) } -// GET /api/snapshots/:name/diff/:withSnapshot +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots/{name}/diff/{withSnapshot} [get] func apiSnapshotsDiff(c *gin.Context) { onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1" @@ -387,7 +436,14 @@ func apiSnapshotsDiff(c *gin.Context) { c.JSON(200, result) } -// GET /api/snapshots/:name/packages +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Snapshots +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/snapshots/{name}/packages [get] func apiSnapshotsSearchPackages(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.SnapshotCollection() diff --git a/api/task.go b/api/task.go index 189d7931..a33510aa 100644 --- a/api/task.go +++ b/api/task.go @@ -20,21 +20,41 @@ func apiTasksList(c *gin.Context) { c.JSON(200, list.GetTasks()) } -// POST /tasks-clear +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Router /api/tasks-clear [post] func apiTasksClear(c *gin.Context) { list := context.TaskList() list.Clear() c.JSON(200, gin.H{}) } -// GET /tasks-wait +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks-wait [get] func apiTasksWait(c *gin.Context) { list := context.TaskList() list.Wait() c.JSON(200, gin.H{}) } -// GET /tasks/:id/wait +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id}/wait [get] func apiTasksWaitForTaskByID(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -52,7 +72,14 @@ func apiTasksWaitForTaskByID(c *gin.Context) { c.JSON(200, task) } -// GET /tasks/:id +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id} [get] func apiTasksShow(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -71,7 +98,14 @@ func apiTasksShow(c *gin.Context) { c.JSON(200, task) } -// GET /tasks/:id/output +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id}/output [get] func apiTasksOutputShow(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -90,7 +124,14 @@ func apiTasksOutputShow(c *gin.Context) { c.JSON(200, output) } -// GET /tasks/:id/detail +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id}/detail [get] func apiTasksDetailShow(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -109,7 +150,14 @@ func apiTasksDetailShow(c *gin.Context) { c.JSON(200, detail) } -// GET /tasks/:id/return_value +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id}/return_value [get] func apiTasksReturnValueShow(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -127,7 +175,14 @@ func apiTasksReturnValueShow(c *gin.Context) { c.JSON(200, output) } -// DELETE /tasks/:id +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks/{id} [delete] func apiTasksDelete(c *gin.Context) { list := context.TaskList() id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0) @@ -146,7 +201,16 @@ func apiTasksDelete(c *gin.Context) { c.JSON(200, delTask) } -// POST /tasks-dummy +// FIXME: used for testing only, remove: + +// @Summary TODO +// @Description **ToDo** +// @Description To Do +// @Tags Tasks +// @Produce json +// @Success 200 {object} string "msg" +// @Failure 404 {object} Error "Not Found" +// @Router /api/tasks-dummy [post] func apiTasksDummy(c *gin.Context) { resources := []string{"dummy"} taskName := "Dummy task" From cc1fc7ccfe06b29ac2d7c87a070f40aba550137f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 01:54:34 +0200 Subject: [PATCH 14/64] allow comments in config file --- utils/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/config.go b/utils/config.go index b25ba4ea..b061649f 100644 --- a/utils/config.go +++ b/utils/config.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/DisposaBoy/JsonConfigReader" ) // ConfigStructure is structure of main configuration @@ -192,7 +194,7 @@ func LoadConfig(filename string, config *ConfigStructure) error { } defer f.Close() - dec := json.NewDecoder(f) + dec := json.NewDecoder(JsonConfigReader.New(f)) return dec.Decode(&config) } From 2df82e87b7067190bfaf4b3dc36134977466e12d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 15:48:18 +0200 Subject: [PATCH 15/64] document aptly.conf --- debian/aptly.conf | 258 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 234 insertions(+), 24 deletions(-) diff --git a/debian/aptly.conf b/debian/aptly.conf index d091e474..4d627fc4 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,38 +1,248 @@ +// vim: : filetype=json { + +// General +/////////// + + // Aptly storage directory + // - downloaded packages (`rootDir`/pool) + // - database (`rootDir`/db) + // - published repositories (`rootDir`/public) "rootDir": "~/.aptly", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", + + // number of attempts to open DB if it's locked by other instance; can be overridden with option `-db-open-attempts` "databaseOpenAttempts": -1, - "architectures": [], - "dependencyFollowSuggests": false, - "dependencyFollowRecommends": false, - "dependencyFollowAllVariants": false, - "dependencyFollowSource": false, - "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "skipLegacyPool": true, - "ppaDistributorID": "ubuntu", - "ppaCodename": "", - "skipContentsPublishing": false, - "skipBz2Publishing": false, - "FileSystemPublishEndpoints": {}, - "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {}, - "AzurePublishEndpoints": {}, + + // "AsyncAPI": false, + + // "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // "logLevel": "info", + + // "logFormat": "default", + + // "serveInAPIMode": false, + + // OBSOLETE + // in aptly up to version 1.0.0, package files were stored in internal package pool + // with MD5-dervied path, since 1.1.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + +// Database +//////////// + + // "databaseBackend": { + // "type": "", + // "url": "", + // "dbPath": "" + // }, - "enableSwaggerEndpoint": false + +// Mirroring +///////////// + + // downloader to use + // - "default" (normal downloader) + // - "grab" (more robust) + "downloader": "default", + + // number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // number of retries for download attempts + "downloadRetries": 0, + + // download source packages per default + "downloadSourcePackages": false, + + + // list of architectures to process; if left empty defaults to all available architectures; can be overridden with option `-architectures` + "architectures": [], + + // follow contents of `Suggests:` field when processing dependencies for the package + "dependencyFollowSuggests": false, + + // follow contents of `Recommends:` field when processing dependencies for the package + "dependencyFollowRecommends": false, + + // when dependency looks like `package-a | package-b`, follow both variants always + "dependencyFollowAllVariants": false, + + // follow dependency from binary package to source package + "dependencyFollowSource": false, + + // print additional details while resolving dependencies (useful for debugging) + "dependencyVerboseResolve": false, + +// Signing +/////////// + + // gpg provider to use: + // - "internal" (Go internal implementation) + // - 'gpg" (external `gpg` utility, uses GnuPG 1.x if available or GnuPG 2.x otherwise) + "gpgProvider": "gpg", + + // don't sign published repositories with gpg(1), also can be disabled on per-repo basis using `-skip-signing` flag when publishing + "gpgDisableSign": false, + + // don't verify remote mirrors with gpg(1), also can be disabled on per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors + "gpgDisableVerify": false, + + +// PPA +/////// + + // specifies paramaters for short PPA url expansion, if left blank they default to output of `lsb_release` command + "ppaDistributorID": "ubuntu", + + // cwcodename for short PPA url expansion + "ppaCodename": "", + + // + "skipContentsPublishing": false, + + // + "skipBz2Publishing": false, + +// Storage Endpoints +///////////////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name + // and the following associated settings: + // + // * `rootDir`: + // The publish directory, e.g., `/opt/srv/aptly_public`. + // * `linkMethod`: + // This is one of `hardlink`, `symlink` or `copy`. It specifies how aptly links the + // files from the internal pool to the published directory. + // If not specified, empty or wrong, this defaults to `hardlink`. + // * `verifyMethod`: + // This is used only when setting the `linkMethod` to `copy`. Possible values are + // `md5` and `size`. It specifies how aptly compares existing links from the + // internal pool to the published directory. The `size` method compares only the + // file sizes, whereas the `md5` method calculates the md5 checksum of the found + // file and compares it to the desired one. + // If not specified, empty or wrong, this defaults to `md5`. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` + // with `endpoint-name` as the name given in the aptly configuration file. For example: + // + // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` + "FileSystemPublishEndpoints": { + }, + + // S3 Endpoint Support + // + // cloud storage). First, publishing + // endpoints should be described in aptly configuration file. Each endpoint has name + // and associated settings: + // + // * `region`: + // Amazon region for S3 bucket (e.g. `us-east-1`) + // * `bucket`: + // bucket name + // * `endpoint`: + // (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here, + // region is ignored if endpoint is set (set region to some human-readable name) + // (should be left blank for real Amazon S3) + // * `prefix`: + // (optional) do publishing under specified prefix in the bucket, defaults to + // no prefix (bucket root) + // * `acl`: + // (optional) assign ACL to published files (one of the canned ACLs in Amazon + // terminology). Useful values: `private` (default), `public-read` (public + // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using + // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // for private repositories special apt S3 transport is required. + // * `awsAccessKeyID`, `awsSecretAccessKey`: + // (optional) Amazon credentials to access S3 bucket. If not supplied, + // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // are used. + // * `storageClass`: + // (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values + // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // * `encryptionMethod`: + // (optional) server-side encryption method, defaults to none. Currently + // the only available encryption method is `AES256` + // * `plusWorkaround`: + // (optional) workaround misbehavior in apt and Amazon S3 + // for files with `+` in filename by + // creating two copies of package files with `+` in filename: one original + // and another one with spaces instead of plus signs + // With `plusWorkaround` enabled, package files with plus sign + // would be stored twice. aptly might not cleanup files with spaces when published + // repository is dropped or updated (switched) to new version of repository (snapshot) + // * `disableMultiDel`: + // (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API, + // enable this setting (file deletion would be slower with this setting enabled) + // * `forceSigV2`: + // (optional) disable Signature V4 support, useful with non-AWS S3-compatible object stores + // which do not support SigV4, shouldn't be enabled for AWS + // * `forceVirtualHostedStyle`: + // (optional) disable path style visit, useful with non-AWS S3-compatible object stores + // which only support virtual hosted style + // * `debug`: + // (optional) enables detailed request/response dump for each S3 operation + // + // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main s3:test:` + "S3PublishEndpoints": { + }, + + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift. First, + // publishing endpoints should be described in aptly configuration file. Each endpoint + // has name and associated settings: + // + // * `container`: + // container name + // * `prefix`: + // (optional) do publishing under specified prefix in the container, defaults to + // no prefix (container root) + // * `osname`, `password`: + // (optional) OpenStack credentials to access Keystone. If not supplied, + // environment variables `OS_USERNAME` and `OS_PASSWORD` are used. + // * `tenant`, `tenantid`: + // (optional) OpenStack tenant name and id (in order to use v2 authentication). + // * `authurl`: + // (optional) the full url of Keystone server (including port, and version). + // example `http://identity.example.com:5000/v2.0` + // + // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot jessie-main swift:test:` + "SwiftPublishEndpoints": { + }, + + // Azure Endpoint Support + // + "AzurePublishEndpoints": { + } + } From 69a1e2561d86aa6691645361629be5cfef47288f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 5 Oct 2024 15:26:16 +0200 Subject: [PATCH 16/64] docs: improve swagger - use markdown files in swagger - automate version, use swager.conf template - embed swagger ui index.html as docs.html --- .gitignore | 5 +- Makefile | 10 ++- api/router.go | 27 ++++---- docs/Database.md | 1 + docs/Publish.md | 29 +++++++++ docs/Status.md | 10 +++ docs/api.md | 112 +++++++++++++++++++++++++++++++++ docs/docs.html | 142 ++++++++++++++++++++++++++++++++++++++++++ docs/index.go | 8 ++- docs/swagger.conf.tpl | 16 +++++ 10 files changed, 341 insertions(+), 19 deletions(-) create mode 100644 docs/Database.md create mode 100644 docs/Publish.md create mode 100644 docs/Status.md create mode 100644 docs/api.md create mode 100644 docs/docs.html create mode 100644 docs/swagger.conf.tpl diff --git a/.gitignore b/.gitignore index 673918cb..f2333751 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,7 @@ usr/bin/aptly dpkgs/ debian/changelog.dpkg-bak -docs/ +docs/docs.go +docs/swagger.json +docs/swagger.yaml +docs/swagger.conf diff --git a/Makefile b/Makefile index 632d7066..f0b945c1 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ version: ## Print aptly version swagger-install: # Install swag @test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest + # Generate swagger.conf + cp docs/swagger.conf.tpl docs/swagger.conf + echo "// @version $(VERSION)" >> docs/swagger.conf azurite-start: azurite & \ @@ -55,7 +58,7 @@ azurite-stop: swagger: swagger-install # Generate swagger docs - @PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs + @PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs --generalInfo docs/swagger.conf etcd-install: # Install etcd @@ -107,7 +110,7 @@ serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) @@ -205,7 +208,8 @@ man: ## Create man pages clean: ## remove local build and module cache # Clean all generated and build files - test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true + find .go/ -type d ! -perm -u=w -exec chmod u+w {} \; + rm -rf .go/ rm -rf build/ obj-*-linux-gnu* tmp/ rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true diff --git a/api/router.go b/api/router.go index ada14b88..f7c95fd3 100644 --- a/api/router.go +++ b/api/router.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" - _ "github.com/aptly-dev/aptly/docs" // import docs + "github.com/aptly-dev/aptly/docs" swaggerFiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" ) @@ -27,26 +27,22 @@ func apiMetricsGet() gin.HandlerFunc { } func redirectSwagger(c *gin.Context) { + if c.Request.URL.Path == "/docs/index.html" { + c.Redirect(http.StatusMovedPermanently, "/docs.html") + return + } if c.Request.URL.Path == "/docs/" { - c.Redirect(http.StatusMovedPermanently, "/docs/index.html") + c.Redirect(http.StatusMovedPermanently, "/docs.html") + return + } + if c.Request.URL.Path == "/docs" { + c.Redirect(http.StatusMovedPermanently, "/docs.html") return } c.Next() } // Router returns prebuilt with routes http.Handler -// @title Aptly API -// @version 1.0 -// @description Aptly REST API Documentation - -// @contact.name Aptly -// @contact.url http://github.com/aptly-dev/aptly -// @contact.email support@aptly.info - -// @license.name MIT License -// @license.url http://www. - -// @BasePath /api func Router(c *ctx.AptlyContext) http.Handler { if aptly.EnableDebug { gin.SetMode(gin.DebugMode) @@ -73,6 +69,9 @@ func Router(c *ctx.AptlyContext) http.Handler { router.Use(gin.Recovery(), gin.ErrorLogger()) if c.Config().EnableSwaggerEndpoint { + router.GET("docs.html", func(c *gin.Context) { + c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML) + }) router.Use(redirectSwagger) url := ginSwagger.URL("/docs/doc.json") router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url)) diff --git a/docs/Database.md b/docs/Database.md new file mode 100644 index 00000000..8786b3ff --- /dev/null +++ b/docs/Database.md @@ -0,0 +1 @@ +# Aptly Database Operations diff --git a/docs/Publish.md b/docs/Publish.md new file mode 100644 index 00000000..d87c87cc --- /dev/null +++ b/docs/Publish.md @@ -0,0 +1,29 @@ +# Aptly Publish Points +
+ +Publish snapshot or local repo as Debian repository which could be +served by HTTP/FTP/rsync server. Repository is signed by user's key with +GnuPG. Key should be created beforehand (see section GPG Keys below). +Published repository could be consumed directly by apt. + +Repositories could be published to Amazon S3 service: create bucket, +[configure publishing endpoint](/doc/feature/s3/) and use S3 endpoint when +publishing. + + +#### GPG Keys + +GPG key is required to sign any published repository. Key should be +generated before publishing first repository. + +Key generation, storage, backup and revocation is out of scope of this +document, there are many tutorials available, e.g. [this one](http://fedoraproject.org/wiki/Creating_GPG_Keys). + +Publiс part of the key should be exported from your keyring using `gpg --export --armor` and +imported into apt keyring using `apt-key` tool on all machines that would be using published +repositories. + +Signing releases is highly recommended, but if you want to skip it, you +can either use `gpgDisableSign` configuration option or `--skip-signing` +flag. +
diff --git a/docs/Status.md b/docs/Status.md new file mode 100644 index 00000000..e19de2db --- /dev/null +++ b/docs/Status.md @@ -0,0 +1,10 @@ +# Aptly Status Information +
+ +## Something + +Very interesting ... asd wer + + + +
diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 00000000..9d236ce2 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,112 @@ + +Using aptly via REST API allows to achieve two goals: + +1. Remote access to aptly service: e.g. uploading packages and publishing them from CI server. +2. Concurrent access to aptly by multiple users. + +#### Quickstart + +Run `aptly api serve` to start HTTP service: + + $ aptly api serve + Starting web server at: :8080 (press Ctrl+C to quit)... + [GIN-debug] GET /api/version --> github.com/aptly-dev/aptly/api.apiVersion (4 handlers) + ... + +By default aptly would listen on `:8080`, but it could be changed with `-listen` flag. + +Usage: + + $ aptly api serve -listen=:8080 + +Flags: + +- `-listen=":8080"`: host:port for HTTP listening +- `-no-lock`: don't lock the database + +When `-no-lock` option is enabled, API server acquires and drops the lock +around all the operations, so that API and CLI could be used concurrently. + +Try some APIs: + + $ curl http://localhost:8080/api/version + {"Version":"0.9~dev"} + + $ curl -F file=@aptly_0.8_i386.deb http://localhost:8080/api/files/aptly_0.8 + ["aptly_0.8/aptly_0.8_i386.deb"] + + $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos + {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""} + + $ curl -X POST http://localhost:8080/api/repos/aptly-repo/file/aptly_0.8 + {"failedFiles":[],"report":{"warnings":[],"added":["aptly_0.8_i386 added"],"removed":[]}} + + $ curl http://localhost:8080/api/repos/aptly-repo/packages + ["Pi386 aptly 0.8 966561016b44ed80"] + + $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos + {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} + +#### Security + +For security reasons it is advised to let Aptly listen on a Unix domain socket rather than a port. Sockets are subject to file permissions and thus allow for user-level access control while binding to a port only gives host-level access control. To use a socket simply run Aptly with a suitable listen flag such as `aptly api serve -listen=unix:///var/run/aptly.sock`. + +Aptly's HTTP API shouldn't be directly exposed to the Internet: there's no authentication/protection of APIs. To publish the API it could be proxied through a HTTP proxy or server (e.g. nginx) to add an authentication mechanism or disallow all non-GET requests. [Reference example](https://github.com/sepich/nginx-ldap) for LDAP based per-repo access with nginx. + +#### Notes + +1. Some things (for example, S3 publishing endpoints) could be set up only using configuration file, so it requires restart of aptly HTTP server in order for changes to take effect. +1. GPG key passphrase can't be input on console, so either passwordless GPG keys are required or passphrase should be specified in API parameters. +1. Some script might be required to start/stop aptly HTTP service. +1. Some parameters are given as part of URLs, which requires proper url encoding. Unfortunately, at the moment it's not possible to pass URL arguments with `/` in them. + +### How to implement equivalent of aptly commands using API + +* `aptly mirror`: [mirror API](/doc/api/mirror) + * `list`: list + * `create`: create + * `drop`: delete + * `show`: show + * `search`: show packages/search + * `update`: update mirror +* `aptly repo`: [local repos API](/doc/api/repos) + * `add`: [file upload API](/doc/api/files) + add packages from uploaded file + * `copy`: show packages/search + add packages by key + * `create`: create + * `drop`: delete + * `edit`: edit + * `import`: not available, as mirror API is not implemented yet + * `list`: list + * `move`: show packages/search + add packages by key + delete packages by key + * `remove`: show packages/search + delete packages by key + * `rename`: not available yet, should be part of edit API + * `search`: show packages/search + * `show`: show +* `aptly snapshot`: [snapshots API](/doc/api/snapshots) + * `create`: + * empty: create snapshot with empty package references + * from mirror: not available yet + * from local repo: create snapshot from local repo + * `diff`: snapshot difference API + * `drop`: delete + * `filter`: show packages/search + create snapshot from package references + * `list`: list + * `merge`: show packages/search + processing + create snapshot from package references + * `pull`: show packages/search + processing + create snapshot from package references (might be implemented as API in future versions) + * `rename`: edit + * `search`: show packages/search + * `show`: show + * `verify`: not available yet +* `aptly publish`: [publish API](/doc/api/publish) + * `drop`: delete + * `list`: list + * `repo`: publish repo + * `snapshot`: publish snapshot + * `switch`: switch/update + * `update`: switch/update +* `aptly package`: [packages API](/doc/api/packages) + * `search`: not available yet + * `show`: only one format, with package key as input +* `aptly graph`: [graph API](/doc/api/misc) +* `aptly version`: [version API](/doc/api/misc) +* `aptly db`: not available yet diff --git a/docs/docs.html b/docs/docs.html new file mode 100644 index 00000000..cb405d9b --- /dev/null +++ b/docs/docs.html @@ -0,0 +1,142 @@ + + + + + Swagger UI + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + diff --git a/docs/index.go b/docs/index.go index da325d03..8350449a 100644 --- a/docs/index.go +++ b/docs/index.go @@ -1,3 +1,9 @@ package docs -import _ "github.com/swaggo/swag" // make sure swag is in go.mod +import ( + _ "embed" + _ "github.com/swaggo/swag" // make sure swag is in go.mod +) + +//go:embed docs.html +var DocsHTML []byte diff --git a/docs/swagger.conf.tpl b/docs/swagger.conf.tpl new file mode 100644 index 00000000..670470b9 --- /dev/null +++ b/docs/swagger.conf.tpl @@ -0,0 +1,16 @@ +package docs + +// @title Aptly API +// @description.markdown + +// @contact.name Aptly +// @contact.url http://github.com/aptly-dev/aptly + +// @Tag.name Publish +// @Tag.description.markdown +// @Tag.name Database +// @Tag.description.markdown +// @Tag.name Status +// @Tag.description.markdown + +// version will be appended here: From e677a2e84a7fe3897e0c7dbbf5fbae42fd5c57d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 16:13:37 +0200 Subject: [PATCH 17/64] go mod tidy --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 91f7e565..75e96377 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,8 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/swaggo/swag v1.16.3 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/etcd/api/v3 v3.5.15 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect @@ -127,6 +129,5 @@ require ( github.com/aws/smithy-go v1.22.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 - github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 ) From 096fa47c6df1b099c8e7cbade5853b9970b5eee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 15 Oct 2024 13:59:22 +0200 Subject: [PATCH 18/64] update doc --- docs/Publish.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/Publish.md b/docs/Publish.md index d87c87cc..b99a2cc8 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -26,4 +26,12 @@ repositories. Signing releases is highly recommended, but if you want to skip it, you can either use `gpgDisableSign` configuration option or `--skip-signing` flag. + +#### Parameters + +Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`. +`:distribution` is distribution name, while `:prefix` is `[:]` (storage is optional, it defaults +to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores +should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambigious in URLs. + From 9b8f6b1d56fb403cc40005d3186c212c633df4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 1 Nov 2024 21:09:23 +0100 Subject: [PATCH 19/64] fix conflict --- api/publish.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/api/publish.go b/api/publish.go index ae9bec52..9b10e65f 100644 --- a/api/publish.go +++ b/api/publish.go @@ -102,7 +102,6 @@ func apiPublishList(c *gin.Context) { c.JSON(http.StatusOK, repos) } -<<<<<<< HEAD // @Summary Show Published Repository // @Description **Get published repository information** // @Description @@ -118,17 +117,6 @@ func apiPublishList(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution} [get] func apiPublishShow(c *gin.Context) { -======= -// @Summary TODO -// @Description **ToDo** -// @Description To Do -// @Tags Publish -// @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" -// @Router /api/publish/{prefix} [post] -func apiPublishRepoOrSnapshot(c *gin.Context) { ->>>>>>> d237283f (add api documentation stubs) param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) From c0556119147c49a7421eef016ecd43279f0607fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 1 Nov 2024 21:10:28 +0100 Subject: [PATCH 20/64] update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 75e96377..d7bbe7af 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect - github.com/swaggo/swag v1.16.3 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/etcd/api/v3 v3.5.15 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect @@ -129,5 +128,6 @@ require ( github.com/aws/smithy-go v1.22.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 + github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 ) From 8f8de4bd29b92b06f7c3e06965043dcad75c9004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 2 Nov 2024 18:09:43 +0100 Subject: [PATCH 21/64] update --- api/files.go | 7 ++++-- api/repos.go | 9 +++++--- api/snapshot.go | 23 +++++++++++-------- docs/Database.md | 3 +++ docs/Files.md | 4 ++++ docs/GPG.md | 4 ++++ docs/Graph.md | 4 ++++ docs/Mirrors.md | 5 +++++ docs/Packages.md | 5 +++++ docs/Publish.md | 23 +++++-------------- docs/Repos.md | 5 +++++ docs/S3.md | 5 +++++ docs/Snapshots.md | 9 ++++++++ docs/Tasks.md | 4 ++++ docs/api.md | 52 ------------------------------------------- docs/docs.html | 13 ++++++++--- docs/swagger.conf.tpl | 18 +++++++++++++++ 17 files changed, 107 insertions(+), 86 deletions(-) create mode 100644 docs/Files.md create mode 100644 docs/GPG.md create mode 100644 docs/Graph.md create mode 100644 docs/Mirrors.md create mode 100644 docs/Packages.md create mode 100644 docs/Repos.md create mode 100644 docs/S3.md create mode 100644 docs/Snapshots.md create mode 100644 docs/Tasks.md diff --git a/api/files.go b/api/files.go index 64e3a527..9adf1d4b 100644 --- a/api/files.go +++ b/api/files.go @@ -67,10 +67,13 @@ func apiFilesListDirs(c *gin.Context) { c.JSON(200, list) } -// @Summary TODO -// @Description **ToDo** +// @Summary Upload file glaa +// @Description **Upload a file to a directory** // @Description To Do // @Tags Files +// @Accept multipart/form-data +// @Param dir path string true "Directory to upload files to" +// @Param files formData file true "Files to upload" // @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" diff --git a/api/repos.go b/api/repos.go index bf540c52..413a49e0 100644 --- a/api/repos.go +++ b/api/repos.go @@ -399,10 +399,13 @@ func apiReposPackagesDelete(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** +// @Summary Add packages +// @Description **Add package file** // @Description To Do // @Tags Repos +// @Param name path string true "Repository name" +// @Param dir path string true "Directory of packages" +// @Param file path string false "Filename (optional)" // @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" @@ -427,7 +430,7 @@ func apiReposPackageFromFile(c *gin.Context) { // @Failure 400 {object} Error "wrong file" // @Failure 404 {object} Error "Repository not found" // @Failure 500 {object} Error "Error adding files" -// @Router /api/repos/{name}/{dir} [post] +// @Router /api/repos/{name}/file/{dir} [post] func apiReposPackageFromDir(c *gin.Context) { forceReplace := c.Request.URL.Query().Get("forceReplace") == "1" noRemove := c.Request.URL.Query().Get("noRemove") == "1" diff --git a/api/snapshot.go b/api/snapshot.go index 75110f71..54433096 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -187,10 +187,18 @@ func apiSnapshotsCreate(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** +type snapshotRepositoryParams struct { + Name string `binding:"required"` + Description string +} + +// @Summary Snapshot Repository +// @Description **Create a sbalshot of a repository** // @Description To Do // @Tags Snapshots +// @Param name path string true "Repository name" +// @Consume json +// @Param request body snapshotRepositoryParams true "Parameters" // @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" @@ -200,13 +208,9 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { err error repo *deb.LocalRepo snapshot *deb.Snapshot + b snapshotRepositoryParams ) - var b struct { - Name string `binding:"required"` - Description string - } - if c.Bind(&b) != nil { return } @@ -305,10 +309,11 @@ func apiSnapshotsUpdate(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** +// @Summary Get snapshot information +// @Description **Get information about a snapshot** // @Description To Do // @Tags Snapshots +// @Param name path string true "Name of the snapshot" // @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" diff --git a/docs/Database.md b/docs/Database.md index 8786b3ff..96cde17d 100644 --- a/docs/Database.md +++ b/docs/Database.md @@ -1 +1,4 @@ # Aptly Database Operations +
+ +
diff --git a/docs/Files.md b/docs/Files.md new file mode 100644 index 00000000..b33f80da --- /dev/null +++ b/docs/Files.md @@ -0,0 +1,4 @@ +# File Operations +
+ +
diff --git a/docs/GPG.md b/docs/GPG.md new file mode 100644 index 00000000..c33bf937 --- /dev/null +++ b/docs/GPG.md @@ -0,0 +1,4 @@ +# GPG Keys +
+ +
diff --git a/docs/Graph.md b/docs/Graph.md new file mode 100644 index 00000000..25e2a2df --- /dev/null +++ b/docs/Graph.md @@ -0,0 +1,4 @@ +# Dependency Graph +
+ +
diff --git a/docs/Mirrors.md b/docs/Mirrors.md new file mode 100644 index 00000000..160cf3c4 --- /dev/null +++ b/docs/Mirrors.md @@ -0,0 +1,5 @@ +# Repository Mirrors +
+ +
+ diff --git a/docs/Packages.md b/docs/Packages.md new file mode 100644 index 00000000..694061b4 --- /dev/null +++ b/docs/Packages.md @@ -0,0 +1,5 @@ +# Packages +
+ +
+ diff --git a/docs/Publish.md b/docs/Publish.md index b99a2cc8..44d5fdb1 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -1,10 +1,7 @@ # Aptly Publish Points
-Publish snapshot or local repo as Debian repository which could be -served by HTTP/FTP/rsync server. Repository is signed by user's key with -GnuPG. Key should be created beforehand (see section GPG Keys below). -Published repository could be consumed directly by apt. +Publish snapshot or local repo as Debian repository which could be served by HTTP/FTP/rsync server. Repository is signed by user's key with GnuPG. Key should be created beforehand (see section GPG Keys below). Published repository could be consumed directly by apt. Repositories could be published to Amazon S3 service: create bucket, [configure publishing endpoint](/doc/feature/s3/) and use S3 endpoint when @@ -13,25 +10,17 @@ publishing. #### GPG Keys -GPG key is required to sign any published repository. Key should be -generated before publishing first repository. +GPG key is required to sign any published repository. Key should be generated before publishing first repository. -Key generation, storage, backup and revocation is out of scope of this -document, there are many tutorials available, e.g. [this one](http://fedoraproject.org/wiki/Creating_GPG_Keys). +Key generation, storage, backup and revocation is out of scope of this document, there are many tutorials available, e.g. [this one](http://fedoraproject.org/wiki/Creating_GPG_Keys). -Publiс part of the key should be exported from your keyring using `gpg --export --armor` and -imported into apt keyring using `apt-key` tool on all machines that would be using published -repositories. +Publiс part of the key should be exported from your keyring using `gpg --export --armor` and imported into apt keyring using `apt-key` tool on all machines that would be using published repositories. -Signing releases is highly recommended, but if you want to skip it, you -can either use `gpgDisableSign` configuration option or `--skip-signing` -flag. +Signing releases is highly recommended, but if you want to skip it, you can either use `gpgDisableSign` configuration option or `--skip-signing` flag. #### Parameters -Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`. -`:distribution` is distribution name, while `:prefix` is `[:]` (storage is optional, it defaults -to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores +Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`. `:distribution` is distribution name, while `:prefix` is `[:]` (storage is optional, it defaults to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambigious in URLs.
diff --git a/docs/Repos.md b/docs/Repos.md new file mode 100644 index 00000000..3a756365 --- /dev/null +++ b/docs/Repos.md @@ -0,0 +1,5 @@ +# Repositories +
+ +
+ diff --git a/docs/S3.md b/docs/S3.md new file mode 100644 index 00000000..d5c31088 --- /dev/null +++ b/docs/S3.md @@ -0,0 +1,5 @@ +# S3 Endpoints +
+ +
+ diff --git a/docs/Snapshots.md b/docs/Snapshots.md new file mode 100644 index 00000000..c02b668e --- /dev/null +++ b/docs/Snapshots.md @@ -0,0 +1,9 @@ +# Repository and Mirror Snapshots +
+Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. +Snapshot is immutable, i.e. it can't be changed since it has been created. Snapshots could be [merged](/doc/aptly/snapshot/merge/), +[filtered](/doc/aptly/snapshot/pull/), +individual packages could be [pulled](/doc/aptly/snapshot/pull/), snapshot could be +[verified](/doc/aptly/snapshot/verify/) for missing dependencies. Finally, snapshots could be +[published as repositories](/doc/aptly/publish/snapshot) +
diff --git a/docs/Tasks.md b/docs/Tasks.md new file mode 100644 index 00000000..2dbf8072 --- /dev/null +++ b/docs/Tasks.md @@ -0,0 +1,4 @@ +# Background Tasks +
+ +
diff --git a/docs/api.md b/docs/api.md index 9d236ce2..8618e2d7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,4 +1,3 @@ - Using aptly via REST API allows to achieve two goals: 1. Remote access to aptly service: e.g. uploading packages and publishing them from CI server. @@ -59,54 +58,3 @@ Aptly's HTTP API shouldn't be directly exposed to the Internet: there's no authe 1. GPG key passphrase can't be input on console, so either passwordless GPG keys are required or passphrase should be specified in API parameters. 1. Some script might be required to start/stop aptly HTTP service. 1. Some parameters are given as part of URLs, which requires proper url encoding. Unfortunately, at the moment it's not possible to pass URL arguments with `/` in them. - -### How to implement equivalent of aptly commands using API - -* `aptly mirror`: [mirror API](/doc/api/mirror) - * `list`: list - * `create`: create - * `drop`: delete - * `show`: show - * `search`: show packages/search - * `update`: update mirror -* `aptly repo`: [local repos API](/doc/api/repos) - * `add`: [file upload API](/doc/api/files) + add packages from uploaded file - * `copy`: show packages/search + add packages by key - * `create`: create - * `drop`: delete - * `edit`: edit - * `import`: not available, as mirror API is not implemented yet - * `list`: list - * `move`: show packages/search + add packages by key + delete packages by key - * `remove`: show packages/search + delete packages by key - * `rename`: not available yet, should be part of edit API - * `search`: show packages/search - * `show`: show -* `aptly snapshot`: [snapshots API](/doc/api/snapshots) - * `create`: - * empty: create snapshot with empty package references - * from mirror: not available yet - * from local repo: create snapshot from local repo - * `diff`: snapshot difference API - * `drop`: delete - * `filter`: show packages/search + create snapshot from package references - * `list`: list - * `merge`: show packages/search + processing + create snapshot from package references - * `pull`: show packages/search + processing + create snapshot from package references (might be implemented as API in future versions) - * `rename`: edit - * `search`: show packages/search - * `show`: show - * `verify`: not available yet -* `aptly publish`: [publish API](/doc/api/publish) - * `drop`: delete - * `list`: list - * `repo`: publish repo - * `snapshot`: publish snapshot - * `switch`: switch/update - * `update`: switch/update -* `aptly package`: [packages API](/doc/api/packages) - * `search`: not available yet - * `show`: only one format, with package key as input -* `aptly graph`: [graph API](/doc/api/misc) -* `aptly version`: [version API](/doc/api/misc) -* `aptly db`: not available yet diff --git a/docs/docs.html b/docs/docs.html index cb405d9b..35eb3c9b 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -36,7 +36,7 @@ margin-left: auto !important; margin-right: 2em; font-weight: bold; - font-size: 14pt !important; + font-size: 16pt !important; } /* Tag Group */ @@ -47,9 +47,16 @@ width: 7em; } - /* */ + /* Group Heading */ .swagger-ui .opblock-tag-section .opblock-tag .markdown > h1 { - font-size: 16pt; + font-size: 14pt; + margin-top: 0px; + margin-left: 1em; + } + + /* Group Description */ + .swagger-ui .opblock-tag-section .opblock-tag .markdown > div { + font-size: 12pt; } /* Show Tag Group description only if opened */ diff --git a/docs/swagger.conf.tpl b/docs/swagger.conf.tpl index 670470b9..e846c570 100644 --- a/docs/swagger.conf.tpl +++ b/docs/swagger.conf.tpl @@ -6,11 +6,29 @@ package docs // @contact.name Aptly // @contact.url http://github.com/aptly-dev/aptly +// @Tag.name Repos +// @Tag.description.markdown +// @Tag.name Files +// @Tag.description.markdown +// @Tag.name Packages +// @Tag.description.markdown +// @Tag.name Mirrors +// @Tag.description.markdown +// @Tag.name Snapshots +// @Tag.description.markdown // @Tag.name Publish // @Tag.description.markdown +// @Tag.name Graph +// @Tag.description.markdown +// @Tag.name S3 +// @Tag.description.markdown +// @Tag.name GPG +// @Tag.description.markdown // @Tag.name Database // @Tag.description.markdown // @Tag.name Status // @Tag.description.markdown +// @Tag.name Tasks +// @Tag.description.markdown // version will be appended here: From 2171c05ef8e93cd25d8594fe9d26d02263eb62a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 3 Nov 2024 14:13:57 +0100 Subject: [PATCH 22/64] fix lint --- api/snapshot.go | 8 ++++---- docs/index.go | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index 54433096..7e625ec9 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -187,9 +187,9 @@ func apiSnapshotsCreate(c *gin.Context) { }) } -type snapshotRepositoryParams struct { - Name string `binding:"required"` - Description string +type snapshotRepositoryParams struct { + Name string `binding:"required"` + Description string } // @Summary Snapshot Repository @@ -208,7 +208,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { err error repo *deb.LocalRepo snapshot *deb.Snapshot - b snapshotRepositoryParams + b snapshotRepositoryParams ) if c.Bind(&b) != nil { diff --git a/docs/index.go b/docs/index.go index 8350449a..ca4c914a 100644 --- a/docs/index.go +++ b/docs/index.go @@ -1,8 +1,9 @@ package docs import ( - _ "embed" - _ "github.com/swaggo/swag" // make sure swag is in go.mod + _ "embed" // embed html below + + _ "github.com/swaggo/swag" // make sure swag is in go.mod ) //go:embed docs.html From a5f8ce25039d74b0c31da83cc19bf8ad456fef93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 3 Nov 2024 15:00:20 +0100 Subject: [PATCH 23/64] doc: import chapters from aptly.info --- docs/Database.md | 2 +- docs/Files.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++ docs/Graph.md | 26 +++++++++++++++ docs/Mirrors.md | 2 ++ docs/Packages.md | 2 +- docs/Repos.md | 6 ++++ docs/Snapshots.md | 7 +---- docs/Tasks.md | 4 +++ 8 files changed, 121 insertions(+), 8 deletions(-) diff --git a/docs/Database.md b/docs/Database.md index 96cde17d..804cde5e 100644 --- a/docs/Database.md +++ b/docs/Database.md @@ -1,4 +1,4 @@ # Aptly Database Operations
- +Manage aptly’s internal metadata database and package pool.
diff --git a/docs/Files.md b/docs/Files.md index b33f80da..e2b9c832 100644 --- a/docs/Files.md +++ b/docs/Files.md @@ -1,4 +1,84 @@ # File Operations
+Upload package files temporarily to aptly service. These files +could be added to local repositories using [local repositories API](/doc/api/repos). + +All uploaded files are stored under `/upload` directory (see [configuration](/doc/configuration)). +This directory would be created automatically if it doesn't exist. + +Uploaded files are grouped by directories to support concurrent uploads from multiple +package sources. Local repos add API can operate on directory (adding all files from directory) or +on individual package files. By default, all successfully added package files would be removed. + +### List Directories + +`GET /api/files` + +List all directories. + +Response: list of directory names. + +Example: + + $ curl http://localhost:8080/api/files + ["aptly-0.9"] + +### Upload File(s) + +`POST /api/files/:dir` + +Parameter `:dir` is upload directory name. Directory would be created if it doesn't exist. + +Any number of files can be uploaded in one call, aptly would preserve filenames. No check is performed +if existing uploaded would be overwritten. + +Response: list of uploaded files as `:dir/:file`. + +Example: + + $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9 + ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"] + +### List Files in Directory + +`GET /api/files/:dir` + +Returns list of files in directory. + +Response: list of filenames. + +HTTP Errors: + + Code | Description +----------|------------------------- + 404 | directory doesn't exist + +Example: + + $ curl http://localhost:8080/api/files/aptly-0.9 + ["aptly_0.9~dev+217+ge5d646c_i386.deb"] + + +### Delete Directory + +`DELETE /api/files/:dir` + +Deletes all files in upload directory and directory itself. + +Example: + + $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9 + {} + +### Delete File in Directory + +`DELETE /api/files/:dir/:file` + +Delete single file in directory. + +Example: + + $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb + {}
diff --git a/docs/Graph.md b/docs/Graph.md index 25e2a2df..56a31b2f 100644 --- a/docs/Graph.md +++ b/docs/Graph.md @@ -1,4 +1,30 @@ # Dependency Graph
+aptly generates graph showing dependencies between mirrors, local repositories, +snapshots and published repositories. +  + +Command graph generates graph of dependencies: + +* between snapshots and mirrors (what mirror was used to create each snapshot) +* between snapshots and local repos (what local repo was used to create snapshot) +* between snapshots (pulling, merging, etc.) +* between snapshots, local repos and published repositories (how snapshots were published). + +Graph is rendered to PNG file using graphviz package. + +Usage: + + $ aptly graph + +Flags: + +- `-format="png"`: graph output format, could be anything graphviz supports, e.g. `png`, `pdf`, `svg`, ... +- `-output=""`: specify output filename, default is to open result in viewer +- `-layout="horizontal"`: change between a `horizontal` and a `vertical` graph layout. + +Example: + +Example graph from aptly graph (horizontal)
diff --git a/docs/Mirrors.md b/docs/Mirrors.md index 160cf3c4..a86e6cca 100644 --- a/docs/Mirrors.md +++ b/docs/Mirrors.md @@ -1,5 +1,7 @@ # Repository Mirrors
+aptly allows to create mirrors of remote Debian repositories, HTTP, HTTPS and FTP repositories are supported. +Mirrors are created with [`aptly mirror create`](/doc/aptly/mirror/create/) command, mirror contents are downloaded with [`aptly mirror update`](/doc/aptly/mirror/update/) command. Mirror could be updated at any moment. In order to preserve current mirror state, [create snapshot](/doc/aptly/snapshot/create/) of the mirror. Snapshot could be published or used to build other snapshots.
diff --git a/docs/Packages.md b/docs/Packages.md index 694061b4..77bc5de4 100644 --- a/docs/Packages.md +++ b/docs/Packages.md @@ -1,5 +1,5 @@ # Packages
- +Perform operations on the whole collection of packages in apty database.
diff --git a/docs/Repos.md b/docs/Repos.md index 3a756365..8e07ae54 100644 --- a/docs/Repos.md +++ b/docs/Repos.md @@ -1,5 +1,11 @@ # Repositories
+Local repository is a collection of packages (most usually custom packages created internally). +Packages could be added to local repository at any moment, removed, moved or copied between repos. +Multiple versions of the same package could be added to the repository. +In order to capture current repository state, [create snapshot](/doc/aptly/snapshot/create/). +Local repositories could be published either [directly](/doc/aptly/publish/repo/) +or from [snapshot](/doc/aptly/publish/snapshot).
diff --git a/docs/Snapshots.md b/docs/Snapshots.md index c02b668e..42259a62 100644 --- a/docs/Snapshots.md +++ b/docs/Snapshots.md @@ -1,9 +1,4 @@ # Repository and Mirror Snapshots
-Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. -Snapshot is immutable, i.e. it can't be changed since it has been created. Snapshots could be [merged](/doc/aptly/snapshot/merge/), -[filtered](/doc/aptly/snapshot/pull/), -individual packages could be [pulled](/doc/aptly/snapshot/pull/), snapshot could be -[verified](/doc/aptly/snapshot/verify/) for missing dependencies. Finally, snapshots could be -[published as repositories](/doc/aptly/publish/snapshot) +Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. Snapshot is immutable, i.e. it can’t be changed since it has been created. Snapshots could be merged, filtered, individual packages could be pulled, snapshot could be verified for missing dependencies. Finally, snapshots could be published as repositories.
diff --git a/docs/Tasks.md b/docs/Tasks.md index 2dbf8072..b71ca31a 100644 --- a/docs/Tasks.md +++ b/docs/Tasks.md @@ -1,4 +1,8 @@ # Background Tasks
+An aptly task is a sequence of multiple aptly commands run within a single aptly thread. +For example, a task could be to [create a new repository](/doc/aptly/repo/create), [add packages to it](/doc/aptly/repo/add), [create a snapshot](/doc/aptly/snapshot/create) and [serve it](/doc/aptly/serve). + +Four commands can be now run in a single command.
From 74bc3f5db3379eaceea410ab84ca83f9b2e1e30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 17 Nov 2024 19:37:24 +0100 Subject: [PATCH 24/64] update go.mod and make sure make lint has the VERSION generated --- Makefile | 2 +- go.mod | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f0b945c1..c17e17b6 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ etcd-install: flake8: ## run flake8 on system test python files flake8 system/ -lint: +lint: prepare # Install golangci-lint @test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) # Running lint diff --git a/go.mod b/go.mod index d7bbe7af..91f7e565 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,6 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/etcd/api/v3 v3.5.15 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect From 9fc7ebdac2614999c122b671d5b0dcb4bc5f5f30 Mon Sep 17 00:00:00 2001 From: iofq Date: Sun, 17 Nov 2024 16:20:41 -0600 Subject: [PATCH 25/64] Update repos, task, snapshot api docs --- Makefile | 2 +- api/repos.go | 179 +++++++++++++++++++++++++++++++----------------- api/snapshot.go | 167 +++++++++++++++++++++++++++----------------- api/task.go | 98 +++++++++++++------------- 4 files changed, 270 insertions(+), 176 deletions(-) diff --git a/Makefile b/Makefile index c17e17b6..ff224561 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ azurite-stop: swagger: swagger-install # Generate swagger docs - @PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs --generalInfo docs/swagger.conf + @PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf etcd-install: # Install etcd diff --git a/api/repos.go b/api/repos.go index 413a49e0..fa6c3e17 100644 --- a/api/repos.go +++ b/api/repos.go @@ -18,13 +18,12 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Serve HTML listing of repo +// @Description If ServeInAPIMode is enabled in aptly config, +// @Description this endpoint is enabled which returns an HTML listing of each repo that can be browsed // @Tags Repos -// @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Produce html +// @Success 200 {object} string "HTML" // @Router /api/repos [get] func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc { return func(c *gin.Context) { @@ -42,15 +41,14 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H } } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Serve package in API mode +// @Description If ServeInAPIMode is enabled in aptly config, +// @Description this api serves a specified package from storage // @Tags Repos // @Param storage path string true "Storage" // @Param pkgPath path string true "Package Path" allowReserved=true // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 "" // @Router /api/{storage}/{pkgPath} [get] func reposServeInAPIMode(c *gin.Context) { pkgpath := c.Param("pkgPath") @@ -100,7 +98,8 @@ type repoCreateParams struct { } // @Summary Create repository -// @Description Create a local repository. +// @Description Create a local repository with specified parameters. +// @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot. // @Tags Repos // @Produce json // @Consume json @@ -159,22 +158,24 @@ func apiReposCreate(c *gin.Context) { c.JSON(http.StatusCreated, repo) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type reposEditParams struct { + Name *string + Comment *string + DefaultDistribution *string + DefaultComponent *string +} + +// @Summary Update repo +// @Description **Update local repository meta information** // @Tags Repos // @Produce json -// @Success 200 {object} string "msg" +// @Param request body reposEditParams true "Parameters" +// @Success 200 {object} deb.LocalRepo "msg" // @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/repos/{name} [put] func apiReposEdit(c *gin.Context) { - var b struct { - Name *string - Comment *string - DefaultDistribution *string - DefaultComponent *string - } - + var b reposEditParams if c.Bind(&b) != nil { return } @@ -238,13 +239,17 @@ func apiReposShow(c *gin.Context) { c.JSON(200, repo) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Drop Repository +// @Description Drop/delete a repo +// @Description Cannot drop repos that are published. +// @Description Needs force=1 to drop repos used as source by other repos. // @Tags Repos // @Produce json -// @Success 200 {object} string "msg" +// @Param _async query bool false "Run task in background using tasks API" +// @Param force query int false "force: 1 to enable" +// @Success 200 {object} task.ProcessReturnValue "Repo object" // @Failure 404 {object} Error "Not Found" +// @Failure 404 {object} Error "Repo Conflict" // @Router /api/repos/{name} [delete] func apiReposDrop(c *gin.Context) { force := c.Request.URL.Query().Get("force") == "1" @@ -280,13 +285,19 @@ func apiReposDrop(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary List Repo Packages +// @Description **Return a list of packages present in the repo** +// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q // @Tags Repos // @Produce json +// @Param name path string true "Snapshot to search" +// @Param q query string true "Package query (e.g Name%20(~%20matlab))" +// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query" +// @Param format query string true "Set to 'details' to return extra info about each package" +// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name" // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" +// @Failure 404 {object} Error "Internal Server Error" // @Router /api/repos/{name}/packages [get] func apiReposPackagesShow(c *gin.Context) { collectionFactory := context.NewCollectionFactory() @@ -307,11 +318,13 @@ func apiReposPackagesShow(c *gin.Context) { showPackages(c, repo.RefList(), collectionFactory) } +type reposPackagesAddDeleteParams struct { + PackageRefs []string +} + // Handler for both add and delete func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) { - var b struct { - PackageRefs []string - } + var b reposPackagesAddDeleteParams if c.Bind(&b) != nil { return @@ -368,13 +381,20 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Add Packages by Key +// @Description **Add packages to local repository by package keys.** +// @Description +// @Description Any package can be added that is present in the aptly database (from any mirror, snapshot, local repository). This API combined with package list (search) APIs allows one to implement importing, copying, moving packages around. +// @Description +// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository. // @Tags Repos // @Produce json +// @Param request body reposPackagesAddDeleteParams true "Parameters" +// @Param _async query bool false "Run task in background using tasks API" // @Success 200 {object} string "msg" +// @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" +// @Failure 400 {object} Error "Internal Server Error" // @Router /api/repos/{name}/packages [post] func apiReposPackagesAdd(c *gin.Context) { apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error { @@ -383,13 +403,18 @@ func apiReposPackagesAdd(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Delete Packages by Key +// @Description **Remove packages from local repository by package keys.** +// @Description +// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages. // @Tags Repos // @Produce json +// @Param request body reposPackagesAddDeleteParams true "Parameters" +// @Param _async query bool false "Run task in background using tasks API" // @Success 200 {object} string "msg" +// @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" +// @Failure 400 {object} Error "Internal Server Error" // @Router /api/repos/{name}/packages [delete] func apiReposPackagesDelete(c *gin.Context) { apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error { @@ -399,23 +424,27 @@ func apiReposPackagesDelete(c *gin.Context) { }) } -// @Summary Add packages -// @Description **Add package file** -// @Description To Do +// @Summary Add packages from uploaded file +// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically. +// @Description Adding same package to local repository is not an error. +// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import). // @Tags Repos // @Param name path string true "Repository name" // @Param dir path string true "Directory of packages" // @Param file path string false "Filename (optional)" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 {string} string "OK" +// @Failure 400 {object} Error "wrong file" +// @Failure 404 {object} Error "Repository not found" +// @Failure 500 {object} Error "Error adding files" // @Router /api/repos/{name}/file/{dir}/{file} [post] func apiReposPackageFromFile(c *gin.Context) { // redirect all work to dir method apiReposPackageFromDir(c) } -// @Summary Add packages from uploaded file/directory +// @Summary Add packages from uploaded directory // @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically. // @Description Adding same package to local repository is not an error. // @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import). @@ -425,6 +454,7 @@ func apiReposPackageFromFile(c *gin.Context) { // @Consume json // @Param noRemove query string false "when value is set to 1, don’t remove any files" // @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json // @Success 200 {string} string "OK" // @Failure 400 {object} Error "wrong file" @@ -549,23 +579,31 @@ func apiReposPackageFromDir(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type reposCopyPackageParams struct { + WithDeps bool `json:"with-deps,omitempty"` + DryRun bool `json:"dry-run,omitempty"` +} + +// @Summary Copy Package +// @Description Copies a package from a source to destination repository // @Tags Repos // @Produce json -// @Success 200 {object} string "msg" +// @Param name path string true "Source repo" +// @Param src path string true "Destination repo" +// @Param file path string true "File/packages to copy" +// @Param _async query bool false "Run task in background using tasks API" +// @Success 200 {object} task.ProcessReturnValue "msg" +// @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" +// @Failure 422 {object} Error "Unprocessable Entity" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/repos/{name}/copy/{src}/{file} [post] func apiReposCopyPackage(c *gin.Context) { dstRepoName := c.Params.ByName("name") srcRepoName := c.Params.ByName("src") fileName := c.Params.ByName("file") - jsonBody := struct { - WithDeps bool `json:"with-deps,omitempty"` - DryRun bool `json:"dry-run,omitempty"` - }{ + jsonBody := reposCopyPackageParams{ WithDeps: false, DryRun: false, } @@ -619,7 +657,6 @@ func apiReposCopyPackage(c *gin.Context) { dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err) - } srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress()) @@ -695,11 +732,15 @@ func apiReposCopyPackage(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Include Packages from File Upload +// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos // @Produce json +// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" +// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" +// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" +// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" +// @Param _async query bool false "Run task in background using tasks API" // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir}/{file} [post] @@ -708,12 +749,27 @@ func apiReposIncludePackageFromFile(c *gin.Context) { apiReposIncludePackageFromDir(c) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type reposIncludePackageFromDirReport struct { + Warnings []string + Added []string + Deleted []string +} + +type reposIncludePackageFromDirResponse struct { + Report reposIncludePackageFromDirReport + FailedFiles []string +} + +// @Summary Include Packages from Dir Upload +// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos // @Produce json -// @Success 200 {object} string "msg" +// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" +// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" +// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" +// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" +// @Param _async query bool false "Run task in background using tasks API" +// @Success 200 {object} reposIncludePackageFromDirResponse "Response" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir} [post] func apiReposIncludePackageFromDir(c *gin.Context) { @@ -817,6 +873,5 @@ func apiReposIncludePackageFromDir(c *gin.Context) { "Report": reporter, "FailedFiles": failedFiles, }}, nil - }) } diff --git a/api/snapshot.go b/api/snapshot.go index 7e625ec9..ba94b191 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -15,7 +15,8 @@ import ( ) // @Summary Get snapshots -// @Description Get list of available snapshots. Each snapshot is returned as in “show” API. +// @Description **Get list of available snapshots** +// @Description Each snapshot is returned as in “show” API. // @Tags Snapshots // @Produce json // @Success 200 {array} deb.Snapshot @@ -39,26 +40,32 @@ func apiSnapshotsList(c *gin.Context) { c.JSON(200, result) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type snapshotsCreateFromMirrorParams struct { + Name string `binding:"required"` + Description string +} + +// @Summary Create snapshot from mirror +// @Description **Create snapshot of a mirror from given name** // @Tags Snapshots // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param request body snapshotsCreateFromMirrorParams true "Parameters" +// @Param name path string true "Mirror name" +// @Param _async query bool false "Run task in background using tasks API" +// @Success 201 {object} deb.Snapshot "Created Snapshot" +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Mirror Not Found" +// @Failure 409 {object} Error "Conflicting snapshot" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/mirrors/{name}/snapshots [post] func apiSnapshotsCreateFromMirror(c *gin.Context) { var ( err error repo *deb.RemoteRepo snapshot *deb.Snapshot + b snapshotsCreateFromMirrorParams ) - var b struct { - Name string `binding:"required"` - Description string - } - if c.Bind(&b) != nil { return } @@ -105,27 +112,32 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type snapshotsCreateParams struct { + Name string `binding:"required"` + Description string + SourceSnapshots []string + PackageRefs []string +} + +// @Summary Create snapshot from repo +// @Description **Create a snapshot from package refs** +// @Description Refs can be obtained from snapshots, local repos, or mirrors // @Tags Snapshots +// @Param request body snapshotsCreateParams true "Parameters" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 201 {object} deb.Snapshot "Created snapshot" +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Source snapshot or package refs not found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots [post] func apiSnapshotsCreate(c *gin.Context) { var ( err error snapshot *deb.Snapshot + b snapshotsCreateParams ) - var b struct { - Name string `binding:"required"` - Description string - SourceSnapshots []string - PackageRefs []string - } - if c.Bind(&b) != nil { return } @@ -187,28 +199,31 @@ func apiSnapshotsCreate(c *gin.Context) { }) } -type snapshotRepositoryParams struct { +type snapshotsCreateFromRepositoryParams struct { Name string `binding:"required"` Description string } // @Summary Snapshot Repository -// @Description **Create a sbalshot of a repository** -// @Description To Do +// @Description **Create a snapshot of a repository by name** // @Tags Snapshots // @Param name path string true "Repository name" // @Consume json -// @Param request body snapshotRepositoryParams true "Parameters" +// @Param request body snapshotsCreateFromRepositoryParams true "Parameters" +// @Param name path string true "Name of the snapshot" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 201 {object} deb.Snapshot "Created snapshot object" +// @Failure 400 {object} Error "Bad Request" +// @Failure 500 {object} Error "Internal Server Error" +// @Failure 404 {object} Error "Repo Not Found" // @Router /api/repos/{name}/snapshots [post] func apiSnapshotsCreateFromRepository(c *gin.Context) { var ( err error repo *deb.LocalRepo snapshot *deb.Snapshot - b snapshotRepositoryParams + b snapshotsCreateFromRepositoryParams ) if c.Bind(&b) != nil { @@ -252,25 +267,30 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type snapshotsUpdateParams struct { + Name string + Description string +} + +// @Summary Update Snapshot +// @Description **Update snapshot metadata (Name, Description)** // @Tags Snapshots +// @Param request body snapshotsUpdateParams true "Parameters" +// @Param name path string true "Snapshot name" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 {object} deb.Snapshot "Updated snapshot object" +// @Failure 404 {object} Error "Snapshot Not Found" +// @Failure 409 {object} Error "Conflicting snapshot" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots/{name} [put] func apiSnapshotsUpdate(c *gin.Context) { var ( err error snapshot *deb.Snapshot + b snapshotsUpdateParams ) - var b struct { - Name string - Description string - } - if c.Bind(&b) != nil { return } @@ -310,13 +330,13 @@ func apiSnapshotsUpdate(c *gin.Context) { } // @Summary Get snapshot information -// @Description **Get information about a snapshot** -// @Description To Do +// @Description **Query detailed information about a snapshot by name** // @Tags Snapshots // @Param name path string true "Name of the snapshot" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 {object} deb.Snapshot "msg" +// @Failure 404 {object} Error "Snapshot Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots/{name} [get] func apiSnapshotsShow(c *gin.Context) { collectionFactory := context.NewCollectionFactory() @@ -337,13 +357,19 @@ func apiSnapshotsShow(c *gin.Context) { c.JSON(200, snapshot) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Drop Snapshot +// @Description **Drop/delete snapshot by name** +// @Description Cannot drop snapshots that are published. +// @Description Needs force=1 to drop snapshots used as source by other snapshots. // @Tags Snapshots +// @Param name path string true "Snapshot name" +// @Param force query string false "Force operation" +// @Param _async query bool false "Run task in background using tasks API" // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 "" +// @Failure 404 {object} Error "Snapshot Not Found" +// @Failure 409 {object} Error "Snapshot in use" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots/{name} [delete] func apiSnapshotsDrop(c *gin.Context) { name := c.Params.ByName("name") @@ -383,13 +409,18 @@ func apiSnapshotsDrop(c *gin.Context) { }) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Snapshot diff +// @Description **Return the diff between two snapshots (name & withSnapshot)** +// @Description Provide `onlyMatching=1` to return only packages present in both snapshots. +// @Description Otherwise, returns a `left` and `right` result providing packages only in the first and second snapshots // @Tags Snapshots // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param name path string true "Snapshot name" +// @Param withSnapshot path string true "Snapshot name to diff against" +// @Param onlyMatching query string false "Only return packages present in both snapshots" +// @Success 200 {array} deb.PackageDiff "Package Diff" +// @Failure 404 {object} Error "Snapshot Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots/{name}/diff/{withSnapshot} [get] func apiSnapshotsDiff(c *gin.Context) { onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1" @@ -441,13 +472,19 @@ func apiSnapshotsDiff(c *gin.Context) { c.JSON(200, result) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary List Snapshot Packages +// @Description **List all packages in snapshot or perform search on snapshot contents and return results** +// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q // @Tags Snapshots // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param name path string true "Snapshot to search" +// @Param q query string false "Package query (e.g Name%20(~%20matlab))" +// @Param withDeps query string false "Set to 1 to include dependencies when evaluating package query" +// @Param format query string false "Set to 'details' to return extra info about each package" +// @Param maximumVersion query string false "Set to 1 to only return the highest version for each package name" +// @Success 200 {array} string "Package info" +// @Failure 404 {object} Error "Snapshot Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/snapshots/{name}/packages [get] func apiSnapshotsSearchPackages(c *gin.Context) { collectionFactory := context.NewCollectionFactory() @@ -480,13 +517,14 @@ type snapshotsMergeParams struct { // @Description // @Description If only one snapshot is specified, merge copies source into destination. // @Tags Snapshots +// @Consume json +// @Produce json // @Param name path string true "Name of the snapshot to be created" // @Param latest query int false "merge only the latest version of each package" // @Param no-remove query int false "all versions of packages are preserved during merge" -// @Consume json // @Param request body snapshotsMergeParams true "Parameters" -// @Produce json -// @Success 200 +// @Param _async query bool false "Run task in background using tasks API" +// @Success 201 {object} deb.Snapshot "Resulting snapshot object" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" // @Failure 500 {object} Error "Internal Error" @@ -591,15 +629,16 @@ type snapshotsPullParams struct { // @Description // @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled. // @Tags Snapshots +// @Param request body snapshotsPullParams true "Parameters" // @Param name path string true "Name of the snapshot to be created" // @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable" // @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable" // @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable" // @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable" +// @Param _async query bool false "Run task in background using tasks API" // @Consume json -// @Param request body snapshotsPullParams true "Parameters" // @Produce json -// @Success 200 +// @Success 200 {object} deb.Snapshot "Resulting Snapshot object" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" // @Failure 500 {object} Error "Internal Error" diff --git a/api/task.go b/api/task.go index a33510aa..d9ea737b 100644 --- a/api/task.go +++ b/api/task.go @@ -10,9 +10,9 @@ import ( ) // @Summary Get tasks -// @Description Get list of available tasks. Each task is returned as in “show” API. +// @Description **Get list of available tasks. Each task is returned as in “show” API** // @Tags Tasks -// @Produce json +// @Produce json // @Success 200 {array} task.Task // @Router /api/tasks [get] func apiTasksList(c *gin.Context) { @@ -20,12 +20,11 @@ func apiTasksList(c *gin.Context) { c.JSON(200, list.GetTasks()) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Clear finished and failed tasks +// @Description **Removes finished and failed tasks from internal task list** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" +// @Success 200 "" // @Router /api/tasks-clear [post] func apiTasksClear(c *gin.Context) { list := context.TaskList() @@ -33,13 +32,11 @@ func apiTasksClear(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Wait for task completion +// @Description **Waits for and returns when all running tasks are complete** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Success 200 "" // @Router /api/tasks-wait [get] func apiTasksWait(c *gin.Context) { list := context.TaskList() @@ -47,13 +44,14 @@ func apiTasksWait(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Wait for task to process +// @Description **Waits for and returns when given Task ID is complete** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param id path int true "Task ID" +// @Success 200 {object} task.Task +// @Failure 500 {object} Error "invalid syntax, bad id?" +// @Failure 400 {object} Error "Task Not Found" // @Router /api/tasks/{id}/wait [get] func apiTasksWaitForTaskByID(c *gin.Context) { list := context.TaskList() @@ -72,13 +70,14 @@ func apiTasksWaitForTaskByID(c *gin.Context) { c.JSON(200, task) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Return task information +// @Description **Return task information for a given ID** // @Tags Tasks -// @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Produce plain +// @Param id path int true "Task ID" +// @Success 200 {object} task.Task +// @Failure 500 {object} Error "invalid syntax, bad id?" +// @Failure 404 {object} Error "Task Not Found" // @Router /api/tasks/{id} [get] func apiTasksShow(c *gin.Context) { list := context.TaskList() @@ -98,13 +97,14 @@ func apiTasksShow(c *gin.Context) { c.JSON(200, task) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Return task output +// @Description **Return task output for a given ID** // @Tags Tasks -// @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Produce plain +// @Param id path int true "Task ID" +// @Success 200 {object} string "Task output" +// @Failure 500 {object} Error "invalid syntax, bad ID?" +// @Failure 404 {object} Error "Task Not Found" // @Router /api/tasks/{id}/output [get] func apiTasksOutputShow(c *gin.Context) { list := context.TaskList() @@ -124,13 +124,14 @@ func apiTasksOutputShow(c *gin.Context) { c.JSON(200, output) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Return task detail +// @Description **Return task detail for a given ID** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param id path int true "Task ID" +// @Success 200 {object} string "Task detail" +// @Failure 500 {object} Error "invalid syntax, bad ID?" +// @Failure 404 {object} Error "Task Not Found" // @Router /api/tasks/{id}/detail [get] func apiTasksDetailShow(c *gin.Context) { list := context.TaskList() @@ -150,12 +151,13 @@ func apiTasksDetailShow(c *gin.Context) { c.JSON(200, detail) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Return task return value (status code) +// @Description **Return task return value (status code) by given ID** // @Tags Tasks -// @Produce json +// @Produce plain +// @Param id path int true "Task ID" // @Success 200 {object} string "msg" +// @Failure 500 {object} Error "invalid syntax, bad ID?" // @Failure 404 {object} Error "Not Found" // @Router /api/tasks/{id}/return_value [get] func apiTasksReturnValueShow(c *gin.Context) { @@ -175,13 +177,14 @@ func apiTasksReturnValueShow(c *gin.Context) { c.JSON(200, output) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Delete task +// @Description **Delete completed task by given ID. Does not stop task execution** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param id path int true "Task ID" +// @Success 200 {object} task.Task +// @Failure 500 {object} Error "invalid syntax, bad ID?" +// @Failure 400 {object} Error "Task in progress or not found" // @Router /api/tasks/{id} [delete] func apiTasksDelete(c *gin.Context) { list := context.TaskList() @@ -201,15 +204,12 @@ func apiTasksDelete(c *gin.Context) { c.JSON(200, delTask) } -// FIXME: used for testing only, remove: - -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Dummy endpoint used for testing. +// @Description **Dummy endpoint used for testing** // @Tags Tasks // @Produce json -// @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Param _async query bool false "Run task in background using tasks API" +// @Success 200 {object} task.ProcessReturnValue // @Router /api/tasks-dummy [post] func apiTasksDummy(c *gin.Context) { resources := []string{"dummy"} From 398fec13b02d40a6da968631eab3ec10792ba421 Mon Sep 17 00:00:00 2001 From: iofq Date: Mon, 18 Nov 2024 20:13:59 -0600 Subject: [PATCH 26/64] Update packages api docs --- api/packages.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api/packages.go b/api/packages.go index e649a80e..f1b6919a 100644 --- a/api/packages.go +++ b/api/packages.go @@ -4,12 +4,13 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Show packages +// @Description **Show information about package by package key** +// @Description Package keys could be obtained from various GET .../packages APIs. // @Tags Packages // @Produce json -// @Success 200 {object} string "msg" +// @Param key path string true "package key (unique package identifier)" +// @Success 200 {object} deb.Package "OK" // @Failure 404 {object} Error "Not Found" // @Router /api/packages/{key} [get] func apiPackagesShow(c *gin.Context) { From 68fe2bc852e641b45eceda3f8633dc0d42623d82 Mon Sep 17 00:00:00 2001 From: iofq Date: Mon, 18 Nov 2024 20:24:15 -0600 Subject: [PATCH 27/64] Update gpg, graph api docs --- api/gpg.go | 23 ++++++++++++----------- api/graph.go | 11 +++++++---- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/api/gpg.go b/api/gpg.go index 835bbb71..10554660 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -12,22 +12,23 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary TODO -// @Description **ToDo** -// @Description To Do +type gpgAddKeyParams struct { + Keyserver string + GpgKeyID string + GpgKeyArmor string + Keyring string +} + +// @Summary Add GPG Key +// @Description Adds a GPG key to aptly keyring // @Tags GPG // @Produce json -// @Success 200 {object} string "msg" +// @Success 200 {object} string "OK" +// @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" // @Router /api/gpg [post] func apiGPGAddKey(c *gin.Context) { - var b struct { - Keyserver string - GpgKeyID string - GpgKeyArmor string - Keyring string - } - + b := gpgAddKeyParams{} if c.Bind(&b) != nil { return } diff --git a/api/graph.go b/api/graph.go index cd04063b..c23b5f02 100644 --- a/api/graph.go +++ b/api/graph.go @@ -12,13 +12,16 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Graph Output +// @Description **Generate graph of aptly objects (same as in aptly graph command).** +// @Description e.g open url `http://localhost:8080/api/graph.svg?layout=vertical` in browser // @Tags Graph // @Produce json -// @Success 200 {object} string "msg" +// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg." +// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout." +// @Success 200 {object} []byte "Output" // @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/graph [get] // GET /api/graph.:ext?layout=[vertical|horizontal(default)] func apiGraph(c *gin.Context) { From 39921809eeba56165c10e211f69cac10f5b01048 Mon Sep 17 00:00:00 2001 From: iofq Date: Mon, 18 Nov 2024 20:30:19 -0600 Subject: [PATCH 28/64] Update db api docs --- api/db.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/db.go b/api/db.go index 5f7727ff..f6d0e5a8 100644 --- a/api/db.go +++ b/api/db.go @@ -11,16 +11,17 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary DB Cleanup +// @Description **Cleanup Aptly DB** +// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore. +// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos. // @Tags Database // @Produce json -// @Success 200 {object} string "msg" +// @Param _async query bool false "Run task in background using tasks API" +// @Success 200 {object} string "Output" // @Failure 404 {object} Error "Not Found" // @Router /api/db/cleanup [post] func apiDbCleanup(c *gin.Context) { - resources := []string{string(task.AllResourcesKey)} maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { var err error From d5571c41c7c2ad7eac124ab9eb1ba94431b19d41 Mon Sep 17 00:00:00 2001 From: iofq Date: Mon, 18 Nov 2024 23:21:23 -0600 Subject: [PATCH 29/64] Update files api docs --- api/files.go | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/api/files.go b/api/files.go index 9adf1d4b..6b2ddba5 100644 --- a/api/files.go +++ b/api/files.go @@ -69,14 +69,16 @@ func apiFilesListDirs(c *gin.Context) { // @Summary Upload file glaa // @Description **Upload a file to a directory** -// @Description To Do +// @Description Any number of files can be uploaded in one call; aptly preserves filenames. No check is performed if existing uploaded file would be overwritten. // @Tags Files // @Accept multipart/form-data -// @Param dir path string true "Directory to upload files to" +// @Param dir path string true "Directory to upload files to. Created if does not exist" // @Param files formData file true "Files to upload" // @Produce json -// @Success 200 {object} string "msg" +// @Success 200 {array} string "list of uploaded files" +// @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/files/{dir} [post] func apiFilesUpload(c *gin.Context) { if !verifyDir(c) { @@ -128,16 +130,16 @@ func apiFilesUpload(c *gin.Context) { apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc() c.JSON(200, stored) - } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary List files in directory +// @Description **Returns list of files in directory** // @Tags Files // @Produce json -// @Success 200 {object} string "msg" +// @Param dir path string true "Directory to list" +// @Success 200 {array} string "Files found in directory" // @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/files/{dir} [get] func apiFilesListFiles(c *gin.Context) { if !verifyDir(c) { @@ -176,13 +178,13 @@ func apiFilesListFiles(c *gin.Context) { c.JSON(200, list) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Delete Directory +// @Description **Deletes all files in upload directory and directory itself** // @Tags Files // @Produce json +// @Param dir path string true "Directory" // @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/files/{dir} [delete] func apiFilesDeleteDir(c *gin.Context) { if !verifyDir(c) { @@ -198,13 +200,15 @@ func apiFilesDeleteDir(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary TODO -// @Description **ToDo** -// @Description To Do +// @Summary Delete File in Directory +// @Description **Delete single file in directory** // @Tags Files // @Produce json +// @Param dir path string true "Directory to delete from" +// @Param name path string true "File to delete" // @Success 200 {object} string "msg" -// @Failure 404 {object} Error "Not Found" +// @Failure 400 {object} Error "Bad Request" +// @Failure 500 {object} Error "Internal Server Error" // @Router /api/files/{dir}/{name} [delete] func apiFilesDeleteFile(c *gin.Context) { if !verifyDir(c) { From 397362bb1a5e0023f37df30cbe0685148d947215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 21 Nov 2024 12:53:40 +0100 Subject: [PATCH 30/64] fix swagger build --- api/packages.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/packages.go b/api/packages.go index f1b6919a..32c086f6 100644 --- a/api/packages.go +++ b/api/packages.go @@ -2,6 +2,7 @@ package api import ( "github.com/gin-gonic/gin" + _ "github.com/aptly-dev/aptly/deb" ) // @Summary Show packages From 83f7c869f0db7992f7b7940c2ab3bacc4153d592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 12:03:42 +0100 Subject: [PATCH 31/64] doc: improve cmd usage arguments --- cmd/repo_add.go | 2 +- cmd/repo_include.go | 2 +- cmd/snapshot_create.go | 2 +- cmd/task_run.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/repo_add.go b/cmd/repo_add.go index 8189e783..e7ccfac7 100644 --- a/cmd/repo_add.go +++ b/cmd/repo_add.go @@ -91,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error { func makeCmdRepoAdd() *commander.Command { cmd := &commander.Command{ Run: aptlyRepoAdd, - UsageLine: "add | ...", + UsageLine: "add (|)...", Short: "add packages to local repository", Long: ` Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files. diff --git a/cmd/repo_include.go b/cmd/repo_include.go index 72e24c6d..cba3fb30 100644 --- a/cmd/repo_include.go +++ b/cmd/repo_include.go @@ -86,7 +86,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error { func makeCmdRepoInclude() *commander.Command { cmd := &commander.Command{ Run: aptlyRepoInclude, - UsageLine: "include | ...", + UsageLine: "include (|)...", Short: "add packages to local repositories based on .changes files", Long: ` Command include looks for .changes files in list of arguments or specified directories. Each diff --git a/cmd/snapshot_create.go b/cmd/snapshot_create.go index 000a78d9..74e3c966 100644 --- a/cmd/snapshot_create.go +++ b/cmd/snapshot_create.go @@ -84,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error { func makeCmdSnapshotCreate() *commander.Command { cmd := &commander.Command{ Run: aptlySnapshotCreate, - UsageLine: "create from mirror | from repo | empty", + UsageLine: "create (from mirror | from repo | empty)", Short: "creates snapshot of mirror (local repository) contents", Long: ` Command create from mirror makes persistent immutable snapshot of remote diff --git a/cmd/task_run.go b/cmd/task_run.go index ff0e1dd6..50519fe8 100644 --- a/cmd/task_run.go +++ b/cmd/task_run.go @@ -131,7 +131,7 @@ func formatCommands(args []string) [][]string { func makeCmdTaskRun() *commander.Command { cmd := &commander.Command{ Run: aptlyTaskRun, - UsageLine: "run -filename= | , , ...", + UsageLine: "run (-filename= | ...)", Short: "run aptly tasks", Long: ` Command helps organise multiple aptly commands in one single aptly task, running as single thread. From 4f229a5bcf996effc8d4f6a7804c886eb6bd4af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 13:09:41 +0100 Subject: [PATCH 32/64] update doc --- api/gpg.go | 8 +++++--- api/graph.go | 19 ++++++++++++++----- api/mirror.go | 6 +++--- api/s3.go | 6 ++++-- api/storage.go | 2 ++ docs/Database.md | 2 +- docs/Files.md | 2 +- docs/GPG.md | 4 ---- docs/Graph.md | 30 ------------------------------ docs/Mirrors.md | 7 ++++--- docs/Packages.md | 2 +- docs/Publish.md | 2 +- docs/Repos.md | 2 +- docs/S3.md | 5 ----- docs/Snapshots.md | 2 +- docs/Status.md | 7 +------ docs/docs.html | 6 +++++- docs/swagger.conf.tpl | 12 +++--------- 18 files changed, 47 insertions(+), 77 deletions(-) delete mode 100644 docs/GPG.md delete mode 100644 docs/Graph.md delete mode 100644 docs/S3.md diff --git a/api/gpg.go b/api/gpg.go index 10554660..578ed18f 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -19,9 +19,11 @@ type gpgAddKeyParams struct { Keyring string } -// @Summary Add GPG Key -// @Description Adds a GPG key to aptly keyring -// @Tags GPG +// @Summary Add GPG Keys +// @Description **Adds GPG keys to aptly keyring** +// @Description +// @Description Add GPG public keys for veryfing remote repositories for mirroring. +// @Tags Mirrors // @Produce json // @Success 200 {object} string "OK" // @Failure 400 {object} Error "Bad Request" diff --git a/api/graph.go b/api/graph.go index c23b5f02..efeb8384 100644 --- a/api/graph.go +++ b/api/graph.go @@ -13,17 +13,26 @@ import ( ) // @Summary Graph Output -// @Description **Generate graph of aptly objects (same as in aptly graph command).** -// @Description e.g open url `http://localhost:8080/api/graph.svg?layout=vertical` in browser -// @Tags Graph +// @Description **Generate dependency graph** +// @Description +// @Description Command graph generates graph of dependencies: +// @Description +// @Description * between snapshots and mirrors (what mirror was used to create each snapshot) +// @Description * between snapshots and local repos (what local repo was used to create snapshot) +// @Description * between snapshots (pulling, merging, etc.) +// @Description * between snapshots, local repos and published repositories (how snapshots were published). +// @Description +// @Description Graph is rendered to PNG file using graphviz package. +// @Description +// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical` +// @Tags Status // @Produce json // @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg." // @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout." // @Success 200 {object} []byte "Output" // @Failure 404 {object} Error "Not Found" // @Failure 500 {object} Error "Internal Server Error" -// @Router /api/graph [get] -// GET /api/graph.:ext?layout=[vertical|horizontal(default)] +// @Router /api/graph.{ext} [get] func apiGraph(c *gin.Context) { var ( err error diff --git a/api/mirror.go b/api/mirror.go index 0052f2a5..952c67d3 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -31,7 +31,7 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) { return verifier, nil } -// @Summary Get mirrors +// @Summary Get Mirrors // @Description **Show list of currently available mirrors** // @Description Each mirror is returned as in “show” API. // @Tags Mirrors @@ -82,8 +82,8 @@ type mirrorCreateParams struct { IgnoreSignatures bool ` json:"IgnoreSignatures"` } -// @Summary Create mirror -// @Description **Create a mirror** +// @Summary Create Mirror +// @Description **Create a mirror of a remote repositoru** // @Tags Mirrors // @Consume json // @Param request body mirrorCreateParams true "Parameters" diff --git a/api/s3.go b/api/s3.go index 19be7537..f38b0847 100644 --- a/api/s3.go +++ b/api/s3.go @@ -6,8 +6,10 @@ import ( // @Summary S3 buckets // @Description **Get list of S3 buckets** -// @Tags S3 -// @Produce json +// @Description +// @Description List configured S3 buckets. +// @Tags Status +// @Produce json // @Success 200 {array} string "List of S3 buckets" // @Router /api/s3 [get] func apiS3List(c *gin.Context) { diff --git a/api/storage.go b/api/storage.go index 4d306b0b..c4f478b8 100644 --- a/api/storage.go +++ b/api/storage.go @@ -18,6 +18,8 @@ type diskFree struct { // @Summary Get Storage Utilization // @Description **Get disk free information of aptly storage** +// @Description +// @Description Units in MiB. // @Tags Status // @Produce json // @Success 200 {object} diskFree "Storage information" diff --git a/docs/Database.md b/docs/Database.md index 804cde5e..60e55ef0 100644 --- a/docs/Database.md +++ b/docs/Database.md @@ -1,4 +1,4 @@ -# Aptly Database Operations +# Maintenance Operations
Manage aptly’s internal metadata database and package pool.
diff --git a/docs/Files.md b/docs/Files.md index e2b9c832..84ed30a3 100644 --- a/docs/Files.md +++ b/docs/Files.md @@ -1,4 +1,4 @@ -# File Operations +# Upload Package Files
Upload package files temporarily to aptly service. These files could be added to local repositories using [local repositories API](/doc/api/repos). diff --git a/docs/GPG.md b/docs/GPG.md deleted file mode 100644 index c33bf937..00000000 --- a/docs/GPG.md +++ /dev/null @@ -1,4 +0,0 @@ -# GPG Keys -
- -
diff --git a/docs/Graph.md b/docs/Graph.md deleted file mode 100644 index 56a31b2f..00000000 --- a/docs/Graph.md +++ /dev/null @@ -1,30 +0,0 @@ -# Dependency Graph -
-aptly generates graph showing dependencies between mirrors, local repositories, -snapshots and published repositories. -  - -Command graph generates graph of dependencies: - -* between snapshots and mirrors (what mirror was used to create each snapshot) -* between snapshots and local repos (what local repo was used to create snapshot) -* between snapshots (pulling, merging, etc.) -* between snapshots, local repos and published repositories (how snapshots were published). - -Graph is rendered to PNG file using graphviz package. - -Usage: - - $ aptly graph - -Flags: - -- `-format="png"`: graph output format, could be anything graphviz supports, e.g. `png`, `pdf`, `svg`, ... -- `-output=""`: specify output filename, default is to open result in viewer -- `-layout="horizontal"`: change between a `horizontal` and a `vertical` graph layout. - -Example: - -Example graph from aptly graph (horizontal) - -
diff --git a/docs/Mirrors.md b/docs/Mirrors.md index a86e6cca..69aef80a 100644 --- a/docs/Mirrors.md +++ b/docs/Mirrors.md @@ -1,7 +1,8 @@ -# Repository Mirrors +# Remote Repository Mirrors
-aptly allows to create mirrors of remote Debian repositories, HTTP, HTTPS and FTP repositories are supported. +Manage mirrors of remote Debian repositories. + +HTTP, HTTPS and FTP repositories are supported. -Mirrors are created with [`aptly mirror create`](/doc/aptly/mirror/create/) command, mirror contents are downloaded with [`aptly mirror update`](/doc/aptly/mirror/update/) command. Mirror could be updated at any moment. In order to preserve current mirror state, [create snapshot](/doc/aptly/snapshot/create/) of the mirror. Snapshot could be published or used to build other snapshots.
diff --git a/docs/Packages.md b/docs/Packages.md index 77bc5de4..8c3731d4 100644 --- a/docs/Packages.md +++ b/docs/Packages.md @@ -1,4 +1,4 @@ -# Packages +# Search Packages
Perform operations on the whole collection of packages in apty database.
diff --git a/docs/Publish.md b/docs/Publish.md index 44d5fdb1..2d1f2af0 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -1,4 +1,4 @@ -# Aptly Publish Points +# Publish repositories and mirrors
Publish snapshot or local repo as Debian repository which could be served by HTTP/FTP/rsync server. Repository is signed by user's key with GnuPG. Key should be created beforehand (see section GPG Keys below). Published repository could be consumed directly by apt. diff --git a/docs/Repos.md b/docs/Repos.md index 8e07ae54..aeffda24 100644 --- a/docs/Repos.md +++ b/docs/Repos.md @@ -1,4 +1,4 @@ -# Repositories +# Manage Local Repositories
Local repository is a collection of packages (most usually custom packages created internally). Packages could be added to local repository at any moment, removed, moved or copied between repos. diff --git a/docs/S3.md b/docs/S3.md deleted file mode 100644 index d5c31088..00000000 --- a/docs/S3.md +++ /dev/null @@ -1,5 +0,0 @@ -# S3 Endpoints -
- -
- diff --git a/docs/Snapshots.md b/docs/Snapshots.md index 42259a62..882c64c0 100644 --- a/docs/Snapshots.md +++ b/docs/Snapshots.md @@ -1,4 +1,4 @@ -# Repository and Mirror Snapshots +# Manage repository and mirror Snapshots
Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. Snapshot is immutable, i.e. it can’t be changed since it has been created. Snapshots could be merged, filtered, individual packages could be pulled, snapshot could be verified for missing dependencies. Finally, snapshots could be published as repositories.
diff --git a/docs/Status.md b/docs/Status.md index e19de2db..5ccf8e14 100644 --- a/docs/Status.md +++ b/docs/Status.md @@ -1,10 +1,5 @@ # Aptly Status Information
- -## Something - -Very interesting ... asd wer - - +Various status information.
diff --git a/docs/docs.html b/docs/docs.html index 35eb3c9b..e0f4eda0 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -39,6 +39,11 @@ font-size: 16pt !important; } + /* Group Spacing */ + .swagger-ui .opblock-tag-section { + padding-top: 40px !important; + } + /* Tag Group */ .swagger-ui .opblock-tag-section > .opblock-tag > a { color: grey !important; @@ -51,7 +56,6 @@ .swagger-ui .opblock-tag-section .opblock-tag .markdown > h1 { font-size: 14pt; margin-top: 0px; - margin-left: 1em; } /* Group Description */ diff --git a/docs/swagger.conf.tpl b/docs/swagger.conf.tpl index e846c570..31b0dfbf 100644 --- a/docs/swagger.conf.tpl +++ b/docs/swagger.conf.tpl @@ -10,24 +10,18 @@ package docs // @Tag.description.markdown // @Tag.name Files // @Tag.description.markdown -// @Tag.name Packages -// @Tag.description.markdown // @Tag.name Mirrors // @Tag.description.markdown // @Tag.name Snapshots // @Tag.description.markdown // @Tag.name Publish // @Tag.description.markdown -// @Tag.name Graph -// @Tag.description.markdown -// @Tag.name S3 -// @Tag.description.markdown -// @Tag.name GPG -// @Tag.description.markdown -// @Tag.name Database +// @Tag.name Packages // @Tag.description.markdown // @Tag.name Status // @Tag.description.markdown +// @Tag.name Database +// @Tag.description.markdown // @Tag.name Tasks // @Tag.description.markdown From a69c00a5bc9daa8b886125288dba23aded33fddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 13:34:33 +0100 Subject: [PATCH 33/64] swagger: improve layout and fix lint --- api/packages.go | 2 +- docs/Mirrors.md | 2 +- docs/Packages.md | 2 +- docs/Publish.md | 2 +- docs/Snapshots.md | 2 +- docs/Status.md | 2 +- docs/docs.html | 17 +++++++++++++++++ 7 files changed, 23 insertions(+), 6 deletions(-) diff --git a/api/packages.go b/api/packages.go index 32c086f6..631d3014 100644 --- a/api/packages.go +++ b/api/packages.go @@ -1,8 +1,8 @@ package api import ( + _ "github.com/aptly-dev/aptly/deb" // for swagger "github.com/gin-gonic/gin" - _ "github.com/aptly-dev/aptly/deb" ) // @Summary Show packages diff --git a/docs/Mirrors.md b/docs/Mirrors.md index 69aef80a..18717d29 100644 --- a/docs/Mirrors.md +++ b/docs/Mirrors.md @@ -1,4 +1,4 @@ -# Remote Repository Mirrors +# Manage Remote Repository Mirrors
Manage mirrors of remote Debian repositories. diff --git a/docs/Packages.md b/docs/Packages.md index 8c3731d4..18385c0e 100644 --- a/docs/Packages.md +++ b/docs/Packages.md @@ -1,4 +1,4 @@ -# Search Packages +# Search Package Collection
Perform operations on the whole collection of packages in apty database.
diff --git a/docs/Publish.md b/docs/Publish.md index 2d1f2af0..b5e172b9 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -1,4 +1,4 @@ -# Publish repositories and mirrors +# Publish Repositories and Mirrors
Publish snapshot or local repo as Debian repository which could be served by HTTP/FTP/rsync server. Repository is signed by user's key with GnuPG. Key should be created beforehand (see section GPG Keys below). Published repository could be consumed directly by apt. diff --git a/docs/Snapshots.md b/docs/Snapshots.md index 882c64c0..e8f41b75 100644 --- a/docs/Snapshots.md +++ b/docs/Snapshots.md @@ -1,4 +1,4 @@ -# Manage repository and mirror Snapshots +# Manage Snapshots of Repositories and Mirrors
Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. Snapshot is immutable, i.e. it can’t be changed since it has been created. Snapshots could be merged, filtered, individual packages could be pulled, snapshot could be verified for missing dependencies. Finally, snapshots could be published as repositories.
diff --git a/docs/Status.md b/docs/Status.md index 5ccf8e14..2aac9e18 100644 --- a/docs/Status.md +++ b/docs/Status.md @@ -1,4 +1,4 @@ -# Aptly Status Information +# Status Information
Various status information. diff --git a/docs/docs.html b/docs/docs.html index e0f4eda0..6507fe97 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -73,6 +73,10 @@ align-self: flex-start; } + #swagger-ui { + padding-bottom: 112px; + } + @@ -117,6 +121,17 @@ From 63b8cc9ad9ff30c1152618c761e58637ddd92b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 14:13:50 +0100 Subject: [PATCH 34/64] dos: improve api info --- docs/api.md | 44 +++++++------------------------------------- docs/docs.html | 7 ++++++- 2 files changed, 13 insertions(+), 38 deletions(-) diff --git a/docs/api.md b/docs/api.md index 8618e2d7..36dc761d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,32 +1,8 @@ -Using aptly via REST API allows to achieve two goals: +Aptly operations are also available via REST API served with `aptly api serve`. -1. Remote access to aptly service: e.g. uploading packages and publishing them from CI server. -2. Concurrent access to aptly by multiple users. +On Debian based systems, a package `aptly-api` is available, which will run aptly as systemd service as dedicated aptly-api user. -#### Quickstart - -Run `aptly api serve` to start HTTP service: - - $ aptly api serve - Starting web server at: :8080 (press Ctrl+C to quit)... - [GIN-debug] GET /api/version --> github.com/aptly-dev/aptly/api.apiVersion (4 handlers) - ... - -By default aptly would listen on `:8080`, but it could be changed with `-listen` flag. - -Usage: - - $ aptly api serve -listen=:8080 - -Flags: - -- `-listen=":8080"`: host:port for HTTP listening -- `-no-lock`: don't lock the database - -When `-no-lock` option is enabled, API server acquires and drops the lock -around all the operations, so that API and CLI could be used concurrently. - -Try some APIs: +#### Example API calls $ curl http://localhost:8080/api/version {"Version":"0.9~dev"} @@ -46,15 +22,9 @@ Try some APIs: $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} -#### Security - -For security reasons it is advised to let Aptly listen on a Unix domain socket rather than a port. Sockets are subject to file permissions and thus allow for user-level access control while binding to a port only gives host-level access control. To use a socket simply run Aptly with a suitable listen flag such as `aptly api serve -listen=unix:///var/run/aptly.sock`. - -Aptly's HTTP API shouldn't be directly exposed to the Internet: there's no authentication/protection of APIs. To publish the API it could be proxied through a HTTP proxy or server (e.g. nginx) to add an authentication mechanism or disallow all non-GET requests. [Reference example](https://github.com/sepich/nginx-ldap) for LDAP based per-repo access with nginx. - #### Notes -1. Some things (for example, S3 publishing endpoints) could be set up only using configuration file, so it requires restart of aptly HTTP server in order for changes to take effect. -1. GPG key passphrase can't be input on console, so either passwordless GPG keys are required or passphrase should be specified in API parameters. -1. Some script might be required to start/stop aptly HTTP service. -1. Some parameters are given as part of URLs, which requires proper url encoding. Unfortunately, at the moment it's not possible to pass URL arguments with `/` in them. +- Some configuration changes (S3 publishing endpoints, ...) will require restarting the aptly service +- Aptly's HTTP API shouldn't be directly exposed to the Internet as there is no authentication/protection of APIs. Consider using a HTTP proxy or server (e.g. nginx) to add an authentication mechanism. + +#### Aptly REST API Documentation diff --git a/docs/docs.html b/docs/docs.html index 6507fe97..142791ed 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -30,6 +30,11 @@ display: none; } + /* spacing */ + .swagger-ui .info { + margin: 0px !important; + } + /* API Summary */ .swagger-ui .opblock-summary-description { flex: 0 0 auto !important; @@ -41,7 +46,7 @@ /* Group Spacing */ .swagger-ui .opblock-tag-section { - padding-top: 40px !important; + padding-bottom: 40px !important; } /* Tag Group */ From abfad376407098558c6db971409814fe1800058c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 14:59:16 +0100 Subject: [PATCH 35/64] swagger: cleanup files doc --- api/files.go | 52 +++++++++++++++++++++++++------- docs/Files.md | 80 +++----------------------------------------------- docs/Repos.md | 10 +++---- docs/docs.html | 2 +- 4 files changed, 51 insertions(+), 93 deletions(-) diff --git a/api/files.go b/api/files.go index 6b2ddba5..2d042a5d 100644 --- a/api/files.go +++ b/api/files.go @@ -34,8 +34,14 @@ func verifyDir(c *gin.Context) bool { return true } -// @Summary List Files -// @Description **Get list of uploaded files** +// @Summary List Directories +// @Description **Get list of upload directories** +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl http://localhost:8080/api/files +// @Description ["aptly-0.9"] +// @Description ``` // @Tags Files // @Produce json // @Success 200 {array} string "List of files" @@ -67,9 +73,17 @@ func apiFilesListDirs(c *gin.Context) { c.JSON(200, list) } -// @Summary Upload file glaa -// @Description **Upload a file to a directory** -// @Description Any number of files can be uploaded in one call; aptly preserves filenames. No check is performed if existing uploaded file would be overwritten. +// @Summary Upload Files +// @Description **Upload files to a directory** +// @Description +// @Description - one or more files can be uploaded +// @Description - existing uploaded are overwritten +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9 +// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"] +// @Description ``` // @Tags Files // @Accept multipart/form-data // @Param dir path string true "Directory to upload files to. Created if does not exist" @@ -132,8 +146,14 @@ func apiFilesUpload(c *gin.Context) { c.JSON(200, stored) } -// @Summary List files in directory -// @Description **Returns list of files in directory** +// @Summary List Files +// @Description **Show uploaded files in upload directory** +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl http://localhost:8080/api/files/aptly-0.9 +// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"] +// @Description ``` // @Tags Files // @Produce json // @Param dir path string true "Directory to list" @@ -179,7 +199,13 @@ func apiFilesListFiles(c *gin.Context) { } // @Summary Delete Directory -// @Description **Deletes all files in upload directory and directory itself** +// @Description **Delete upload directory and uploaded files within** +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9 +// @Description {} +// @Description ``` // @Tags Files // @Produce json // @Param dir path string true "Directory" @@ -200,8 +226,14 @@ func apiFilesDeleteDir(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary Delete File in Directory -// @Description **Delete single file in directory** +// @Summary Delete File +// @Description **Delete a uploaded file in upload directory** +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb +// @Description {} +// @Description ``` // @Tags Files // @Produce json // @Param dir path string true "Directory to delete from" diff --git a/docs/Files.md b/docs/Files.md index 84ed30a3..616a64ac 100644 --- a/docs/Files.md +++ b/docs/Files.md @@ -1,84 +1,12 @@ # Upload Package Files
-Upload package files temporarily to aptly service. These files -could be added to local repositories using [local repositories API](/doc/api/repos). -All uploaded files are stored under `/upload` directory (see [configuration](/doc/configuration)). -This directory would be created automatically if it doesn't exist. +In order to add debian package files to a local repository, files are first uploaded to a temporary directory. +Then the directory (or a specific file within) is added to a repository. After adding to a repositorty, the directory resp. files are removed bt default. -Uploaded files are grouped by directories to support concurrent uploads from multiple -package sources. Local repos add API can operate on directory (adding all files from directory) or -on individual package files. By default, all successfully added package files would be removed. +All uploaded files are stored under `/upload/` directory. -### List Directories +For concurrent uploads from CI/CD pipelines, make sure the tempdir is unique. -`GET /api/files` - -List all directories. - -Response: list of directory names. - -Example: - - $ curl http://localhost:8080/api/files - ["aptly-0.9"] - -### Upload File(s) - -`POST /api/files/:dir` - -Parameter `:dir` is upload directory name. Directory would be created if it doesn't exist. - -Any number of files can be uploaded in one call, aptly would preserve filenames. No check is performed -if existing uploaded would be overwritten. - -Response: list of uploaded files as `:dir/:file`. - -Example: - - $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9 - ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"] - -### List Files in Directory - -`GET /api/files/:dir` - -Returns list of files in directory. - -Response: list of filenames. - -HTTP Errors: - - Code | Description -----------|------------------------- - 404 | directory doesn't exist - -Example: - - $ curl http://localhost:8080/api/files/aptly-0.9 - ["aptly_0.9~dev+217+ge5d646c_i386.deb"] - - -### Delete Directory - -`DELETE /api/files/:dir` - -Deletes all files in upload directory and directory itself. - -Example: - - $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9 - {} - -### Delete File in Directory - -`DELETE /api/files/:dir/:file` - -Delete single file in directory. - -Example: - - $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb - {}
diff --git a/docs/Repos.md b/docs/Repos.md index aeffda24..05960c03 100644 --- a/docs/Repos.md +++ b/docs/Repos.md @@ -1,11 +1,9 @@ # Manage Local Repositories
-Local repository is a collection of packages (most usually custom packages created internally). -Packages could be added to local repository at any moment, removed, moved or copied between repos. -Multiple versions of the same package could be added to the repository. -In order to capture current repository state, [create snapshot](/doc/aptly/snapshot/create/). +A local repository is a collection of versionned packages (usually custom packages created internally). -Local repositories could be published either [directly](/doc/aptly/publish/repo/) -or from [snapshot](/doc/aptly/publish/snapshot). +Packages can be added, removed, moved or copied between repos. + +Local repositories can be published (either directly or via snapshot) to be used a APT source on a debian based system.
diff --git a/docs/docs.html b/docs/docs.html index 142791ed..d057ee1d 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -167,7 +167,7 @@ window.onload = function() { window.ui = ui - setTimeout(collapseAll, 700); + setTimeout(collapseAll, 50); } From 4ff3c894faec6a5332750b415c0c431940ed1323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 15:16:12 +0100 Subject: [PATCH 36/64] swagger: cleanup Snapshots --- api/graph.go | 2 +- api/mirror.go | 2 +- api/snapshot.go | 12 +++++++----- docs/Snapshots.md | 6 +++++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/api/graph.go b/api/graph.go index efeb8384..78d4958f 100644 --- a/api/graph.go +++ b/api/graph.go @@ -26,7 +26,7 @@ import ( // @Description // @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical` // @Tags Status -// @Produce json +// @Produce image/png, image/svg+xml // @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg." // @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout." // @Success 200 {object} []byte "Output" diff --git a/api/mirror.go b/api/mirror.go index 952c67d3..f121d2bf 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -83,7 +83,7 @@ type mirrorCreateParams struct { } // @Summary Create Mirror -// @Description **Create a mirror of a remote repositoru** +// @Description **Create a mirror of a remote repository** // @Tags Mirrors // @Consume json // @Param request body mirrorCreateParams true "Parameters" diff --git a/api/snapshot.go b/api/snapshot.go index ba94b191..6138b3de 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -14,8 +14,9 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary Get snapshots -// @Description **Get list of available snapshots** +// @Summary List Snapshots +// @Description **Get list of snapshots** +// @Description // @Description Each snapshot is returned as in “show” API. // @Tags Snapshots // @Produce json @@ -45,8 +46,8 @@ type snapshotsCreateFromMirrorParams struct { Description string } -// @Summary Create snapshot from mirror -// @Description **Create snapshot of a mirror from given name** +// @Summary Snapshot Mirror +// @Description **Create a snapshot of a mirror** // @Tags Snapshots // @Produce json // @Param request body snapshotsCreateFromMirrorParams true "Parameters" @@ -119,8 +120,9 @@ type snapshotsCreateParams struct { PackageRefs []string } -// @Summary Create snapshot from repo +// @Summary Snapshot Packages // @Description **Create a snapshot from package refs** +// @Description // @Description Refs can be obtained from snapshots, local repos, or mirrors // @Tags Snapshots // @Param request body snapshotsCreateParams true "Parameters" diff --git a/docs/Snapshots.md b/docs/Snapshots.md index e8f41b75..d6db4a4d 100644 --- a/docs/Snapshots.md +++ b/docs/Snapshots.md @@ -1,4 +1,8 @@ # Manage Snapshots of Repositories and Mirrors
-Snapshot is a fixed state of remote repository mirror or local repository. Internally snapshot is list of references to packages. Snapshot is immutable, i.e. it can’t be changed since it has been created. Snapshots could be merged, filtered, individual packages could be pulled, snapshot could be verified for missing dependencies. Finally, snapshots could be published as repositories. + +Local Repositories and Mirrors can be snapshotted to get an immutable state. + +Snapshots cab be merged, filtered, verified for missing dependencies. Snapshots can be published to be used as APT source. +
From a59fc6b8e8c3e8789a631431735659b3819a62f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 15:31:01 +0100 Subject: [PATCH 37/64] swwagger: cleanup --- docs/Database.md | 1 + docs/Mirrors.md | 4 ++-- docs/Publish.md | 15 +++++---------- docs/Tasks.md | 6 +++--- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/Database.md b/docs/Database.md index 60e55ef0..b4eca8d0 100644 --- a/docs/Database.md +++ b/docs/Database.md @@ -1,4 +1,5 @@ # Maintenance Operations
Manage aptly’s internal metadata database and package pool. +
diff --git a/docs/Mirrors.md b/docs/Mirrors.md index 18717d29..4bacaec1 100644 --- a/docs/Mirrors.md +++ b/docs/Mirrors.md @@ -1,8 +1,8 @@ # Manage Remote Repository Mirrors
-Manage mirrors of remote Debian repositories. +Manage mirrors of remote Debian repositories (http, https or ftp). -HTTP, HTTPS and FTP repositories are supported. +Flat debian repositories, mirroring source packages and debian installers is supported.
diff --git a/docs/Publish.md b/docs/Publish.md index b5e172b9..3e497a67 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -1,22 +1,17 @@ # Publish Repositories and Mirrors
-Publish snapshot or local repo as Debian repository which could be served by HTTP/FTP/rsync server. Repository is signed by user's key with GnuPG. Key should be created beforehand (see section GPG Keys below). Published repository could be consumed directly by apt. +Publish snapshot or local repo as Debian repository to be used as APT source on Debian based systems. -Repositories could be published to Amazon S3 service: create bucket, -[configure publishing endpoint](/doc/feature/s3/) and use S3 endpoint when -publishing. +The published repository is signed with the user's GnuPG key. +Repositories can be published to local directories, Amazon S3 buckets, Azure or Swift Storage. #### GPG Keys -GPG key is required to sign any published repository. Key should be generated before publishing first repository. +GPG key is required to sign any published repository. The key pari should be generated before publishing. -Key generation, storage, backup and revocation is out of scope of this document, there are many tutorials available, e.g. [this one](http://fedoraproject.org/wiki/Creating_GPG_Keys). - -Publiс part of the key should be exported from your keyring using `gpg --export --armor` and imported into apt keyring using `apt-key` tool on all machines that would be using published repositories. - -Signing releases is highly recommended, but if you want to skip it, you can either use `gpgDisableSign` configuration option or `--skip-signing` flag. +Publiс part of the key should be exported from your keyring using `gpg --export --armor` and imported on the system which uses a published repository. #### Parameters diff --git a/docs/Tasks.md b/docs/Tasks.md index b71ca31a..03668d8e 100644 --- a/docs/Tasks.md +++ b/docs/Tasks.md @@ -1,8 +1,8 @@ # Background Tasks
-An aptly task is a sequence of multiple aptly commands run within a single aptly thread. -For example, a task could be to [create a new repository](/doc/aptly/repo/create), [add packages to it](/doc/aptly/repo/add), [create a snapshot](/doc/aptly/snapshot/create) and [serve it](/doc/aptly/serve). +Several API operations allow to be run in background asynchronously in a task. In that case, a Task object with an ID and a State is returned, which can be queried for progress. + +Tasks should be deleted once they are no longer in progress, in order to not cause memory overflows. -Four commands can be now run in a single command.
From 75e5f95277e36cb747b36aa83eecc9fcf666b1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 15:39:47 +0100 Subject: [PATCH 38/64] task-dummy: remove internal testing API --- api/router.go | 1 - api/task.go | 20 ------------- api/task_test.go | 75 ------------------------------------------------ 3 files changed, 96 deletions(-) delete mode 100644 api/task_test.go diff --git a/api/router.go b/api/router.go index f7c95fd3..3536ab09 100644 --- a/api/router.go +++ b/api/router.go @@ -232,7 +232,6 @@ func Router(c *ctx.AptlyContext) http.Handler { api.GET("/tasks/:id/return_value", apiTasksReturnValueShow) api.GET("/tasks/:id", apiTasksShow) api.DELETE("/tasks/:id", apiTasksDelete) - api.POST("/tasks-dummy", apiTasksDummy) } return router diff --git a/api/task.go b/api/task.go index d9ea737b..1d136325 100644 --- a/api/task.go +++ b/api/task.go @@ -1,10 +1,8 @@ package api import ( - "net/http" "strconv" - "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/task" "github.com/gin-gonic/gin" ) @@ -203,21 +201,3 @@ func apiTasksDelete(c *gin.Context) { c.JSON(200, delTask) } - -// @Summary Dummy endpoint used for testing. -// @Description **Dummy endpoint used for testing** -// @Tags Tasks -// @Produce json -// @Param _async query bool false "Run task in background using tasks API" -// @Success 200 {object} task.ProcessReturnValue -// @Router /api/tasks-dummy [post] -func apiTasksDummy(c *gin.Context) { - resources := []string{"dummy"} - taskName := "Dummy task" - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { - out.Printf("Dummy task started\n") - detail.Store([]int{1, 2, 3}) - out.Printf("Dummy task finished\n") - return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil - }) -} diff --git a/api/task_test.go b/api/task_test.go deleted file mode 100644 index 59c981a0..00000000 --- a/api/task_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/aptly-dev/aptly/task" - - . "gopkg.in/check.v1" -) - -type TaskSuite struct { - ApiSuite -} - -var _ = Suite(&TaskSuite{}) - -func (s *TaskSuite) TestTasksDummy(c *C) { - response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil) - c.Check(response.Code, Equals, 418) - c.Check(response.Body.String(), Equals, "[1,2,3]") -} - -func (s *TaskSuite) TestTasksDummyAsync(c *C) { - response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil) - c.Check(response.Code, Equals, 202) - var t task.Task - err := json.Unmarshal(response.Body.Bytes(), &t) - c.Assert(err, IsNil) - c.Check(t.Name, Equals, "Dummy task") - response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil) - err = json.Unmarshal(response.Body.Bytes(), &t) - c.Assert(err, IsNil) - c.Check(t.State, Equals, task.SUCCEEDED) - response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil) - c.Check(response.Code, Equals, 200) - c.Check(response.Body.String(), Equals, "[1,2,3]") - response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil) - c.Check(response.Code, Equals, 200) - c.Check(response.Body.String(), Matches, "\"Dummy task started.*") -} - -func (s *TaskSuite) TestTaskDelete(c *C) { - response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil) - c.Check(response.Code, Equals, 202) - c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}") - // Give the task time to start - time.Sleep(time.Second) - response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil) - c.Check(response.Code, Equals, 200) -} - -func (s *TaskSuite) TestTasksClear(c *C) { - response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil) - c.Check(response.Code, Equals, 202) - var t task.Task - err := json.Unmarshal(response.Body.Bytes(), &t) - c.Assert(err, IsNil) - c.Check(t.Name, Equals, "Dummy task") - response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil) - c.Check(response.Code, Equals, 200) - response, _ = s.HTTPRequest("GET", "/api/tasks", nil) - c.Check(response.Code, Equals, 200) - var ts []task.Task - err = json.Unmarshal(response.Body.Bytes(), &ts) - c.Assert(err, IsNil) - c.Check(len(ts), Equals, 1) - c.Check(ts[0].State, Equals, task.SUCCEEDED) - response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil) - c.Check(response.Code, Equals, 200) - response, _ = s.HTTPRequest("GET", "/api/tasks", nil) - c.Check(response.Code, Equals, 200) - c.Check(response.Body.String(), Equals, "[]") -} From c6e0a06b141ae2d0ded33da0eb014409e6e3a1eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 15:51:18 +0100 Subject: [PATCH 39/64] swagger: cleanup --- api/api.go | 7 ++++++- api/publish.go | 6 ++++++ api/repos.go | 15 ++++++++++++++- docs/api.md | 25 ++----------------------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/api/api.go b/api/api.go index c66a44b1..adeef9d9 100644 --- a/api/api.go +++ b/api/api.go @@ -30,7 +30,12 @@ type aptlyVersion struct { // @Summary Aptly version // @Description **Get aptly version** -// @Description Returns the aptly version +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl http://localhost:8080/api/version +// @Description {"Version":"0.9~dev"} +// @Description ``` // @Tags Status // @Produce json // @Success 200 {object} aptlyVersion diff --git a/api/publish.go b/api/publish.go index 9b10e65f..9303b382 100644 --- a/api/publish.go +++ b/api/publish.go @@ -179,6 +179,12 @@ type publishedRepoCreateParams struct { // @Description // @Description The prefix may contain a storage specifier, e.g. `s3:packages/`, or it may also be empty to publish to the root directory. // @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos +// @Description {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} +// @Description ``` +// @Description // @Description See also: `aptly publish create` // @Tags Publish // @Param prefix path string true "publishing prefix" diff --git a/api/repos.go b/api/repos.go index fa6c3e17..93475337 100644 --- a/api/repos.go +++ b/api/repos.go @@ -98,8 +98,14 @@ type repoCreateParams struct { } // @Summary Create repository -// @Description Create a local repository with specified parameters. +// @Description **Create a local repository** +// @Description // @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot. +// @Description +// @Description ``` +// @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos +// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""} +// @Description ``` // @Tags Repos // @Produce json // @Consume json @@ -287,7 +293,14 @@ func apiReposDrop(c *gin.Context) { // @Summary List Repo Packages // @Description **Return a list of packages present in the repo** +// @Description // @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ curl http://localhost:8080/api/repos/aptly-repo/packages +// @Description ["Pi386 aptly 0.8 966561016b44ed80"] +// @Description ``` // @Tags Repos // @Produce json // @Param name path string true "Snapshot to search" diff --git a/docs/api.md b/docs/api.md index 36dc761d..aaf87195 100644 --- a/docs/api.md +++ b/docs/api.md @@ -2,29 +2,8 @@ Aptly operations are also available via REST API served with `aptly api serve`. On Debian based systems, a package `aptly-api` is available, which will run aptly as systemd service as dedicated aptly-api user. -#### Example API calls +Some configuration changes (S3 publishing endpoints, ...) will require restarting the aptly service in order to take effect. - $ curl http://localhost:8080/api/version - {"Version":"0.9~dev"} - - $ curl -F file=@aptly_0.8_i386.deb http://localhost:8080/api/files/aptly_0.8 - ["aptly_0.8/aptly_0.8_i386.deb"] - - $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos - {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""} - - $ curl -X POST http://localhost:8080/api/repos/aptly-repo/file/aptly_0.8 - {"failedFiles":[],"report":{"warnings":[],"added":["aptly_0.8_i386 added"],"removed":[]}} - - $ curl http://localhost:8080/api/repos/aptly-repo/packages - ["Pi386 aptly 0.8 966561016b44ed80"] - - $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos - {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} - -#### Notes - -- Some configuration changes (S3 publishing endpoints, ...) will require restarting the aptly service -- Aptly's HTTP API shouldn't be directly exposed to the Internet as there is no authentication/protection of APIs. Consider using a HTTP proxy or server (e.g. nginx) to add an authentication mechanism. +The REST API shouldn't be exposed to the Internet as there is no authentication/protection, consider using a HTTP proxy (e.g. nginx) to add https and authentication. #### Aptly REST API Documentation From e5e3c49aced1dd872ee7576082c194c4bd4042cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 16:07:33 +0100 Subject: [PATCH 40/64] swagger: document async --- api/db.go | 2 +- api/mirror.go | 2 ++ api/publish.go | 9 +++++++++ api/repos.go | 16 ++++++++-------- api/snapshot.go | 14 +++++++------- docs/Publish.md | 2 +- docs/Snapshots.md | 2 +- 7 files changed, 29 insertions(+), 18 deletions(-) diff --git a/api/db.go b/api/db.go index f6d0e5a8..259a94aa 100644 --- a/api/db.go +++ b/api/db.go @@ -17,7 +17,7 @@ import ( // @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos. // @Tags Database // @Produce json -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} string "Output" // @Failure 404 {object} Error "Not Found" // @Router /api/db/cleanup [post] diff --git a/api/mirror.go b/api/mirror.go index f121d2bf..48a505fe 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -164,6 +164,7 @@ func apiMirrorsCreate(c *gin.Context) { // @Tags Mirrors // @Param name path string true "mirror name" // @Param force query int true "force: 1 to enable" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {object} task.ProcessReturnValue // @Failure 404 {object} Error "Mirror not found" @@ -364,6 +365,7 @@ type mirrorUpdateParams struct { // @Param name path string true "mirror name to update" // @Consume json // @Param request body mirrorUpdateParams true "Parameters" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully" // @Success 202 {object} task.Task "Mirror is being updated" diff --git a/api/publish.go b/api/publish.go index 9303b382..453f9a3d 100644 --- a/api/publish.go +++ b/api/publish.go @@ -188,6 +188,7 @@ type publishedRepoCreateParams struct { // @Description See also: `aptly publish create` // @Tags Publish // @Param prefix path string true "publishing prefix" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body publishedRepoCreateParams true "Parameters" // @Produce json @@ -395,6 +396,7 @@ type publishedRepoUpdateSwitchParams struct { // @Produce json // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body publishedRepoUpdateSwitchParams true "Parameters" // @Produce json @@ -525,6 +527,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Param distribution path string true "distribution name" // @Param force query int true "force: 1 to enable" // @Param skipCleanup query int true "skipCleanup: 1 to enable" +// @Param _async query bool false "Run in background and return task object" // @Success 200 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found" @@ -571,6 +574,7 @@ func apiPublishDrop(c *gin.Context) { // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body sourceParams true "Parameters" // @Produce json @@ -688,6 +692,7 @@ func apiPublishListChanges(c *gin.Context) { // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body []sourceParams true "Parameters" // @Produce json @@ -753,6 +758,7 @@ func apiPublishSetSources(c *gin.Context) { // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 // @Failure 400 {object} Error "Bad Request" @@ -805,6 +811,7 @@ func apiPublishDropChanges(c *gin.Context) { // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Param component path string true "component name" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body sourceParams true "Parameters" // @Produce json @@ -884,6 +891,7 @@ func apiPublishUpdateSource(c *gin.Context) { // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Param component path string true "component name" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 // @Failure 400 {object} Error "Bad Request" @@ -960,6 +968,7 @@ type publishedRepoUpdateParams struct { // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Param request body publishedRepoUpdateParams true "Parameters" // @Produce json diff --git a/api/repos.go b/api/repos.go index 93475337..1069be13 100644 --- a/api/repos.go +++ b/api/repos.go @@ -251,7 +251,7 @@ func apiReposShow(c *gin.Context) { // @Description Needs force=1 to drop repos used as source by other repos. // @Tags Repos // @Produce json -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Param force query int false "force: 1 to enable" // @Success 200 {object} task.ProcessReturnValue "Repo object" // @Failure 404 {object} Error "Not Found" @@ -403,7 +403,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li // @Tags Repos // @Produce json // @Param request body reposPackagesAddDeleteParams true "Parameters" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} string "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -423,7 +423,7 @@ func apiReposPackagesAdd(c *gin.Context) { // @Tags Repos // @Produce json // @Param request body reposPackagesAddDeleteParams true "Parameters" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} string "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -445,7 +445,7 @@ func apiReposPackagesDelete(c *gin.Context) { // @Param name path string true "Repository name" // @Param dir path string true "Directory of packages" // @Param file path string false "Filename (optional)" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {string} string "OK" // @Failure 400 {object} Error "wrong file" @@ -467,7 +467,7 @@ func apiReposPackageFromFile(c *gin.Context) { // @Consume json // @Param noRemove query string false "when value is set to 1, don’t remove any files" // @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {string} string "OK" // @Failure 400 {object} Error "wrong file" @@ -604,7 +604,7 @@ type reposCopyPackageParams struct { // @Param name path string true "Source repo" // @Param src path string true "Destination repo" // @Param file path string true "File/packages to copy" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} task.ProcessReturnValue "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -753,7 +753,7 @@ func apiReposCopyPackage(c *gin.Context) { // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir}/{file} [post] @@ -781,7 +781,7 @@ type reposIncludePackageFromDirResponse struct { // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 200 {object} reposIncludePackageFromDirResponse "Response" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir} [post] diff --git a/api/snapshot.go b/api/snapshot.go index 6138b3de..327f22db 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -52,7 +52,7 @@ type snapshotsCreateFromMirrorParams struct { // @Produce json // @Param request body snapshotsCreateFromMirrorParams true "Parameters" // @Param name path string true "Mirror name" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 201 {object} deb.Snapshot "Created Snapshot" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Mirror Not Found" @@ -126,7 +126,7 @@ type snapshotsCreateParams struct { // @Description Refs can be obtained from snapshots, local repos, or mirrors // @Tags Snapshots // @Param request body snapshotsCreateParams true "Parameters" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 201 {object} deb.Snapshot "Created snapshot" // @Failure 400 {object} Error "Bad Request" @@ -213,7 +213,7 @@ type snapshotsCreateFromRepositoryParams struct { // @Consume json // @Param request body snapshotsCreateFromRepositoryParams true "Parameters" // @Param name path string true "Name of the snapshot" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 201 {object} deb.Snapshot "Created snapshot object" // @Failure 400 {object} Error "Bad Request" @@ -279,7 +279,7 @@ type snapshotsUpdateParams struct { // @Tags Snapshots // @Param request body snapshotsUpdateParams true "Parameters" // @Param name path string true "Snapshot name" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {object} deb.Snapshot "Updated snapshot object" // @Failure 404 {object} Error "Snapshot Not Found" @@ -366,7 +366,7 @@ func apiSnapshotsShow(c *gin.Context) { // @Tags Snapshots // @Param name path string true "Snapshot name" // @Param force query string false "Force operation" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 "" // @Failure 404 {object} Error "Snapshot Not Found" @@ -525,7 +525,7 @@ type snapshotsMergeParams struct { // @Param latest query int false "merge only the latest version of each package" // @Param no-remove query int false "all versions of packages are preserved during merge" // @Param request body snapshotsMergeParams true "Parameters" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Success 201 {object} deb.Snapshot "Resulting snapshot object" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -637,7 +637,7 @@ type snapshotsPullParams struct { // @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable" // @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable" // @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable" -// @Param _async query bool false "Run task in background using tasks API" +// @Param _async query bool false "Run in background and return task object" // @Consume json // @Produce json // @Success 200 {object} deb.Snapshot "Resulting Snapshot object" diff --git a/docs/Publish.md b/docs/Publish.md index 3e497a67..0077f4d9 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -1,4 +1,4 @@ -# Publish Repositories and Mirrors +# Publish Repositories, Snapshots, Mirrors
Publish snapshot or local repo as Debian repository to be used as APT source on Debian based systems. diff --git a/docs/Snapshots.md b/docs/Snapshots.md index d6db4a4d..21d0e526 100644 --- a/docs/Snapshots.md +++ b/docs/Snapshots.md @@ -1,4 +1,4 @@ -# Manage Snapshots of Repositories and Mirrors +# Manage Snapshots
Local Repositories and Mirrors can be snapshotted to get an immutable state. From d8b9777b406e1aae218e2bd657f1e36633d63c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 28 Nov 2024 17:16:08 +0100 Subject: [PATCH 41/64] swagger: document params --- api/gpg.go | 12 ++++++++---- api/repos.go | 19 +++++++++++++------ api/snapshot.go | 30 ++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/api/gpg.go b/api/gpg.go index 578ed18f..4ea79b68 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -13,10 +13,14 @@ import ( ) type gpgAddKeyParams struct { - Keyserver string - GpgKeyID string - GpgKeyArmor string - Keyring string + // Keyserver, when downloading GpgKeyIDs + Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"` + // GpgKeyIDs to download from Keyserver, comma separated list + GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500,8B48AD6246925553"` + // Armored gpg public ket, instead of downloading from keyserver + GpgKeyArmor string `json:"GpgKeyArmor" example:""` + // Keyring for adding the keys (default: trustedkeys.gpg) + Keyring string `json:"Keyring" example:"trustedkeys.gpg"` } // @Summary Add GPG Keys diff --git a/api/repos.go b/api/repos.go index 1069be13..3acc29ea 100644 --- a/api/repos.go +++ b/api/repos.go @@ -165,10 +165,14 @@ func apiReposCreate(c *gin.Context) { } type reposEditParams struct { - Name *string - Comment *string - DefaultDistribution *string - DefaultComponent *string + // Name of repository to modify + Name *string `binding:"required" json:"Name" example:"repo1"` + // Change Comment of repository + Comment *string ` json:"Comment" example:"example repo"` + // Change Default Distribution for publishing + DefaultDistribution *string ` json:"DefaultDistribution" example:""` + // Change Devault Component for publishing + DefaultComponent *string ` json:"DefaultComponent" example:""` } // @Summary Update repo @@ -332,7 +336,8 @@ func apiReposPackagesShow(c *gin.Context) { } type reposPackagesAddDeleteParams struct { - PackageRefs []string + // Package Refs + PackageRefs []string `binding:"required" json:"PackageRefs" example:""` } // Handler for both add and delete @@ -593,8 +598,10 @@ func apiReposPackageFromDir(c *gin.Context) { } type reposCopyPackageParams struct { + // Copy also dependencies WithDeps bool `json:"with-deps,omitempty"` - DryRun bool `json:"dry-run,omitempty"` + // Do not perform operations + DryRun bool `json:"dry-run,omitempty"` } // @Summary Copy Package diff --git a/api/snapshot.go b/api/snapshot.go index 327f22db..60722781 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -42,8 +42,10 @@ func apiSnapshotsList(c *gin.Context) { } type snapshotsCreateFromMirrorParams struct { - Name string `binding:"required"` - Description string + // Name of snapshot to create + Name string `binding:"required" json:"Name" example:"snap1"` + // Description of snapshot + Description string ` json:"Description"` } // @Summary Snapshot Mirror @@ -114,10 +116,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { } type snapshotsCreateParams struct { - Name string `binding:"required"` - Description string - SourceSnapshots []string - PackageRefs []string + // Name of snapshot to create + Name string `binding:"required" json:"Name" example:"snap2"` + // Description of snapshot + Description string ` json:"Description"` + // List of source snapshots + SourceSnapshots []string `binding:"required" json:"SourceSnapshots" example:"snap1"` + // List of package refs + PackageRefs []string `binding:"required" json:"PackageRefs" example:""` } // @Summary Snapshot Packages @@ -202,8 +208,10 @@ func apiSnapshotsCreate(c *gin.Context) { } type snapshotsCreateFromRepositoryParams struct { - Name string `binding:"required"` - Description string + // Name of snapshot to create + Name string `binding:"required" json:"Name" example:"snap1"` + // Description of snapshot + Description string ` json:"Description"` } // @Summary Snapshot Repository @@ -270,8 +278,10 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { } type snapshotsUpdateParams struct { - Name string - Description string + // Change Name of snapshot + Name string `binding:"required" json:"Name" example:"snap2"` + // Change Description of snapshot + Description string ` json:"Description"` } // @Summary Update Snapshot From 1f469e23b5a87fb1124231819688f62e2fb96fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 29 Nov 2024 17:38:12 +0100 Subject: [PATCH 42/64] fix optional params --- api/snapshot.go | 12 ++++++------ system/api_lib.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index 60722781..d020c3dc 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -117,13 +117,13 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { type snapshotsCreateParams struct { // Name of snapshot to create - Name string `binding:"required" json:"Name" example:"snap2"` + Name string `binding:"required" json:"Name" example:"snap2"` // Description of snapshot - Description string ` json:"Description"` + Description string ` json:"Description"` // List of source snapshots - SourceSnapshots []string `binding:"required" json:"SourceSnapshots" example:"snap1"` + SourceSnapshots []string ` json:"SourceSnapshots" example:"snap1"` // List of package refs - PackageRefs []string `binding:"required" json:"PackageRefs" example:""` + PackageRefs []string ` json:"PackageRefs" example:""` } // @Summary Snapshot Packages @@ -279,9 +279,9 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { type snapshotsUpdateParams struct { // Change Name of snapshot - Name string `binding:"required" json:"Name" example:"snap2"` + Name string ` json:"Name" example:"snap2"` // Change Description of snapshot - Description string ` json:"Description"` + Description string `json:"Description"` } // @Summary Update Snapshot diff --git a/system/api_lib.py b/system/api_lib.py index a41161ea..20036b86 100644 --- a/system/api_lib.py +++ b/system/api_lib.py @@ -84,7 +84,7 @@ class APITest(BaseTest): self._ensure_async(kwargs) resp = self.post(uri, *args, **kwargs) if resp.status_code != 202: - return resp + raise Exception("async api error: " + resp.text) _id = resp.json()['ID'] resp = self.get("/api/tasks/" + str(_id) + "/wait") From 622072bd503aad1e9b0ff11a8c2e7c4fc8553459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 29 Nov 2024 17:56:01 +0100 Subject: [PATCH 43/64] document aptly.conf --- debian/aptly.conf | 396 ++++++++++++++++++++++++++++++---------------- 1 file changed, 257 insertions(+), 139 deletions(-) diff --git a/debian/aptly.conf b/debian/aptly.conf index 4d627fc4..321adc4e 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,35 +1,59 @@ // vim: : filetype=json +// json configuration file with comments +// validate with: sed '/\/\//d' debian/aptly.conf | json_pp { // General /////////// // Aptly storage directory - // - downloaded packages (`rootDir`/pool) - // - database (`rootDir`/db) - // - published repositories (`rootDir`/public) + // Directory for storing: + // - downloaded packages (`rootDir`/pool) + // - database (`rootDir`/db) + // - published repositories (`rootDir`/public) "rootDir": "~/.aptly", - // number of attempts to open DB if it's locked by other instance; can be overridden with option `-db-open-attempts` + // Number of attempts to open database if it's locked by other instance + // * -1 (no retry) "databaseOpenAttempts": -1, - // - "AsyncAPI": false, - - // - "enableMetricsEndpoint": false, - - // Enable API documentation on /docs - "enableSwaggerEndpoint": false, - - // + // Log Level + // * debug + // * info + // * warning + // * error "logLevel": "info", - // + // Log Format + // * default (text) + // * json "logFormat": "default", - // - "serveInAPIMode": false, + // Default Architectures + // * empty defaults to all available architectures + "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package + "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package + "dependencyFollowRecommends": false, + + // When dependency looks like `package-a | package-b`, follow both variants always + "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package + "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) + "dependencyVerboseResolve": false, + + // Specifies paramaters for short PPA url expansion + // * empty defaults to output of `lsb_release` command + "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion + "ppaCodename": "", // OBSOLETE // in aptly up to version 1.0.0, package files were stored in internal package pool @@ -39,12 +63,29 @@ // upgrading from older versions "skipLegacyPool": true, + +// Aptly Server +//////////////// + + // Serve published repos as well as API + "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + // Database //////////// - // + // Database backend "databaseBackend": { - // + // "type": "", // "url": "", @@ -53,196 +94,273 @@ // }, + // Mirroring ///////////// - // downloader to use - // - "default" (normal downloader) - // - "grab" (more robust) + // Downloader + // * "default" + // * "grab" (more robust) "downloader": "default", - // number of parallel download threads to use when downloading packages + // Number of parallel download threads to use when downloading packages "downloadConcurrency": 4, - // limit in kbytes/sec on download speed while mirroring remote repositories + // Limit in kbytes/sec on download speed while mirroring remote repositories "downloadSpeedLimit": 0, - // number of retries for download attempts + // Number of retries for download attempts "downloadRetries": 0, - // download source packages per default + // Download source packages per default "downloadSourcePackages": false, - // list of architectures to process; if left empty defaults to all available architectures; can be overridden with option `-architectures` - "architectures": [], - - // follow contents of `Suggests:` field when processing dependencies for the package - "dependencyFollowSuggests": false, - - // follow contents of `Recommends:` field when processing dependencies for the package - "dependencyFollowRecommends": false, - - // when dependency looks like `package-a | package-b`, follow both variants always - "dependencyFollowAllVariants": false, - - // follow dependency from binary package to source package - "dependencyFollowSource": false, - - // print additional details while resolving dependencies (useful for debugging) - "dependencyVerboseResolve": false, - // Signing /////////// - // gpg provider to use: - // - "internal" (Go internal implementation) - // - 'gpg" (external `gpg` utility, uses GnuPG 1.x if available or GnuPG 2.x otherwise) + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) "gpgProvider": "gpg", - // don't sign published repositories with gpg(1), also can be disabled on per-repo basis using `-skip-signing` flag when publishing + // Disable signing of published repositories "gpgDisableSign": false, - // don't verify remote mirrors with gpg(1), also can be disabled on per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors + // Disable signature verification of remote repositories "gpgDisableVerify": false, -// PPA -/////// +// Publishing +////////////// - // specifies paramaters for short PPA url expansion, if left blank they default to output of `lsb_release` command - "ppaDistributorID": "ubuntu", - - // cwcodename for short PPA url expansion - "ppaCodename": "", - - // + // Do not publish Contents files "skipContentsPublishing": false, - // + // Do not create bz2 files "skipBz2Publishing": false, -// Storage Endpoints -///////////////////// + +// Storage +/////////// // Filesystem publishing endpoints // // aptly defaults to publish to a single publish directory under `rootDir`/public. For // a more advanced publishing strategy, you can define one or more filesystem endpoints in the // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name - // and the following associated settings: - // - // * `rootDir`: - // The publish directory, e.g., `/opt/srv/aptly_public`. - // * `linkMethod`: - // This is one of `hardlink`, `symlink` or `copy`. It specifies how aptly links the - // files from the internal pool to the published directory. - // If not specified, empty or wrong, this defaults to `hardlink`. - // * `verifyMethod`: - // This is used only when setting the `linkMethod` to `copy`. Possible values are - // `md5` and `size`. It specifies how aptly compares existing links from the - // internal pool to the published directory. The `size` method compares only the - // file sizes, whereas the `md5` method calculates the md5 checksum of the found - // file and compares it to the desired one. - // If not specified, empty or wrong, this defaults to `md5`. + // and the following associated settings. // // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` // with `endpoint-name` as the name given in the aptly configuration file. For example: // // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` + // "FileSystemPublishEndpoints": { + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + // + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + // + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (only compare file size) + // "verifyMethod": "md5" + // } }, // S3 Endpoint Support // // cloud storage). First, publishing // endpoints should be described in aptly configuration file. Each endpoint has name - // and associated settings: - // - // * `region`: - // Amazon region for S3 bucket (e.g. `us-east-1`) - // * `bucket`: - // bucket name - // * `endpoint`: - // (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here, - // region is ignored if endpoint is set (set region to some human-readable name) - // (should be left blank for real Amazon S3) - // * `prefix`: - // (optional) do publishing under specified prefix in the bucket, defaults to - // no prefix (bucket root) - // * `acl`: - // (optional) assign ACL to published files (one of the canned ACLs in Amazon - // terminology). Useful values: `private` (default), `public-read` (public - // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - // HTTP endpoint (Amazon bucket should be configured for "website hosting"), - // for private repositories special apt S3 transport is required. - // * `awsAccessKeyID`, `awsSecretAccessKey`: - // (optional) Amazon credentials to access S3 bucket. If not supplied, - // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - // are used. - // * `storageClass`: - // (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values - // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - // * `encryptionMethod`: - // (optional) server-side encryption method, defaults to none. Currently - // the only available encryption method is `AES256` - // * `plusWorkaround`: - // (optional) workaround misbehavior in apt and Amazon S3 - // for files with `+` in filename by - // creating two copies of package files with `+` in filename: one original - // and another one with spaces instead of plus signs - // With `plusWorkaround` enabled, package files with plus sign - // would be stored twice. aptly might not cleanup files with spaces when published - // repository is dropped or updated (switched) to new version of repository (snapshot) - // * `disableMultiDel`: - // (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API, - // enable this setting (file deletion would be slower with this setting enabled) - // * `forceSigV2`: - // (optional) disable Signature V4 support, useful with non-AWS S3-compatible object stores - // which do not support SigV4, shouldn't be enabled for AWS - // * `forceVirtualHostedStyle`: - // (optional) disable path style visit, useful with non-AWS S3-compatible object stores - // which only support virtual hosted style - // * `debug`: - // (optional) enables detailed request/response dump for each S3 operation + // and associated settings. // // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before // publishing prefix on the command line, e.g.: // // `aptly publish snapshot wheezy-main s3:test:` + // "S3PublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Amazon region for S3 bucket + // "region": "us-east-1", + // + // // Bucket name + // "bucket": "test-bucket", + // + // // Endpoint (optional) + // // When using S3-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + // + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + // + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology). Useful values: `private` (default), `public-read` (public + // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required. + // "acl": "private", + // + // // Credentials (optional) + // // Amazon credentials to access S3 bucket. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + // + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + // + // // Encryption Method (optional) + // // Server-side encryption method, defaults to none. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + // + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + // + // // Disable MultiDel (optional) + // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + // + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores + // // which do not support SigV4, shouldn't be enabled for AWS + // "forceSigV2": false, + // + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non-AWS S3-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + // + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } }, // Swift Endpoint Support // // aptly could be configured to publish repository directly to OpenStack Swift. First, // publishing endpoints should be described in aptly configuration file. Each endpoint - // has name and associated settings: - // - // * `container`: - // container name - // * `prefix`: - // (optional) do publishing under specified prefix in the container, defaults to - // no prefix (container root) - // * `osname`, `password`: - // (optional) OpenStack credentials to access Keystone. If not supplied, - // environment variables `OS_USERNAME` and `OS_PASSWORD` are used. - // * `tenant`, `tenantid`: - // (optional) OpenStack tenant name and id (in order to use v2 authentication). - // * `authurl`: - // (optional) the full url of Keystone server (including port, and version). - // example `http://identity.example.com:5000/v2.0` + // has name and associated settings. // // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before // publishing prefix on the command line, e.g.: // // `aptly publish snapshot jessie-main swift:test:` + // "SwiftPublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Container Name + // "container": "container1", + // + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version). + // // Example `http://identity.example.com:5000/v2.0` + // "authurl": "" + // } }, // Azure Endpoint Support // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage. First, publishing endpoints should be described in the aptly + // configuration file. Each endpoint has its name and associated settings. "AzurePublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Container Name + // "container": "container1", + // + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + // + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + // + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://$accountName.blob.core.windows.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool Path + "type": "local", + "path": "$ROOTDIR/pool" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "azure": { + // // Container Name + // "container": "pool1", + // + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + // + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + // + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://$accountName.blob.core.windows.net" + // "endpoint": "" + // } } +// End of config } From ea80f6d49cc5814ad6ce324ec3e0ea5293129e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 1 Dec 2024 10:51:58 +0100 Subject: [PATCH 44/64] write commented json default config --- aptly/conf.go | 4 + context/context.go | 15 +- context/context_test.go | 5 +- debian/aptly.conf | 36 ++- main.go | 4 + main_test.go | 1 + system/t02_config/CreateConfigTest_gold | 397 ++++++++++++++++++++++-- utils/config.go | 12 + 8 files changed, 421 insertions(+), 53 deletions(-) create mode 100644 aptly/conf.go diff --git a/aptly/conf.go b/aptly/conf.go new file mode 100644 index 00000000..b39c2b8b --- /dev/null +++ b/aptly/conf.go @@ -0,0 +1,4 @@ +package aptly + +// Default aptly.conf (filled in at link time) +var AptlyConf []byte diff --git a/context/context.go b/context/context.go index e599102d..5df5bbfd 100644 --- a/context/context.go +++ b/context/context.go @@ -3,7 +3,6 @@ package context import ( gocontext "context" - "errors" "fmt" "math/rand" "os" @@ -116,9 +115,11 @@ func (context *AptlyContext) config() *utils.ConfigStructure { if err != nil { fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation) - // as this is fresh aptly installation, we don't need to support legacy pool locations - utils.Config.SkipLegacyPool = true - utils.SaveConfig(configLocations[0], &utils.Config) + utils.SaveConfigRaw(homeLocation, aptly.AptlyConf) + err = utils.LoadConfig(homeLocation, &utils.Config) + if err != nil { + Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err)) + } } } @@ -293,10 +294,10 @@ func (context *AptlyContext) _database() (database.Storage, error) { var err error switch context.config().DatabaseBackend.Type { case "leveldb": - if len(context.config().DatabaseBackend.DbPath) == 0 { - return nil, errors.New("leveldb databaseBackend config invalid") + dbPath := filepath.Join(context.config().GetRootDir(), "db") + if len(context.config().DatabaseBackend.DbPath) != 0 { + dbPath = context.config().DatabaseBackend.DbPath } - dbPath := filepath.Join(context.config().GetRootDir(), context.config().DatabaseBackend.DbPath) context.database, err = goleveldb.NewDB(dbPath) case "etcd": context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL) diff --git a/context/context_test.go b/context/context_test.go index 0a48f920..f83d42f2 100644 --- a/context/context_test.go +++ b/context/context_test.go @@ -1,6 +1,8 @@ package context import ( + "fmt" + "os" "reflect" "testing" @@ -82,5 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { // storage never exists. c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, FatalErrorPanicMatches, - &FatalError{ReturnCode: 1, Message: "published local storage fuji not configured"}) + &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: EOF", + os.Getenv("HOME"))}) } diff --git a/debian/aptly.conf b/debian/aptly.conf index 321adc4e..0d498e3f 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,6 +1,6 @@ // vim: : filetype=json // json configuration file with comments -// validate with: sed '/\/\//d' debian/aptly.conf | json_pp +// validate with: sed '/\/\//d' aptly.conf | json_pp { // General @@ -30,7 +30,7 @@ "logFormat": "default", // Default Architectures - // * empty defaults to all available architectures + // empty array defaults to all available architectures "architectures": [], // Follow contents of `Suggests:` field when processing dependencies for the package @@ -49,7 +49,7 @@ "dependencyVerboseResolve": false, // Specifies paramaters for short PPA url expansion - // * empty defaults to output of `lsb_release` command + // empty defaults to output of `lsb_release` command "ppaDistributorID": "ubuntu", // Codename for short PPA url expansion @@ -84,14 +84,20 @@ //////////// // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd "databaseBackend": { - // - "type": "", - // - "url": "", - // + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db "dbPath": "" - // + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127.0.0.1:2379" }, @@ -171,7 +177,7 @@ // // File Copare Method for comparing existing links from the internal pool to the published directory // // Only used when "linkMethod" is set to "copy" // // * md5 (default: compare md5 sum) - // // * size (only compare file size) + // // * size (compare file size) // "verifyMethod": "md5" // } }, @@ -325,7 +331,7 @@ // // // Endpoint URL // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://$accountName.blob.core.windows.net" + // // defaults to "https://.blob.core.windows.net" // "endpoint": "" // } }, @@ -336,9 +342,11 @@ // * local // * azure "packagePoolStorage": { - // Local Pool Path + // Local Pool "type": "local", - "path": "$ROOTDIR/pool" + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" // // Azure Azure Blob Storage Pool // "type": "azure", @@ -357,7 +365,7 @@ // // // Endpoint URL // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://$accountName.blob.core.windows.net" + // // defaults to "https://.blob.core.windows.net" // "endpoint": "" // } } diff --git a/main.go b/main.go index 337575a4..6a5247cc 100644 --- a/main.go +++ b/main.go @@ -13,12 +13,16 @@ import ( //go:embed VERSION var Version string +//go:embed debian/aptly.conf +var AptlyConf []byte + func main() { if Version == "" { Version = "unknown" } aptly.Version = Version + aptly.AptlyConf = AptlyConf os.Exit(cmd.Run(cmd.RootCommand(), os.Args[1:], true)) } diff --git a/main_test.go b/main_test.go index 81ef138d..750073b7 100644 --- a/main_test.go +++ b/main_test.go @@ -51,6 +51,7 @@ func TestRunMain(t *testing.T) { } aptly.Version = Version + aptly.AptlyConf = AptlyConf args := filterOutTestArgs(os.Args[1:]) root := cmd.RootCommand() diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 181efa4d..0d498e3f 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -1,39 +1,374 @@ +// vim: : filetype=json +// json configuration file with comments +// validate with: sed '/\/\//d' aptly.conf | json_pp { - "rootDir": "${HOME}/.aptly", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", + +// General +/////////// + + // Aptly storage directory + // Directory for storing: + // - downloaded packages (`rootDir`/pool) + // - database (`rootDir`/db) + // - published repositories (`rootDir`/public) + "rootDir": "~/.aptly", + + // Number of attempts to open database if it's locked by other instance + // * -1 (no retry) "databaseOpenAttempts": -1, - "architectures": [], - "dependencyFollowSuggests": false, - "dependencyFollowRecommends": false, - "dependencyFollowAllVariants": false, - "dependencyFollowSource": false, - "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": {}, - "skipLegacyPool": true, - "ppaDistributorID": "ubuntu", - "ppaCodename": "", - "skipContentsPublishing": false, - "skipBz2Publishing": false, - "FileSystemPublishEndpoints": {}, - "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {}, - "AzurePublishEndpoints": {}, - "AsyncAPI": false, - "enableMetricsEndpoint": false, - "logLevel": "debug", + + // Log Level + // * debug + // * info + // * warning + // * error + "logLevel": "info", + + // Log Format + // * default (text) + // * json "logFormat": "default", + + // Default Architectures + // empty array defaults to all available architectures + "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package + "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package + "dependencyFollowRecommends": false, + + // When dependency looks like `package-a | package-b`, follow both variants always + "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package + "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) + "dependencyVerboseResolve": false, + + // Specifies paramaters for short PPA url expansion + // empty defaults to output of `lsb_release` command + "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion + "ppaCodename": "", + + // OBSOLETE + // in aptly up to version 1.0.0, package files were stored in internal package pool + // with MD5-dervied path, since 1.1.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + + +// Aptly Server +//////////////// + + // Serve published repos as well as API "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + +// Database +//////////// + + // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd "databaseBackend": { - "type": "", - "url": "", + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db "dbPath": "" + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127.0.0.1:2379" }, - "enableSwaggerEndpoint": false + + +// Mirroring +///////////// + + // Downloader + // * "default" + // * "grab" (more robust) + "downloader": "default", + + // Number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // Limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // Number of retries for download attempts + "downloadRetries": 0, + + // Download source packages per default + "downloadSourcePackages": false, + + +// Signing +/////////// + + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) + "gpgProvider": "gpg", + + // Disable signing of published repositories + "gpgDisableSign": false, + + // Disable signature verification of remote repositories + "gpgDisableVerify": false, + + +// Publishing +////////////// + + // Do not publish Contents files + "skipContentsPublishing": false, + + // Do not create bz2 files + "skipBz2Publishing": false, + + +// Storage +/////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name + // and the following associated settings. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` + // with `endpoint-name` as the name given in the aptly configuration file. For example: + // + // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` + // + "FileSystemPublishEndpoints": { + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + // + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + // + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (compare file size) + // "verifyMethod": "md5" + // } + }, + + // S3 Endpoint Support + // + // cloud storage). First, publishing + // endpoints should be described in aptly configuration file. Each endpoint has name + // and associated settings. + // + // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main s3:test:` + // + "S3PublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Amazon region for S3 bucket + // "region": "us-east-1", + // + // // Bucket name + // "bucket": "test-bucket", + // + // // Endpoint (optional) + // // When using S3-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + // + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + // + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology). Useful values: `private` (default), `public-read` (public + // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required. + // "acl": "private", + // + // // Credentials (optional) + // // Amazon credentials to access S3 bucket. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + // + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + // + // // Encryption Method (optional) + // // Server-side encryption method, defaults to none. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + // + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + // + // // Disable MultiDel (optional) + // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + // + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores + // // which do not support SigV4, shouldn't be enabled for AWS + // "forceSigV2": false, + // + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non-AWS S3-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + // + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } + }, + + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift. First, + // publishing endpoints should be described in aptly configuration file. Each endpoint + // has name and associated settings. + // + // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot jessie-main swift:test:` + // + "SwiftPublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Container Name + // "container": "container1", + // + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version). + // // Example `http://identity.example.com:5000/v2.0` + // "authurl": "" + // } + }, + + // Azure Endpoint Support + // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage. First, publishing endpoints should be described in the aptly + // configuration file. Each endpoint has its name and associated settings. + "AzurePublishEndpoints": { + // // Endpoint Name + // "test": { + // + // // Container Name + // "container": "container1", + // + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + // + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + // + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool + "type": "local", + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "azure": { + // // Container Name + // "container": "pool1", + // + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + // + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + // + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" + // } + } + +// End of config } diff --git a/utils/config.go b/utils/config.go index b061649f..4665c830 100644 --- a/utils/config.go +++ b/utils/config.go @@ -215,6 +215,18 @@ func SaveConfig(filename string, config *ConfigStructure) error { return err } +// SaveConfigRaw write configuration to file +func SaveConfigRaw(filename string, conf []byte) error { + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(conf) + return err +} + // GetRootDir returns the RootDir with expanded ~ as home directory func (conf *ConfigStructure) GetRootDir() string { return strings.Replace(conf.RootDir, "~", os.Getenv("HOME"), 1) From 09c56342d289e49eb9f5211c43a67f8b458dfae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 21:01:54 +0100 Subject: [PATCH 45/64] fix json --- debian/aptly.conf | 86 ++++++++++++------------- system/t02_config/CreateConfigTest_gold | 86 ++++++++++++------------- 2 files changed, 82 insertions(+), 90 deletions(-) diff --git a/debian/aptly.conf b/debian/aptly.conf index 0d498e3f..6eeed77e 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -3,11 +3,10 @@ // validate with: sed '/\/\//d' aptly.conf | json_pp { -// General -/////////// +// Aptly Configuration File +//////////////////////////// - // Aptly storage directory - // Directory for storing: + // Aptly storage directory for: // - downloaded packages (`rootDir`/pool) // - database (`rootDir`/db) // - published repositories (`rootDir`/public) @@ -167,13 +166,13 @@ // "test1": { // // Directory for publishing // "rootDir": "/opt/srv/aptly_public", - // + // // File Link Method for linking files from the internal pool to the published directory // // * hardlink // // * symlink // // * copy // "linkMethod": "hardlink", - // + // // File Copare Method for comparing existing links from the internal pool to the published directory // // Only used when "linkMethod" is set to "copy" // // * md5 (default: compare md5 sum) @@ -196,24 +195,24 @@ "S3PublishEndpoints": { // // Endpoint Name // "test": { - // + // // Amazon region for S3 bucket // "region": "us-east-1", - // + // // Bucket name // "bucket": "test-bucket", - // + // // Endpoint (optional) // // When using S3-compatible cloud storage, specify hostname of service endpoint here, // // region is ignored if endpoint is set (set region to some human-readable name) // // (should be left blank for real Amazon S3) // "endpoint": "", - // + // // Prefix (optional) // // publishing under specified prefix in the bucket, defaults to // // no prefix (bucket root) // "prefix": "", - // + // // Default ACLs (optional) // // assign ACL to published files (one of the canned ACLs in Amazon // // terminology). Useful values: `private` (default), `public-read` (public @@ -221,24 +220,24 @@ // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), // // for private repositories special apt S3 transport is required. // "acl": "private", - // + // // Credentials (optional) // // Amazon credentials to access S3 bucket. If not supplied, // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` // // are used. // "awsAccessKeyID": "", // "awsSecretAccessKey": "", - // + // // Storage Class (optional) // // Amazon S3 storage class, defaults to `STANDARD`. Other values // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) // "storageClass": "STANDARD", - // + // // Encryption Method (optional) // // Server-side encryption method, defaults to none. Currently // // the only available encryption method is `AES256` // "encryptionMethod": "none", - // + // // Plus Workaround (optional) // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by // // creating two copies of package files with `+` in filename: one original @@ -247,22 +246,22 @@ // // would be stored twice. aptly might not cleanup files with spaces when published // // repository is dropped or updated (switched) to new version of repository (snapshot) // "plusWorkaround": false, - // + // // Disable MultiDel (optional) // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, // // enable this setting (file deletion would be slower with this setting enabled) // "disableMultiDel": false, - // + // // ForceSig2 (optional) // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores // // which do not support SigV4, shouldn't be enabled for AWS // "forceSigV2": false, - // + // // ForceVirtualHostedStyle (optional) // // Disable path style visit, useful with non-AWS S3-compatible object stores // // which only support virtual hosted style // "forceVirtualHostedStyle": false, - // + // // Debug (optional) // // Enables detailed request/response dump for each S3 operation // "debug": false @@ -281,12 +280,12 @@ // `aptly publish snapshot jessie-main swift:test:` // "SwiftPublishEndpoints": { - // // Endpoint Name + // Endpoint Name // "test": { - // + // // Container Name - // "container": "container1", - // + // "container": "taylor1", + // // Prefix (optional) // // Publish under specified prefix in the container, defaults to no prefix (container root) // "prefix": "", @@ -316,19 +315,19 @@ "AzurePublishEndpoints": { // // Endpoint Name // "test": { - // + // // Container Name // "container": "container1", - // + // // Prefix (optional) // // Publishing under specified prefix in the container, defaults to no prefix (container root) // "prefix": "", - // + // // Credentials // // Azure storage account access key to access blob storage // "accountName": "", // "accountKey": "", - // + // // Endpoint URL // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string // // defaults to "https://.blob.core.windows.net" @@ -350,24 +349,21 @@ // // Azure Azure Blob Storage Pool // "type": "azure", - // "azure": { - // // Container Name - // "container": "pool1", - // - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - // - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - // - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" } // End of config diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 0d498e3f..6eeed77e 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -3,11 +3,10 @@ // validate with: sed '/\/\//d' aptly.conf | json_pp { -// General -/////////// +// Aptly Configuration File +//////////////////////////// - // Aptly storage directory - // Directory for storing: + // Aptly storage directory for: // - downloaded packages (`rootDir`/pool) // - database (`rootDir`/db) // - published repositories (`rootDir`/public) @@ -167,13 +166,13 @@ // "test1": { // // Directory for publishing // "rootDir": "/opt/srv/aptly_public", - // + // // File Link Method for linking files from the internal pool to the published directory // // * hardlink // // * symlink // // * copy // "linkMethod": "hardlink", - // + // // File Copare Method for comparing existing links from the internal pool to the published directory // // Only used when "linkMethod" is set to "copy" // // * md5 (default: compare md5 sum) @@ -196,24 +195,24 @@ "S3PublishEndpoints": { // // Endpoint Name // "test": { - // + // // Amazon region for S3 bucket // "region": "us-east-1", - // + // // Bucket name // "bucket": "test-bucket", - // + // // Endpoint (optional) // // When using S3-compatible cloud storage, specify hostname of service endpoint here, // // region is ignored if endpoint is set (set region to some human-readable name) // // (should be left blank for real Amazon S3) // "endpoint": "", - // + // // Prefix (optional) // // publishing under specified prefix in the bucket, defaults to // // no prefix (bucket root) // "prefix": "", - // + // // Default ACLs (optional) // // assign ACL to published files (one of the canned ACLs in Amazon // // terminology). Useful values: `private` (default), `public-read` (public @@ -221,24 +220,24 @@ // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), // // for private repositories special apt S3 transport is required. // "acl": "private", - // + // // Credentials (optional) // // Amazon credentials to access S3 bucket. If not supplied, // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` // // are used. // "awsAccessKeyID": "", // "awsSecretAccessKey": "", - // + // // Storage Class (optional) // // Amazon S3 storage class, defaults to `STANDARD`. Other values // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) // "storageClass": "STANDARD", - // + // // Encryption Method (optional) // // Server-side encryption method, defaults to none. Currently // // the only available encryption method is `AES256` // "encryptionMethod": "none", - // + // // Plus Workaround (optional) // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by // // creating two copies of package files with `+` in filename: one original @@ -247,22 +246,22 @@ // // would be stored twice. aptly might not cleanup files with spaces when published // // repository is dropped or updated (switched) to new version of repository (snapshot) // "plusWorkaround": false, - // + // // Disable MultiDel (optional) // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, // // enable this setting (file deletion would be slower with this setting enabled) // "disableMultiDel": false, - // + // // ForceSig2 (optional) // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores // // which do not support SigV4, shouldn't be enabled for AWS // "forceSigV2": false, - // + // // ForceVirtualHostedStyle (optional) // // Disable path style visit, useful with non-AWS S3-compatible object stores // // which only support virtual hosted style // "forceVirtualHostedStyle": false, - // + // // Debug (optional) // // Enables detailed request/response dump for each S3 operation // "debug": false @@ -281,12 +280,12 @@ // `aptly publish snapshot jessie-main swift:test:` // "SwiftPublishEndpoints": { - // // Endpoint Name + // Endpoint Name // "test": { - // + // // Container Name - // "container": "container1", - // + // "container": "taylor1", + // // Prefix (optional) // // Publish under specified prefix in the container, defaults to no prefix (container root) // "prefix": "", @@ -316,19 +315,19 @@ "AzurePublishEndpoints": { // // Endpoint Name // "test": { - // + // // Container Name // "container": "container1", - // + // // Prefix (optional) // // Publishing under specified prefix in the container, defaults to no prefix (container root) // "prefix": "", - // + // // Credentials // // Azure storage account access key to access blob storage // "accountName": "", // "accountKey": "", - // + // // Endpoint URL // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string // // defaults to "https://.blob.core.windows.net" @@ -350,24 +349,21 @@ // // Azure Azure Blob Storage Pool // "type": "azure", - // "azure": { - // // Container Name - // "container": "pool1", - // - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - // - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - // - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" } // End of config From 1301847b7e6b1be9e5a58fc6a56714d008cf4be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 11:42:24 +0100 Subject: [PATCH 46/64] update CONTRIBUTING --- CONTRIBUTING.md | 66 +++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4ab7e55..780dbf5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,15 +110,13 @@ make docker-build To run aptly commands in the development docker container, run: ``` -make docker-aptly +make docker-shell ``` Example: ``` -$ make docker-aptly -bash: cannot set terminal process group (16): Inappropriate ioctl for device -bash: no job control in this shell -aptly@b43e8473ef81:/app$ aptly version +$ make docker-shell +aptly@b43e8473ef81:/work/src$ aptly version aptly version: 1.5.0+189+g0fc90dff ``` @@ -152,42 +150,28 @@ Run `make help` for more information. This section describes local setup to start contributing to aptly. -#### Go & Python - -You would need `Go` (latest version is recommended) and `Python` 3.9 (or newer, the CI currently tests against 3.11). - -If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform -initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that. - -Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose -to prepend it or to skip this test if you're security conscious. - #### Dependencies -You would need some additional tools and Python virtual environment to run tests and checks, install them with: +Building aptly requires go version 1.22. - make prepare dev system/env +On Debian bookworm with backports enabled, go can be installed with: -This is usually one-time action. - -Aptly is using Go modules to manage dependencies, download modules using: - - make modules + apt install -t bookworm-backports golang-go #### Building -If you want to build aptly binary from your current source tree, run: +To build aptly, run: + + make build + +Run aptly: + + build/aptly + +To install aptly into `$GOPATH/bin`, run: make install -This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with: - - aptly - -Or, if it's not on your path: - - ~/go/bin/aptly - #### Unit-tests aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any @@ -244,26 +228,6 @@ There are some packages available under `system/files/` directory which are used this default location. You can run aptly under different user or by using non-default config location with non-default aptly root directory. -#### Style Checks - -Style checks could be run with: - - make check - -aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration -for the linter could be found in [.golangci.yml](.golangci.yml) file. - -Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8). - -#### Vendored Code - -aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source -repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored -packages over packages in `$GOPATH`. - -If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep). -Usually all you need is `dep ensure` or `dep ensure -update`. - ### man Page aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source From 3d8968eff34f5a86519448cefc68ead8ddfef600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 11:42:39 +0100 Subject: [PATCH 47/64] swagger: install native swag --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ff224561..28bb95a8 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ version: ## Print aptly version swagger-install: # Install swag - @test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest + @test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest # Generate swagger.conf cp docs/swagger.conf.tpl docs/swagger.conf echo "// @version $(VERSION)" >> docs/swagger.conf @@ -79,7 +79,9 @@ build: prepare swagger ## Build aptly install: @echo "\e[33m\e[1mBuilding aptly ...\e[0m" - go generate + # go generate + @go generate + # go install -v @out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi test: prepare swagger etcd-install ## Run unit tests From 280563caa876b3b7d97b96bd67f1910435c12114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 19:03:21 +0800 Subject: [PATCH 48/64] unit-tests: allow running as user --- Makefile | 2 +- system/lib.py | 4 ++-- system/t13_etcd/install-etcd.sh | 15 +++++++++++---- system/t13_etcd/start-etcd.sh | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 28bb95a8..3635a0e9 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ swagger: swagger-install etcd-install: # Install etcd - test -d /srv/etcd || system/t13_etcd/install-etcd.sh + test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh flake8: ## run flake8 on system test python files flake8 system/ diff --git a/system/lib.py b/system/lib.py index 2918d805..caba606c 100644 --- a/system/lib.py +++ b/system/lib.py @@ -242,7 +242,7 @@ class BaseTest(object): os.environ["HOME"], self.aptlyDir, "pool"), ignore=shutil.ignore_patterns(".git")) if self.databaseType == "etcd": - if not os.path.exists("/srv/etcd"): + if not os.path.exists("/tmp/aptly-etcd"): self.run_cmd([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/install-etcd.sh")]) if self.fixtureDB and self.databaseType != "etcd": @@ -258,7 +258,7 @@ class BaseTest(object): if self.fixtureDB: print("import etcd") - self.run_cmd(["/srv/etcd/etcdctl", "--data-dir=/tmp/etcd-data", "snapshot", "restore", os.path.join(os.environ["HOME"], "etcd.db")]) + self.run_cmd(["/tmp/aptly-etcd/etcdctl", "--data-dir=/tmp/aptly-etcd-data", "snapshot", "restore", os.path.join(os.environ["HOME"], "etcd.db")]) print("starting etcd") self.EtcdServer = self._start_process([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/start-etcd.sh")], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/system/t13_etcd/install-etcd.sh b/system/t13_etcd/install-etcd.sh index 163a3f85..1521511d 100755 --- a/system/t13_etcd/install-etcd.sh +++ b/system/t13_etcd/install-etcd.sh @@ -4,9 +4,16 @@ ETCD_VER=v3.5.2 DOWNLOAD_URL=https://storage.googleapis.com/etcd -if [ ! -e /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ]; then - curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +ARCH="" +case $(uname -m) in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + *) echo "unsupported cpu arch"; exit 1 ;; +esac + +if [ ! -e /tmp/etcd-${ETCD_VER}-linux-$ARCH.tar.gz ]; then + curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-$ARCH.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-$ARCH.tar.gz fi -mkdir /srv/etcd -tar xf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /srv/etcd --strip-components=1 +mkdir /tmp/aptly-etcd +tar xf /tmp/etcd-${ETCD_VER}-linux-$ARCH.tar.gz -C /tmp/aptly-etcd --strip-components=1 diff --git a/system/t13_etcd/start-etcd.sh b/system/t13_etcd/start-etcd.sh index ae6c0a45..05908def 100755 --- a/system/t13_etcd/start-etcd.sh +++ b/system/t13_etcd/start-etcd.sh @@ -16,7 +16,7 @@ finish() } trap finish INT -/srv/etcd/etcd --max-request-bytes '1073741824' --data-dir /tmp/etcd-data & +/tmp/aptly-etcd/etcd --max-request-bytes '1073741824' --data-dir /tmp/aptly-etcd-data & echo $! > /tmp/etcd.pid etcdpid=`cat /tmp/etcd.pid` wait $etcdpid From e92afd8f78b3d5df0c535cfa4d46d55ed0fd5581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 20:05:45 +0800 Subject: [PATCH 49/64] fix unit tests on arm and fix etcd data dir --- Makefile | 4 ++-- files/package_pool_test.go | 6 +++--- system/lib.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3635a0e9..3661948b 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ install: test: prepare swagger etcd-install ## Run unit tests @echo "\e[33m\e[1mStarting etcd ...\e[0m" - @mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 & + @mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 & @echo "\e[33m\e[1mRunning go test ...\e[0m" go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret @echo "\e[33m\e[1mStopping etcd ...\e[0m" @pid=`cat /tmp/etcd.pid`; kill $$pid - @rm -f /tmp/etcd-data/etcd.log + @rm -f /tmp/aptly-etcd-data/etcd.log @ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret system-test: prepare swagger etcd-install ## Run system tests diff --git a/files/package_pool_test.go b/files/package_pool_test.go index f06d6bcf..4cd9476b 100644 --- a/files/package_pool_test.go +++ b/files/package_pool_test.go @@ -120,7 +120,7 @@ func (s *PackagePoolSuite) TestImportOk(c *C) { if isSameDevice(s) { c.Check(info.Sys().(*syscall.Stat_t).Nlink > 1, Equals, true) } else { - c.Check(info.Sys().(*syscall.Stat_t).Nlink, Equals, uint64(1)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink == 1, Equals, true) } // import as different name @@ -359,7 +359,7 @@ func (s *PackagePoolSuite) TestLink(c *C) { if isSameDevice(s) { c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true) } else { - c.Check(info.Sys().(*syscall.Stat_t).Nlink, Equals, uint64(2)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink == 2, Equals, true) } } @@ -377,7 +377,7 @@ func (s *PackagePoolSuite) TestSymlink(c *C) { if isSameDevice(s) { c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true) } else { - c.Check(info.Sys().(*syscall.Stat_t).Nlink, Equals, uint64(1)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink == 1, Equals, true) } info, err = os.Lstat(dstPath) diff --git a/system/lib.py b/system/lib.py index caba606c..1287f81d 100644 --- a/system/lib.py +++ b/system/lib.py @@ -253,8 +253,8 @@ class BaseTest(object): self.shutdown_etcd() # remove existing database - if os.path.exists("/tmp/etcd-data"): - shutil.rmtree("/tmp/etcd-data") + if os.path.exists("/tmp/aptly-etcd-data"): + shutil.rmtree("/tmp/aptly-etcd-data") if self.fixtureDB: print("import etcd") From e319f3cd14a20ef9a40478afa3f78f4dfa63bcac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 11 Dec 2024 11:19:21 +0100 Subject: [PATCH 50/64] update doc make descrptions consistent --- api/api.go | 6 +++--- api/mirror.go | 4 ++-- api/packages.go | 6 +++--- api/publish.go | 6 +++--- api/repos.go | 22 +++++++++++----------- api/snapshot.go | 6 +++--- api/task.go | 18 +++++++++--------- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/api/api.go b/api/api.go index adeef9d9..1c9883f9 100644 --- a/api/api.go +++ b/api/api.go @@ -28,7 +28,7 @@ type aptlyVersion struct { Version string `json:"Version"` } -// @Summary Aptly version +// @Summary Aptly Version // @Description **Get aptly version** // @Description // @Description **Example:** @@ -49,7 +49,7 @@ type aptlyStatus struct { Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"` } -// @Summary Ready State +// @Summary Get Ready State // @Description **Get aptly ready state** // @Description // @Description Return aptly ready state: @@ -71,7 +71,7 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) { } } -// @Summary Health State +// @Summary Get Health State // @Description **Get aptly health state** // @Description // @Description Return aptly health state: diff --git a/api/mirror.go b/api/mirror.go index 48a505fe..b9adf24c 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -31,7 +31,7 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) { return verifier, nil } -// @Summary Get Mirrors +// @Summary List Mirrors // @Description **Show list of currently available mirrors** // @Description Each mirror is returned as in “show” API. // @Tags Mirrors @@ -209,7 +209,7 @@ func apiMirrorsDrop(c *gin.Context) { }) } -// @Summary Show Mirror +// @Summary Get Mirror Info // @Description **Get mirror information by name** // @Tags Mirrors // @Param name path string true "mirror name" diff --git a/api/packages.go b/api/packages.go index 631d3014..e29d1fa5 100644 --- a/api/packages.go +++ b/api/packages.go @@ -5,7 +5,7 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary Show packages +// @Summary Get Package Info // @Description **Show information about package by package key** // @Description Package keys could be obtained from various GET .../packages APIs. // @Tags Packages @@ -25,8 +25,8 @@ func apiPackagesShow(c *gin.Context) { c.JSON(200, p) } -// @Summary Get packages -// @Description Get list of packages. +// @Summary List Packages +// @Description **Get list of packages** // @Tags Packages // @Consume json // @Produce json diff --git a/api/publish.go b/api/publish.go index 453f9a3d..e19afb17 100644 --- a/api/publish.go +++ b/api/publish.go @@ -635,7 +635,7 @@ func apiPublishAddSource(c *gin.Context) { }) } -// @Summary List pending changes +// @Summary List Pending Changes // @Description **List source component changes to be applied** // @Description // @Description Return added, removed or changed components of snapshots or local repository to be published. @@ -749,8 +749,8 @@ func apiPublishSetSources(c *gin.Context) { }) } -// @Summary Drop pending changes -// @Description **Drop pending source component changes of a published repository** +// @Summary Discard Pending Changes +// @Description **Discard pending source component changes of a published repository** // @Description // @Description Remove all pending changes what would be applied with a subsequent publish update call (i.e. `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description diff --git a/api/repos.go b/api/repos.go index 3acc29ea..4359b81e 100644 --- a/api/repos.go +++ b/api/repos.go @@ -18,7 +18,7 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary Serve HTML listing of repo +// @Summary Serve HTML Listing // @Description If ServeInAPIMode is enabled in aptly config, // @Description this endpoint is enabled which returns an HTML listing of each repo that can be browsed // @Tags Repos @@ -41,7 +41,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H } } -// @Summary Serve package in API mode +// @Summary Serve Packages // @Description If ServeInAPIMode is enabled in aptly config, // @Description this api serves a specified package from storage // @Tags Repos @@ -64,7 +64,7 @@ func reposServeInAPIMode(c *gin.Context) { c.FileFromFS(pkgpath, http.Dir(publicPath)) } -// @Summary Get repos +// @Summary List Repositories // @Description **Get list of available repos** // @Description Each repo is returned as in “show” API. // @Tags Repos @@ -97,7 +97,7 @@ type repoCreateParams struct { FromSnapshot string ` json:"FromSnapshot" example:"snapshot1"` } -// @Summary Create repository +// @Summary Create Repository // @Description **Create a local repository** // @Description // @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot. @@ -175,7 +175,7 @@ type reposEditParams struct { DefaultComponent *string ` json:"DefaultComponent" example:""` } -// @Summary Update repo +// @Summary Update Repository // @Description **Update local repository meta information** // @Tags Repos // @Produce json @@ -228,7 +228,7 @@ func apiReposEdit(c *gin.Context) { } // GET /api/repos/:name -// @Summary Get repository info by name +// @Summary Get Repository Info // @Description Returns basic information about local repository. // @Tags Repos // @Produce json @@ -249,7 +249,7 @@ func apiReposShow(c *gin.Context) { c.JSON(200, repo) } -// @Summary Drop Repository +// @Summary Delete Repository // @Description Drop/delete a repo // @Description Cannot drop repos that are published. // @Description Needs force=1 to drop repos used as source by other repos. @@ -442,7 +442,7 @@ func apiReposPackagesDelete(c *gin.Context) { }) } -// @Summary Add packages from uploaded file +// @Summary Add Uploaded File // @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically. // @Description Adding same package to local repository is not an error. // @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import). @@ -462,7 +462,7 @@ func apiReposPackageFromFile(c *gin.Context) { apiReposPackageFromDir(c) } -// @Summary Add packages from uploaded directory +// @Summary Add Uploaded Directory // @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically. // @Description Adding same package to local repository is not an error. // @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import). @@ -752,7 +752,7 @@ func apiReposCopyPackage(c *gin.Context) { }) } -// @Summary Include Packages from File Upload +// @Summary Include File from Directory // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos // @Produce json @@ -780,7 +780,7 @@ type reposIncludePackageFromDirResponse struct { FailedFiles []string } -// @Summary Include Packages from Dir Upload +// @Summary Include Directory // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos // @Produce json diff --git a/api/snapshot.go b/api/snapshot.go index d020c3dc..4af745e4 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -341,7 +341,7 @@ func apiSnapshotsUpdate(c *gin.Context) { }) } -// @Summary Get snapshot information +// @Summary Get Snapshot Info // @Description **Query detailed information about a snapshot by name** // @Tags Snapshots // @Param name path string true "Name of the snapshot" @@ -369,8 +369,8 @@ func apiSnapshotsShow(c *gin.Context) { c.JSON(200, snapshot) } -// @Summary Drop Snapshot -// @Description **Drop/delete snapshot by name** +// @Summary Delete Snapshot +// @Description **Delete snapshot by name** // @Description Cannot drop snapshots that are published. // @Description Needs force=1 to drop snapshots used as source by other snapshots. // @Tags Snapshots diff --git a/api/task.go b/api/task.go index 1d136325..1339f96a 100644 --- a/api/task.go +++ b/api/task.go @@ -7,7 +7,7 @@ import ( "github.com/gin-gonic/gin" ) -// @Summary Get tasks +// @Summary List Tasks // @Description **Get list of available tasks. Each task is returned as in “show” API** // @Tags Tasks // @Produce json @@ -18,7 +18,7 @@ func apiTasksList(c *gin.Context) { c.JSON(200, list.GetTasks()) } -// @Summary Clear finished and failed tasks +// @Summary Clear Tasks // @Description **Removes finished and failed tasks from internal task list** // @Tags Tasks // @Produce json @@ -30,7 +30,7 @@ func apiTasksClear(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary Wait for task completion +// @Summary Wait for all Tasks // @Description **Waits for and returns when all running tasks are complete** // @Tags Tasks // @Produce json @@ -42,7 +42,7 @@ func apiTasksWait(c *gin.Context) { c.JSON(200, gin.H{}) } -// @Summary Wait for task to process +// @Summary Wait for Task // @Description **Waits for and returns when given Task ID is complete** // @Tags Tasks // @Produce json @@ -68,7 +68,7 @@ func apiTasksWaitForTaskByID(c *gin.Context) { c.JSON(200, task) } -// @Summary Return task information +// @Summary Get Task Info // @Description **Return task information for a given ID** // @Tags Tasks // @Produce plain @@ -95,7 +95,7 @@ func apiTasksShow(c *gin.Context) { c.JSON(200, task) } -// @Summary Return task output +// @Summary Get Task Output // @Description **Return task output for a given ID** // @Tags Tasks // @Produce plain @@ -122,7 +122,7 @@ func apiTasksOutputShow(c *gin.Context) { c.JSON(200, output) } -// @Summary Return task detail +// @Summary Get Task Details // @Description **Return task detail for a given ID** // @Tags Tasks // @Produce json @@ -149,7 +149,7 @@ func apiTasksDetailShow(c *gin.Context) { c.JSON(200, detail) } -// @Summary Return task return value (status code) +// @Summary Get Task Return Value // @Description **Return task return value (status code) by given ID** // @Tags Tasks // @Produce plain @@ -175,7 +175,7 @@ func apiTasksReturnValueShow(c *gin.Context) { c.JSON(200, output) } -// @Summary Delete task +// @Summary Delete Task // @Description **Delete completed task by given ID. Does not stop task execution** // @Tags Tasks // @Produce json From a880a88fc08f529b9d4dc9533084ad84aa524117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 11:10:21 +0100 Subject: [PATCH 51/64] yaml config --- go.mod | 2 +- utils/config.go | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 91f7e565..35786ebe 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,6 @@ require ( google.golang.org/grpc v1.64.1 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -129,4 +128,5 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 + gopkg.in/yaml.v3 v3.0.1 ) diff --git a/utils/config.go b/utils/config.go index 4665c830..20849450 100644 --- a/utils/config.go +++ b/utils/config.go @@ -8,6 +8,8 @@ import ( "strings" "github.com/DisposaBoy/JsonConfigReader" + "gopkg.in/yaml.v3" + ) // ConfigStructure is structure of main configuration @@ -194,8 +196,19 @@ func LoadConfig(filename string, config *ConfigStructure) error { } defer f.Close() - dec := json.NewDecoder(JsonConfigReader.New(f)) - return dec.Decode(&config) + dec_json := json.NewDecoder(JsonConfigReader.New(f)) + if err = dec_json.Decode(&config); err != nil { + f.Seek(0, 0) + dec_yaml := yaml.NewDecoder(f) + if err = dec_yaml.Decode(&config); err != nil { + fmt.Errorf("config file %s is not valid yaml or json\n", filename) + } else { + fmt.Printf("config file %s format is yaml\n", filename) + } + } else { + fmt.Printf("config file %s format is json\n", filename) + } + return err } // SaveConfig write configuration to json file From 0e0189f0ebbf0a19404628b01e65cbd54734feee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 20:56:17 +0100 Subject: [PATCH 52/64] use yaml config file --- cmd/config_show.go | 22 ++- debian/aptly.conf.yaml | 339 +++++++++++++++++++++++++++++++++++++++++ man/aptly.1.ronn.tmpl | 1 + utils/config.go | 207 ++++++++++++++----------- 4 files changed, 479 insertions(+), 90 deletions(-) create mode 100644 debian/aptly.conf.yaml diff --git a/cmd/config_show.go b/cmd/config_show.go index 66adc8f0..56ab33e8 100644 --- a/cmd/config_show.go +++ b/cmd/config_show.go @@ -5,19 +5,30 @@ import ( "fmt" "github.com/smira/commander" + "gopkg.in/yaml.v3" ) func aptlyConfigShow(_ *commander.Command, _ []string) error { + show_yaml := context.Flags().Lookup("yaml").Value.Get().(bool) config := context.Config() - prettyJSON, err := json.MarshalIndent(config, "", " ") - if err != nil { - return fmt.Errorf("unable to dump the config file: %s", err) + if show_yaml { + yamlData, err := yaml.Marshal(&config) + if err != nil { + return fmt.Errorf("error marshaling to YAML: %s", err) + } + + fmt.Println(string(yamlData)) + } else { + prettyJSON, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("unable to dump the config file: %s", err) + } + + fmt.Println(string(prettyJSON)) } - fmt.Println(string(prettyJSON)) - return nil } @@ -35,5 +46,6 @@ Example: `, } + cmd.Flag.Bool("yaml", false, "show yaml config") return cmd } diff --git a/debian/aptly.conf.yaml b/debian/aptly.conf.yaml new file mode 100644 index 00000000..89c70fa6 --- /dev/null +++ b/debian/aptly.conf.yaml @@ -0,0 +1,339 @@ +# Aptly Configuration File +########################### + +# Aptly storage directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly + +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 + +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info + +# Log Format +# * default (text) +# * json +log_format: default + +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 + + +# Dependency following +####################### + +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false + +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false + +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false + +# Follow dependency from binary package to source package +dep_follow_source: false + +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false + + +# PPA +###### + +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu + +# Codename for short PPA url expansion +ppa_codename: "" + +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true + + +# Aptly Server +############### + +# Serve published repos as well as API +serve_in_api_mode: false + +# Enable metrics for Prometheus client +enable_metrics_endpoint: false + +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 0b319c74..a04ac52d 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -48,6 +48,7 @@ Configuration file is stored in JSON format (default values shown below): "gpgProvider": "gpg", "downloadSourcePackages": false, "packagePoolStorage": { + "type": "local", "path": "$ROOTDIR/pool", "azure": { "accountName": "", diff --git a/utils/config.go b/utils/config.go index 20849450..d0e48e94 100644 --- a/utils/config.go +++ b/utils/config.go @@ -9,55 +9,54 @@ import ( "github.com/DisposaBoy/JsonConfigReader" "gopkg.in/yaml.v3" - ) // ConfigStructure is structure of main configuration type ConfigStructure struct { // nolint: maligned - RootDir string `json:"rootDir"` - DownloadConcurrency int `json:"downloadConcurrency"` - DownloadLimit int64 `json:"downloadSpeedLimit"` - DownloadRetries int `json:"downloadRetries"` - Downloader string `json:"downloader"` - DatabaseOpenAttempts int `json:"databaseOpenAttempts"` - Architectures []string `json:"architectures"` - DepFollowSuggests bool `json:"dependencyFollowSuggests"` - DepFollowRecommends bool `json:"dependencyFollowRecommends"` - DepFollowAllVariants bool `json:"dependencyFollowAllVariants"` - DepFollowSource bool `json:"dependencyFollowSource"` - DepVerboseResolve bool `json:"dependencyVerboseResolve"` - GpgDisableSign bool `json:"gpgDisableSign"` - GpgDisableVerify bool `json:"gpgDisableVerify"` - GpgProvider string `json:"gpgProvider"` - DownloadSourcePackages bool `json:"downloadSourcePackages"` - PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage"` - SkipLegacyPool bool `json:"skipLegacyPool"` - PpaDistributorID string `json:"ppaDistributorID"` - PpaCodename string `json:"ppaCodename"` - SkipContentsPublishing bool `json:"skipContentsPublishing"` - SkipBz2Publishing bool `json:"skipBz2Publishing"` - FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints"` - S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints"` - SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"` - AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints"` - AsyncAPI bool `json:"AsyncAPI"` - EnableMetricsEndpoint bool `json:"enableMetricsEndpoint"` - LogLevel string `json:"logLevel"` - LogFormat string `json:"logFormat"` - ServeInAPIMode bool `json:"serveInAPIMode"` - DatabaseBackend DBConfig `json:"databaseBackend"` - EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint"` + RootDir string `json:"rootDir" yaml:"root_dir"` + DownloadConcurrency int `json:"downloadConcurrency" yaml:"download_concurrency"` + DownloadLimit int64 `json:"downloadSpeedLimit" yaml:"download_limit"` + DownloadRetries int `json:"downloadRetries" yaml:"download_retries"` + Downloader string `json:"downloader" yaml:"downloader"` + DatabaseOpenAttempts int `json:"databaseOpenAttempts" yaml:"database_open_attempts"` + Architectures []string `json:"architectures" yaml:"architectures"` + DepFollowSuggests bool `json:"dependencyFollowSuggests" yaml:"dep_follow_suggests"` + DepFollowRecommends bool `json:"dependencyFollowRecommends" yaml:"dep_follow_recommends"` + DepFollowAllVariants bool `json:"dependencyFollowAllVariants" yaml:"dep_follow_all_variants"` + DepFollowSource bool `json:"dependencyFollowSource" yaml:"dep_follow_source"` + DepVerboseResolve bool `json:"dependencyVerboseResolve" yaml:"dep_verboseresolve"` + GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` + GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` + GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` + DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"` + PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage" yaml:"packagepool_storage"` + SkipLegacyPool bool `json:"skipLegacyPool" yaml:"skip_legacy_pool"` + PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"` + PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"` + SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"` + SkipBz2Publishing bool `json:"skipBz2Publishing" yaml:"skip_bz2_publishing"` + FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints" yaml:"filesystem_publish_endpoints"` + S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints" yaml:"s3_publish_endpoints"` + SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints" yaml:"swift_publish_endpoints"` + AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints" yaml:"azure_publish_endpoints"` + AsyncAPI bool `json:"AsyncAPI" yaml:"async_api"` + EnableMetricsEndpoint bool `json:"enableMetricsEndpoint" yaml:"enable_metrics_endpoint"` + LogLevel string `json:"logLevel" yaml:"log_level"` + LogFormat string `json:"logFormat" yaml:"log_format"` + ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"` + DatabaseBackend DBConfig `json:"databaseBackend" yaml:"database_backend"` + EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint" yaml:"enable_swagger_endpoint"` } // DBConfig type DBConfig struct { - Type string `json:"type"` - URL string `json:"url"` - DbPath string `json:"dbPath"` + Type string `json:"type" yaml:"type"` + URL string `json:"url" yaml:"url"` + DbPath string `json:"dbPath" yaml:"db_path"` } type LocalPoolStorage struct { - Path string `json:"path,omitempty"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` } type PackagePoolStorage struct { @@ -86,6 +85,26 @@ func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { } } +func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) error { + var discriminator struct { + Type string `yaml:"type"` + } + if err := unmarshal(&discriminator); err != nil { + return err + } + + switch discriminator.Type { + case "azure": + pool.Azure = &AzureEndpoint{} + return unmarshal(&pool.Azure) + case "local", "": + pool.Local = &LocalPoolStorage{} + return unmarshal(&pool.Local) + default: + return fmt.Errorf("unknown pool storage type: %s", discriminator.Type) + } +} + func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { var wrapper struct { Type string `json:"type,omitempty"` @@ -104,54 +123,72 @@ func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { return json.Marshal(wrapper) } +func (pool PackagePoolStorage) MarshalYAML() (interface{}, error) { + var wrapper struct { + Type string `yaml:"type,omitempty"` + *LocalPoolStorage `yaml:",inline"` + *AzureEndpoint `yaml:",inline"` + } + + if pool.Azure != nil { + wrapper.Type = "azure" + wrapper.AzureEndpoint = pool.Azure + } else if pool.Local.Path != "" { + wrapper.Type = "local" + wrapper.LocalPoolStorage = pool.Local + } + + return wrapper, nil +} + // FileSystemPublishRoot describes single filesystem publishing entry point type FileSystemPublishRoot struct { - RootDir string `json:"rootDir"` - LinkMethod string `json:"linkMethod"` - VerifyMethod string `json:"verifyMethod"` + RootDir string `json:"rootDir" yaml:"root_dir"` + LinkMethod string `json:"linkMethod" yaml:"link_method"` + VerifyMethod string `json:"verifyMethod" yaml:"verify_method"` } // S3PublishRoot describes single S3 publishing entry point type S3PublishRoot struct { - Region string `json:"region"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - AccessKeyID string `json:"awsAccessKeyID"` - SecretAccessKey string `json:"awsSecretAccessKey"` - SessionToken string `json:"awsSessionToken"` - Prefix string `json:"prefix"` - ACL string `json:"acl"` - StorageClass string `json:"storageClass"` - EncryptionMethod string `json:"encryptionMethod"` - PlusWorkaround bool `json:"plusWorkaround"` - DisableMultiDel bool `json:"disableMultiDel"` - ForceSigV2 bool `json:"forceSigV2"` - ForceVirtualHostedStyle bool `json:"forceVirtualHostedStyle"` - Debug bool `json:"debug"` + Region string `json:"region" yaml:"region"` + Bucket string `json:"bucket" yaml:"bucket"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + AccessKeyID string `json:"awsAccessKeyID" yaml:"access_key_id"` + SecretAccessKey string `json:"awsSecretAccessKey" yaml:"secret_access_key"` + SessionToken string `json:"awsSessionToken" yaml:"session_token"` + Prefix string `json:"prefix" yaml:"prefix"` + ACL string `json:"acl" yaml:"acl"` + StorageClass string `json:"storageClass" yaml:"storage_class"` + EncryptionMethod string `json:"encryptionMethod" yaml:"encryption_method"` + PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"` + DisableMultiDel bool `json:"disableMultiDel" yaml:"disable_multidel"` + ForceSigV2 bool `json:"forceSigV2" yaml:"force_sigv2"` + ForceVirtualHostedStyle bool `json:"forceVirtualHostedStyle" yaml:"force_virtualhosted_style"` + Debug bool `json:"debug" yaml:"debug"` } // SwiftPublishRoot describes single OpenStack Swift publishing entry point type SwiftPublishRoot struct { - UserName string `json:"osname"` - Password string `json:"password"` - AuthURL string `json:"authurl"` - Tenant string `json:"tenant"` - TenantID string `json:"tenantid"` - Domain string `json:"domain"` - DomainID string `json:"domainid"` - TenantDomain string `json:"tenantdomain"` - TenantDomainID string `json:"tenantdomainid"` - Prefix string `json:"prefix"` - Container string `json:"container"` + UserName string `json:"osname" yaml:"username"` + Password string `json:"password" yaml:"password"` + AuthURL string `json:"authurl" yaml:"auth_url"` + Tenant string `json:"tenant" yaml:"tenant"` + TenantID string `json:"tenantid" yaml:"tenant_id"` + Domain string `json:"domain" yaml:"domain"` + DomainID string `json:"domainid" yaml:"domain_id"` + TenantDomain string `json:"tenantdomain" yaml:"tenant_domain"` + TenantDomainID string `json:"tenantdomainid" yaml:"tenant_domain_id"` + Prefix string `json:"prefix" yaml:"prefix"` + Container string `json:"container" yaml:"container"` } // AzureEndpoint describes single Azure publishing entry point type AzureEndpoint struct { - AccountName string `json:"accountName"` - AccountKey string `json:"accountKey"` - Container string `json:"container"` - Prefix string `json:"prefix"` - Endpoint string `json:"endpoint"` + AccountName string `json:"accountName" yaml:"account_name"` + AccountKey string `json:"accountKey" yaml:"account_key"` + Container string `json:"container" yaml:"container"` + Prefix string `json:"prefix" yaml:"prefix"` + Endpoint string `json:"endpoint" yaml:"endpoint"` } // Config is configuration for aptly, shared by all modules @@ -182,7 +219,7 @@ var Config = ConfigStructure{ AzurePublishRoots: map[string]AzureEndpoint{}, AsyncAPI: false, EnableMetricsEndpoint: false, - LogLevel: "debug", + LogLevel: "info", LogFormat: "default", ServeInAPIMode: false, EnableSwaggerEndpoint: false, @@ -196,18 +233,18 @@ func LoadConfig(filename string, config *ConfigStructure) error { } defer f.Close() - dec_json := json.NewDecoder(JsonConfigReader.New(f)) - if err = dec_json.Decode(&config); err != nil { - f.Seek(0, 0) - dec_yaml := yaml.NewDecoder(f) - if err = dec_yaml.Decode(&config); err != nil { - fmt.Errorf("config file %s is not valid yaml or json\n", filename) - } else { - fmt.Printf("config file %s format is yaml\n", filename) - } - } else { - fmt.Printf("config file %s format is json\n", filename) - } + dec_json := json.NewDecoder(JsonConfigReader.New(f)) + if err = dec_json.Decode(&config); err != nil { + f.Seek(0, 0) + dec_yaml := yaml.NewDecoder(f) + if err = dec_yaml.Decode(&config); err != nil { + fmt.Errorf("config file %s is not valid yaml or json\n", filename) + } else { + fmt.Printf("config file %s format is yaml\n", filename) + } + } else { + fmt.Printf("config file %s format is json\n", filename) + } return err } From 87a36633e9a55026fd1d15911bff704acb996c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 21:27:38 +0100 Subject: [PATCH 53/64] group and order config items and fix lint --- debian/aptly.conf.yaml | 34 ++++++------- utils/config.go | 107 ++++++++++++++++++++++++----------------- 2 files changed, 79 insertions(+), 62 deletions(-) diff --git a/debian/aptly.conf.yaml b/debian/aptly.conf.yaml index 89c70fa6..24bbbecf 100644 --- a/debian/aptly.conf.yaml +++ b/debian/aptly.conf.yaml @@ -7,10 +7,6 @@ # - published repositories (`rootDir`/public) root_dir: ~/.aptly -# Number of attempts to open database if it's locked by other instance -# * -1 (no retry) -database_open_attempts: -1 - # Log Level # * debug # * info @@ -23,11 +19,23 @@ log_level: info # * json log_format: default +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 + # Default Architectures # empty list defaults to all available architectures architectures: # - amd64 +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true + # Dependency following ####################### @@ -58,14 +66,6 @@ ppa_distributor_id: ubuntu # Codename for short PPA url expansion ppa_codename: "" -# OBSOLETE -# in aptly up to version 1.0.0, package files were stored in internal package pool -# with MD5-dervied path, since 1.1.0 package pool layout was changed; -# if option is enabled, aptly stops checking for legacy paths; -# by default option is enabled for new aptly installations and disabled when -# upgrading from older versions -skip_legacy_pool: true - # Aptly Server ############### @@ -196,11 +196,6 @@ s3_publish_endpoints: # region: us-east-1 # # Bucket name # bucket: test-bucket - # # Endpoint (optional) - # # When using S3-compatible cloud storage, specify hostname of service endpoint here, - # # region is ignored if endpoint is set (set region to some human-readable name) - # # (should be left blank for real Amazon S3) - # endpoint: "" # # Prefix (optional) # # publishing under specified prefix in the bucket, defaults to # # no prefix (bucket root) @@ -217,6 +212,11 @@ s3_publish_endpoints: # access_key_id: "" # secret_access_key: "" # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" # # Storage Class (optional) # # Amazon S3 storage class, defaults to `STANDARD`. Other values # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) diff --git a/utils/config.go b/utils/config.go index d0e48e94..5b8aedd0 100644 --- a/utils/config.go +++ b/utils/config.go @@ -13,46 +13,63 @@ import ( // ConfigStructure is structure of main configuration type ConfigStructure struct { // nolint: maligned - RootDir string `json:"rootDir" yaml:"root_dir"` - DownloadConcurrency int `json:"downloadConcurrency" yaml:"download_concurrency"` - DownloadLimit int64 `json:"downloadSpeedLimit" yaml:"download_limit"` - DownloadRetries int `json:"downloadRetries" yaml:"download_retries"` - Downloader string `json:"downloader" yaml:"downloader"` - DatabaseOpenAttempts int `json:"databaseOpenAttempts" yaml:"database_open_attempts"` - Architectures []string `json:"architectures" yaml:"architectures"` - DepFollowSuggests bool `json:"dependencyFollowSuggests" yaml:"dep_follow_suggests"` - DepFollowRecommends bool `json:"dependencyFollowRecommends" yaml:"dep_follow_recommends"` - DepFollowAllVariants bool `json:"dependencyFollowAllVariants" yaml:"dep_follow_all_variants"` - DepFollowSource bool `json:"dependencyFollowSource" yaml:"dep_follow_source"` - DepVerboseResolve bool `json:"dependencyVerboseResolve" yaml:"dep_verboseresolve"` - GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` - GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` - GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` - DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"` - PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage" yaml:"packagepool_storage"` - SkipLegacyPool bool `json:"skipLegacyPool" yaml:"skip_legacy_pool"` - PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"` - PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"` - SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"` - SkipBz2Publishing bool `json:"skipBz2Publishing" yaml:"skip_bz2_publishing"` + // General + RootDir string `json:"rootDir" yaml:"root_dir"` + LogLevel string `json:"logLevel" yaml:"log_level"` + LogFormat string `json:"logFormat" yaml:"log_format"` + DatabaseOpenAttempts int `json:"databaseOpenAttempts" yaml:"database_open_attempts"` + Architectures []string `json:"architectures" yaml:"architectures"` + SkipLegacyPool bool `json:"skipLegacyPool" yaml:"skip_legacy_pool"` // OBSOLETE + + // Dependency following + DepFollowSuggests bool `json:"dependencyFollowSuggests" yaml:"dep_follow_suggests"` + DepFollowRecommends bool `json:"dependencyFollowRecommends" yaml:"dep_follow_recommends"` + DepFollowAllVariants bool `json:"dependencyFollowAllVariants" yaml:"dep_follow_all_variants"` + DepFollowSource bool `json:"dependencyFollowSource" yaml:"dep_follow_source"` + DepVerboseResolve bool `json:"dependencyVerboseResolve" yaml:"dep_verboseresolve"` + + // PPA + PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"` + PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"` + + // Server + ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"` + EnableMetricsEndpoint bool `json:"enableMetricsEndpoint" yaml:"enable_metrics_endpoint"` + EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint" yaml:"enable_swagger_endpoint"` + AsyncAPI bool `json:"AsyncAPI" yaml:"async_api"` // OBSOLETE + + // Database + DatabaseBackend DBConfig `json:"databaseBackend" yaml:"database_backend"` + + // Mirroring + Downloader string `json:"downloader" yaml:"downloader"` + DownloadConcurrency int `json:"downloadConcurrency" yaml:"download_concurrency"` + DownloadLimit int64 `json:"downloadSpeedLimit" yaml:"download_limit"` + DownloadRetries int `json:"downloadRetries" yaml:"download_retries"` + DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"` + + // Signing + GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` + GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` + GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` + + // Publishing + SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"` + SkipBz2Publishing bool `json:"skipBz2Publishing" yaml:"skip_bz2_publishing"` + + // Storage FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints" yaml:"filesystem_publish_endpoints"` S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints" yaml:"s3_publish_endpoints"` SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints" yaml:"swift_publish_endpoints"` AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints" yaml:"azure_publish_endpoints"` - AsyncAPI bool `json:"AsyncAPI" yaml:"async_api"` - EnableMetricsEndpoint bool `json:"enableMetricsEndpoint" yaml:"enable_metrics_endpoint"` - LogLevel string `json:"logLevel" yaml:"log_level"` - LogFormat string `json:"logFormat" yaml:"log_format"` - ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"` - DatabaseBackend DBConfig `json:"databaseBackend" yaml:"database_backend"` - EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint" yaml:"enable_swagger_endpoint"` + PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage" yaml:"packagepool_storage"` } // DBConfig type DBConfig struct { Type string `json:"type" yaml:"type"` - URL string `json:"url" yaml:"url"` DbPath string `json:"dbPath" yaml:"db_path"` + URL string `json:"url" yaml:"url"` } type LocalPoolStorage struct { @@ -125,20 +142,20 @@ func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { func (pool PackagePoolStorage) MarshalYAML() (interface{}, error) { var wrapper struct { - Type string `yaml:"type,omitempty"` + Type string `yaml:"type,omitempty"` *LocalPoolStorage `yaml:",inline"` - *AzureEndpoint `yaml:",inline"` + *AzureEndpoint `yaml:",inline"` } if pool.Azure != nil { - wrapper.Type = "azure" - wrapper.AzureEndpoint = pool.Azure + wrapper.Type = "azure" + wrapper.AzureEndpoint = pool.Azure } else if pool.Local.Path != "" { - wrapper.Type = "local" - wrapper.LocalPoolStorage = pool.Local + wrapper.Type = "local" + wrapper.LocalPoolStorage = pool.Local } - return wrapper, nil + return wrapper, nil } // FileSystemPublishRoot describes single filesystem publishing entry point @@ -152,12 +169,12 @@ type FileSystemPublishRoot struct { type S3PublishRoot struct { Region string `json:"region" yaml:"region"` Bucket string `json:"bucket" yaml:"bucket"` - Endpoint string `json:"endpoint" yaml:"endpoint"` + Prefix string `json:"prefix" yaml:"prefix"` + ACL string `json:"acl" yaml:"acl"` AccessKeyID string `json:"awsAccessKeyID" yaml:"access_key_id"` SecretAccessKey string `json:"awsSecretAccessKey" yaml:"secret_access_key"` SessionToken string `json:"awsSessionToken" yaml:"session_token"` - Prefix string `json:"prefix" yaml:"prefix"` - ACL string `json:"acl" yaml:"acl"` + Endpoint string `json:"endpoint" yaml:"endpoint"` StorageClass string `json:"storageClass" yaml:"storage_class"` EncryptionMethod string `json:"encryptionMethod" yaml:"encryption_method"` PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"` @@ -169,25 +186,25 @@ type S3PublishRoot struct { // SwiftPublishRoot describes single OpenStack Swift publishing entry point type SwiftPublishRoot struct { + Container string `json:"container" yaml:"container"` + Prefix string `json:"prefix" yaml:"prefix"` UserName string `json:"osname" yaml:"username"` Password string `json:"password" yaml:"password"` - AuthURL string `json:"authurl" yaml:"auth_url"` Tenant string `json:"tenant" yaml:"tenant"` TenantID string `json:"tenantid" yaml:"tenant_id"` Domain string `json:"domain" yaml:"domain"` DomainID string `json:"domainid" yaml:"domain_id"` TenantDomain string `json:"tenantdomain" yaml:"tenant_domain"` TenantDomainID string `json:"tenantdomainid" yaml:"tenant_domain_id"` - Prefix string `json:"prefix" yaml:"prefix"` - Container string `json:"container" yaml:"container"` + AuthURL string `json:"authurl" yaml:"auth_url"` } // AzureEndpoint describes single Azure publishing entry point type AzureEndpoint struct { - AccountName string `json:"accountName" yaml:"account_name"` - AccountKey string `json:"accountKey" yaml:"account_key"` Container string `json:"container" yaml:"container"` Prefix string `json:"prefix" yaml:"prefix"` + AccountName string `json:"accountName" yaml:"account_name"` + AccountKey string `json:"accountKey" yaml:"account_key"` Endpoint string `json:"endpoint" yaml:"endpoint"` } From 9d0c7b5ade7c93f5342ac7e00dc0a7e9b1fb4b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 22:26:43 +0100 Subject: [PATCH 54/64] make yaml config default --- debian/aptly.conf | 663 ++++++++++++++++++++--------------------- debian/aptly.conf.yaml | 339 --------------------- 2 files changed, 316 insertions(+), 686 deletions(-) delete mode 100644 debian/aptly.conf.yaml diff --git a/debian/aptly.conf b/debian/aptly.conf index 6eeed77e..24bbbecf 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,370 +1,339 @@ -// vim: : filetype=json -// json configuration file with comments -// validate with: sed '/\/\//d' aptly.conf | json_pp -{ +# Aptly Configuration File +########################### -// Aptly Configuration File -//////////////////////////// +# Aptly storage directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly - // Aptly storage directory for: - // - downloaded packages (`rootDir`/pool) - // - database (`rootDir`/db) - // - published repositories (`rootDir`/public) - "rootDir": "~/.aptly", +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info - // Number of attempts to open database if it's locked by other instance - // * -1 (no retry) - "databaseOpenAttempts": -1, +# Log Format +# * default (text) +# * json +log_format: default - // Log Level - // * debug - // * info - // * warning - // * error - "logLevel": "info", +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 - // Log Format - // * default (text) - // * json - "logFormat": "default", +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 - // Default Architectures - // empty array defaults to all available architectures - "architectures": [], +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true - // Follow contents of `Suggests:` field when processing dependencies for the package - "dependencyFollowSuggests": false, - // Follow contents of `Recommends:` field when processing dependencies for the package - "dependencyFollowRecommends": false, +# Dependency following +####################### - // When dependency looks like `package-a | package-b`, follow both variants always - "dependencyFollowAllVariants": false, +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false - // Follow dependency from binary package to source package - "dependencyFollowSource": false, +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false - // Log additional details while resolving dependencies (useful for debugging) - "dependencyVerboseResolve": false, +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false - // Specifies paramaters for short PPA url expansion - // empty defaults to output of `lsb_release` command - "ppaDistributorID": "ubuntu", +# Follow dependency from binary package to source package +dep_follow_source: false - // Codename for short PPA url expansion - "ppaCodename": "", +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false - // OBSOLETE - // in aptly up to version 1.0.0, package files were stored in internal package pool - // with MD5-dervied path, since 1.1.0 package pool layout was changed; - // if option is enabled, aptly stops checking for legacy paths; - // by default option is enabled for new aptly installations and disabled when - // upgrading from older versions - "skipLegacyPool": true, +# PPA +###### -// Aptly Server -//////////////// +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu - // Serve published repos as well as API - "serveInAPIMode": false, +# Codename for short PPA url expansion +ppa_codename: "" - // Enable metrics for Prometheus client - "enableMetricsEndpoint": false, - // Enable API documentation on /docs - "enableSwaggerEndpoint": false, +# Aptly Server +############### - // OBSOLETE: use via url param ?_async=true - "AsyncAPI": false, +# Serve published repos as well as API +serve_in_api_mode: false +# Enable metrics for Prometheus client +enable_metrics_endpoint: false -// Database -//////////// +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" - // Database backend - // Type must be one of: - // * leveldb (default) - // * etcd - "databaseBackend": { - // LevelDB - "type": "leveldb", - // Path to leveldb files - // empty dbPath defaults to `rootDir`/db - "dbPath": "" - - // // etcd - // "type": "etcd", - // // URL to db server - // "url": "127.0.0.1:2379" - }, - - -// Mirroring -///////////// - - // Downloader - // * "default" - // * "grab" (more robust) - "downloader": "default", - - // Number of parallel download threads to use when downloading packages - "downloadConcurrency": 4, - - // Limit in kbytes/sec on download speed while mirroring remote repositories - "downloadSpeedLimit": 0, - - // Number of retries for download attempts - "downloadRetries": 0, - - // Download source packages per default - "downloadSourcePackages": false, - - -// Signing -/////////// - - // GPG Provider - // * "internal" (Go internal implementation) - // * "gpg" (External `gpg` utility) - "gpgProvider": "gpg", - - // Disable signing of published repositories - "gpgDisableSign": false, - - // Disable signature verification of remote repositories - "gpgDisableVerify": false, - - -// Publishing -////////////// - - // Do not publish Contents files - "skipContentsPublishing": false, - - // Do not create bz2 files - "skipBz2Publishing": false, - - -// Storage -/////////// - - // Filesystem publishing endpoints - // - // aptly defaults to publish to a single publish directory under `rootDir`/public. For - // a more advanced publishing strategy, you can define one or more filesystem endpoints in the - // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name - // and the following associated settings. - // - // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` - // with `endpoint-name` as the name given in the aptly configuration file. For example: - // - // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - // - "FileSystemPublishEndpoints": { - // // Endpoint Name - // "test1": { - // // Directory for publishing - // "rootDir": "/opt/srv/aptly_public", - - // // File Link Method for linking files from the internal pool to the published directory - // // * hardlink - // // * symlink - // // * copy - // "linkMethod": "hardlink", - - // // File Copare Method for comparing existing links from the internal pool to the published directory - // // Only used when "linkMethod" is set to "copy" - // // * md5 (default: compare md5 sum) - // // * size (compare file size) - // "verifyMethod": "md5" - // } - }, - - // S3 Endpoint Support - // - // cloud storage). First, publishing - // endpoints should be described in aptly configuration file. Each endpoint has name - // and associated settings. - // - // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot wheezy-main s3:test:` - // - "S3PublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Amazon region for S3 bucket - // "region": "us-east-1", - - // // Bucket name - // "bucket": "test-bucket", - - // // Endpoint (optional) - // // When using S3-compatible cloud storage, specify hostname of service endpoint here, - // // region is ignored if endpoint is set (set region to some human-readable name) - // // (should be left blank for real Amazon S3) - // "endpoint": "", - - // // Prefix (optional) - // // publishing under specified prefix in the bucket, defaults to - // // no prefix (bucket root) - // "prefix": "", - - // // Default ACLs (optional) - // // assign ACL to published files (one of the canned ACLs in Amazon - // // terminology). Useful values: `private` (default), `public-read` (public - // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), - // // for private repositories special apt S3 transport is required. - // "acl": "private", - - // // Credentials (optional) - // // Amazon credentials to access S3 bucket. If not supplied, - // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - // // are used. - // "awsAccessKeyID": "", - // "awsSecretAccessKey": "", - - // // Storage Class (optional) - // // Amazon S3 storage class, defaults to `STANDARD`. Other values - // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - // "storageClass": "STANDARD", - - // // Encryption Method (optional) - // // Server-side encryption method, defaults to none. Currently - // // the only available encryption method is `AES256` - // "encryptionMethod": "none", - - // // Plus Workaround (optional) - // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - // // creating two copies of package files with `+` in filename: one original - // // and another one with spaces instead of plus signs - // // With `plusWorkaround` enabled, package files with plus sign - // // would be stored twice. aptly might not cleanup files with spaces when published - // // repository is dropped or updated (switched) to new version of repository (snapshot) - // "plusWorkaround": false, - - // // Disable MultiDel (optional) - // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, - // // enable this setting (file deletion would be slower with this setting enabled) - // "disableMultiDel": false, - - // // ForceSig2 (optional) - // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores - // // which do not support SigV4, shouldn't be enabled for AWS - // "forceSigV2": false, - - // // ForceVirtualHostedStyle (optional) - // // Disable path style visit, useful with non-AWS S3-compatible object stores - // // which only support virtual hosted style - // "forceVirtualHostedStyle": false, - - // // Debug (optional) - // // Enables detailed request/response dump for each S3 operation - // "debug": false - // } - }, - - // Swift Endpoint Support - // - // aptly could be configured to publish repository directly to OpenStack Swift. First, - // publishing endpoints should be described in aptly configuration file. Each endpoint - // has name and associated settings. - // - // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot jessie-main swift:test:` - // - "SwiftPublishEndpoints": { - // Endpoint Name - // "test": { - - // // Container Name - // "container": "taylor1", - - // // Prefix (optional) - // // Publish under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials (optional) - // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - // "osname": "", - // "password": "", - - // // Tenant (optional) - // // OpenStack tenant name and id (in order to use v2 authentication) - // "tenant": "", - // "tenantid": "", - - // // Auth URL (optional) - // // Full url of Keystone server (including port, and version). - // // Example `http://identity.example.com:5000/v2.0` - // "authurl": "" - // } - }, - - // Azure Endpoint Support - // - // aptly can be configured to publish repositories directly to Microsoft Azure Blob - // Storage. First, publishing endpoints should be described in the aptly - // configuration file. Each endpoint has its name and associated settings. - "AzurePublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Container Name - // "container": "container1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } - }, - - // Package Pool - // Location for storing downloaded packages - // Type must be one of: - // * local - // * azure - "packagePoolStorage": { - // Local Pool - "type": "local", - // Local Pool Path - // empty path defaults to `rootDir`/pool - "path": "" - - // // Azure Azure Blob Storage Pool - // "type": "azure", - // "container": "pool1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - } - -// End of config -} diff --git a/debian/aptly.conf.yaml b/debian/aptly.conf.yaml deleted file mode 100644 index 24bbbecf..00000000 --- a/debian/aptly.conf.yaml +++ /dev/null @@ -1,339 +0,0 @@ -# Aptly Configuration File -########################### - -# Aptly storage directory for: -# - downloaded packages (`rootDir`/pool) -# - database (`rootDir`/db) -# - published repositories (`rootDir`/public) -root_dir: ~/.aptly - -# Log Level -# * debug -# * info -# * warning -# * error -log_level: info - -# Log Format -# * default (text) -# * json -log_format: default - -# Number of attempts to open database if it's locked by other instance -# * -1 (no retry) -database_open_attempts: -1 - -# Default Architectures -# empty list defaults to all available architectures -architectures: -# - amd64 - -# OBSOLETE -# in aptly up to version 1.0.0, package files were stored in internal package pool -# with MD5-dervied path, since 1.1.0 package pool layout was changed; -# if option is enabled, aptly stops checking for legacy paths; -# by default option is enabled for new aptly installations and disabled when -# upgrading from older versions -skip_legacy_pool: true - - -# Dependency following -####################### - -# Follow contents of `Suggests:` field when processing dependencies for the package -dep_follow_suggests: false - -# Follow contents of `Recommends:` field when processing dependencies for the package -dep_follow_recommends: false - -# When dependency looks like `package-a | package-b`, follow both variants always -dep_follow_allvariants: false - -# Follow dependency from binary package to source package -dep_follow_source: false - -# Log additional details while resolving dependencies (useful for debugging) -dep_verbose_resolve: false - - -# PPA -###### - -# Specify paramaters for short PPA url expansion -# empty defaults to output of `lsb_release` command -ppa_distributor_id: ubuntu - -# Codename for short PPA url expansion -ppa_codename: "" - - -# Aptly Server -############### - -# Serve published repos as well as API -serve_in_api_mode: false - -# Enable metrics for Prometheus client -enable_metrics_endpoint: false - -# Enable API documentation on /docs -enable_swagger_endpoint: false - -# OBSOLETE: use via url param ?_async=true -async_api: false - - -# Database -########### - -# Database backend -# Type must be one of: -# * leveldb (default) -# * etcd -database_backend: - type: leveldb - # Path to leveldb files - # empty dbPath defaults to `rootDir`/db - db_path: "" - - # type: etcd - # # URL to db server - # url: "127.0.0.1:2379" - - -# Mirroring -############ - -# Downloader -# * "default" -# * "grab" (more robust) -downloader: default - -# Number of parallel download threads to use when downloading packages -download_concurrency: 4 - -# Limit in kbytes/sec on download speed while mirroring remote repositories -download_limit: 0 - -# Number of retries for download attempts -download_retries: 0 - -# Download source packages per default -download_sourcepackages: false - - -# Signing -########## - -# GPG Provider -# * "internal" (Go internal implementation) -# * "gpg" (External `gpg` utility) -gpg_provider: gpg - -# Disable signing of published repositories -gpg_disable_sign: false - -# Disable signature verification of remote repositories -gpg_disable_verify: false - - -# Publishing -############# - -# Do not publish Contents files -skip_contents_publishing: false - -# Do not create bz2 files -skip_bz2_publishing: false - - -# Storage -########## - -# Filesystem publishing endpoints -# -# aptly defaults to publish to a single publish directory under `rootDir`/public. For -# a more advanced publishing strategy, you can define one or more filesystem endpoints in the -# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name -# and the following associated settings. -# -# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` -# with `endpoint-name` as the name given in the aptly configuration file. For example: -# -# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` -# -filesystem_publish_endpoints: - # # Endpoint Name - # test1: - # # Directory for publishing - # root_dir: /opt/srv/aptly_public - # # File Link Method for linking files from the internal pool to the published directory - # # * hardlink - # # * symlink - # # * copy - # link_method: hardlink - # # File Copare Method for comparing existing links from the internal pool to the published directory - # # Only used when "linkMethod" is set to "copy" - # # * md5 (default: compare md5 sum) - # # * size (compare file size) - # verify_method: md5 - -# S3 Endpoint Support -# -# cloud storage). First, publishing -# endpoints should be described in aptly configuration file. Each endpoint has name -# and associated settings. -# -# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before -# publishing prefix on the command line, e.g.: -# -# `aptly publish snapshot wheezy-main s3:test:` -# -s3_publish_endpoints: - # # Endpoint Name - # test: - # # Amazon region for S3 bucket - # region: us-east-1 - # # Bucket name - # bucket: test-bucket - # # Prefix (optional) - # # publishing under specified prefix in the bucket, defaults to - # # no prefix (bucket root) - # prefix: "" - # # Default ACLs (optional) - # # assign ACL to published files: - # # * private (default, for use with apt S3 transport) - # # * public-read (public repository) - # # * none (don't set ACL) - # acl: private - # # Credentials (optional) - # # Amazon credentials to access S3 bucket. If not supplied, environment variables - # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used - # access_key_id: "" - # secret_access_key: "" - # session_token: "" - # # Endpoint (optional) - # # When using S3-compatible cloud storage, specify hostname of service endpoint here, - # # region is ignored if endpoint is set (set region to some human-readable name) - # # (should be left blank for real Amazon S3) - # endpoint: "" - # # Storage Class (optional) - # # Amazon S3 storage class, defaults to `STANDARD`. Other values - # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - # storage_class: STANDARD - # # Encryption Method (optional) - # # Server-side encryption method, defaults to none. Currently - # # the only available encryption method is `AES256` - # encryption_method: none - # # Plus Workaround (optional) - # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - # # creating two copies of package files with `+` in filename: one original - # # and another one with spaces instead of plus signs - # # With `plusWorkaround` enabled, package files with plus sign - # # would be stored twice. aptly might not cleanup files with spaces when published - # # repository is dropped or updated (switched) to new version of repository (snapshot) - # plus_workaround: false - # # Disable MultiDel (optional) - # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, - # # enable this setting (file deletion would be slower with this setting enabled) - # disable_multidel: false - # # Force Signature v2 (optional) - # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores - # # which do not support SigV4, shouldn't be enabled for AWS - # force_sigv2: false - # # Force VirtualHosted Style (optional) - # # Disable path style visit, useful with non-AWS S3-compatible object stores - # # which only support virtual hosted style - # force_virtualhosted_style: false - # # Debug (optional) - # # Enables detailed request/response dump for each S3 operation - # debug: false - -# Swift Endpoint Support -# -# aptly can publish a repository directly to OpenStack Swift. -# Each endpoint has name and associated settings. -# -# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before -# publishing prefix on the command line, e.g.: -# -# `aptly publish snapshot jessie-main swift:test:` -# -swift_publish_endpoints: - # # Endpoint Name - # test: - # # Container Name - # container: taylor1 - # # Prefix (optional) - # # Publish under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials (optional) - # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - # username: "" - # password: "" - # # Domain (optional) - # # OpenStack domain - # domain: "" - # domain_id: "" - # # Tenant (optional) - # # OpenStack tenant (in order to use v2 authentication) - # tenant: "" - # tenant_id: "" - # tenant_domain: "" - # tenant_domain_id: "" - # # Auth URL (optional) - # # Full url of Keystone server (including port, and version). - # # Example `http://identity.example.com:5000/v2.0` - # auth_url: "" - -# Azure Endpoint Support -# -# aptly can be configured to publish repositories directly to Microsoft Azure Blob -# Storage. First, publishing endpoints should be described in the aptly -# configuration file. Each endpoint has its name and associated settings. -azure_publish_endpoints: - # # Endpoint Name - # test: - # # Container Name - # container: container1 - # # Prefix (optional) - # # Publishing under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials - # # Azure storage account access key to access blob storage - # account_name: "" - # account_key: "" - # # Endpoint URL - # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - # # defaults to "https://.blob.core.windows.net" - # endpoint: "" - -# Package Pool -# -# Location for storing downloaded packages -# Type must be one of: -# * local -# * azure -packagepool_storage: - # Local Pool - type: local - # Local Pool Path - # empty path defaults to `rootDir`/pool - path: - - # # Azure Azure Blob Storage Pool - # type: azure - # # Container Name - # container: pool1 - # # Prefix (optional) - # # Publishing under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials - # # Azure storage account access key to access blob storage - # account_name: "" - # account_key: "" - # # Endpoint URL - # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - # # defaults to "https://.blob.core.windows.net" - # endpoint: "" - From 4e566b46920b99965631c09c2e77e37aa8021b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 2 Dec 2024 22:29:48 +0100 Subject: [PATCH 55/64] fix tests and lint --- Makefile | 2 +- cmd/config_show.go | 4 +- context/context_test.go | 2 +- system/t02_config/BadConfigTest_gold | 2 +- system/t02_config/ConfigShowTest_gold | 42 +- system/t02_config/CreateConfigTest_gold | 663 +++++++++++------------- utils/config.go | 26 +- utils/config_test.go | 174 +++---- 8 files changed, 442 insertions(+), 473 deletions(-) diff --git a/Makefile b/Makefile index 3661948b..58af3b69 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) -# Uncomment to update test outputs +# Uncomment to update system test gold files # CAPTURE := "--capture" help: ## Print this help diff --git a/cmd/config_show.go b/cmd/config_show.go index 56ab33e8..6568604f 100644 --- a/cmd/config_show.go +++ b/cmd/config_show.go @@ -9,11 +9,11 @@ import ( ) func aptlyConfigShow(_ *commander.Command, _ []string) error { - show_yaml := context.Flags().Lookup("yaml").Value.Get().(bool) + showYaml := context.Flags().Lookup("yaml").Value.Get().(bool) config := context.Config() - if show_yaml { + if showYaml { yamlData, err := yaml.Marshal(&config) if err != nil { return fmt.Errorf("error marshaling to YAML: %s", err) diff --git a/context/context_test.go b/context/context_test.go index f83d42f2..b2e42fe8 100644 --- a/context/context_test.go +++ b/context/context_test.go @@ -84,6 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { // storage never exists. c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, FatalErrorPanicMatches, - &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: EOF", + &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: not valid yaml or json", os.Getenv("HOME"))}) } diff --git a/system/t02_config/BadConfigTest_gold b/system/t02_config/BadConfigTest_gold index 10f5b8ce..0b0ca447 100644 --- a/system/t02_config/BadConfigTest_gold +++ b/system/t02_config/BadConfigTest_gold @@ -1 +1 @@ -ERROR: error loading config file ${HOME}/.aptly.conf: invalid character 's' looking for beginning of object key string +ERROR: error loading config file ${HOME}/.aptly.conf: not valid yaml or json diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index 0529de71..5a4a2273 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -1,39 +1,39 @@ { "rootDir": "${HOME}/.aptly", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 5, - "downloader": "default", + "logLevel": "debug", + "logFormat": "default", "databaseOpenAttempts": 10, "architectures": [], + "skipLegacyPool": false, "dependencyFollowSuggests": false, "dependencyFollowRecommends": false, "dependencyFollowAllVariants": false, "dependencyFollowSource": false, "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": {}, - "skipLegacyPool": false, "ppaDistributorID": "ubuntu", "ppaCodename": "", + "serveInAPIMode": true, + "enableMetricsEndpoint": true, + "enableSwaggerEndpoint": false, + "AsyncAPI": false, + "databaseBackend": { + "type": "", + "dbPath": "", + "url": "" + }, + "downloader": "default", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "downloadRetries": 5, + "downloadSourcePackages": false, + "gpgProvider": "gpg", + "gpgDisableSign": false, + "gpgDisableVerify": false, "skipContentsPublishing": false, "skipBz2Publishing": false, "FileSystemPublishEndpoints": {}, "S3PublishEndpoints": {}, "SwiftPublishEndpoints": {}, "AzurePublishEndpoints": {}, - "AsyncAPI": false, - "enableMetricsEndpoint": true, - "logLevel": "debug", - "logFormat": "default", - "serveInAPIMode": true, - "databaseBackend": { - "type": "", - "url": "", - "dbPath": "" - }, - "enableSwaggerEndpoint": false + "packagePoolStorage": {} } diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 6eeed77e..24bbbecf 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -1,370 +1,339 @@ -// vim: : filetype=json -// json configuration file with comments -// validate with: sed '/\/\//d' aptly.conf | json_pp -{ +# Aptly Configuration File +########################### -// Aptly Configuration File -//////////////////////////// +# Aptly storage directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly - // Aptly storage directory for: - // - downloaded packages (`rootDir`/pool) - // - database (`rootDir`/db) - // - published repositories (`rootDir`/public) - "rootDir": "~/.aptly", +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info - // Number of attempts to open database if it's locked by other instance - // * -1 (no retry) - "databaseOpenAttempts": -1, +# Log Format +# * default (text) +# * json +log_format: default - // Log Level - // * debug - // * info - // * warning - // * error - "logLevel": "info", +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 - // Log Format - // * default (text) - // * json - "logFormat": "default", +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 - // Default Architectures - // empty array defaults to all available architectures - "architectures": [], +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true - // Follow contents of `Suggests:` field when processing dependencies for the package - "dependencyFollowSuggests": false, - // Follow contents of `Recommends:` field when processing dependencies for the package - "dependencyFollowRecommends": false, +# Dependency following +####################### - // When dependency looks like `package-a | package-b`, follow both variants always - "dependencyFollowAllVariants": false, +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false - // Follow dependency from binary package to source package - "dependencyFollowSource": false, +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false - // Log additional details while resolving dependencies (useful for debugging) - "dependencyVerboseResolve": false, +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false - // Specifies paramaters for short PPA url expansion - // empty defaults to output of `lsb_release` command - "ppaDistributorID": "ubuntu", +# Follow dependency from binary package to source package +dep_follow_source: false - // Codename for short PPA url expansion - "ppaCodename": "", +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false - // OBSOLETE - // in aptly up to version 1.0.0, package files were stored in internal package pool - // with MD5-dervied path, since 1.1.0 package pool layout was changed; - // if option is enabled, aptly stops checking for legacy paths; - // by default option is enabled for new aptly installations and disabled when - // upgrading from older versions - "skipLegacyPool": true, +# PPA +###### -// Aptly Server -//////////////// +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu - // Serve published repos as well as API - "serveInAPIMode": false, +# Codename for short PPA url expansion +ppa_codename: "" - // Enable metrics for Prometheus client - "enableMetricsEndpoint": false, - // Enable API documentation on /docs - "enableSwaggerEndpoint": false, +# Aptly Server +############### - // OBSOLETE: use via url param ?_async=true - "AsyncAPI": false, +# Serve published repos as well as API +serve_in_api_mode: false +# Enable metrics for Prometheus client +enable_metrics_endpoint: false -// Database -//////////// +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" - // Database backend - // Type must be one of: - // * leveldb (default) - // * etcd - "databaseBackend": { - // LevelDB - "type": "leveldb", - // Path to leveldb files - // empty dbPath defaults to `rootDir`/db - "dbPath": "" - - // // etcd - // "type": "etcd", - // // URL to db server - // "url": "127.0.0.1:2379" - }, - - -// Mirroring -///////////// - - // Downloader - // * "default" - // * "grab" (more robust) - "downloader": "default", - - // Number of parallel download threads to use when downloading packages - "downloadConcurrency": 4, - - // Limit in kbytes/sec on download speed while mirroring remote repositories - "downloadSpeedLimit": 0, - - // Number of retries for download attempts - "downloadRetries": 0, - - // Download source packages per default - "downloadSourcePackages": false, - - -// Signing -/////////// - - // GPG Provider - // * "internal" (Go internal implementation) - // * "gpg" (External `gpg` utility) - "gpgProvider": "gpg", - - // Disable signing of published repositories - "gpgDisableSign": false, - - // Disable signature verification of remote repositories - "gpgDisableVerify": false, - - -// Publishing -////////////// - - // Do not publish Contents files - "skipContentsPublishing": false, - - // Do not create bz2 files - "skipBz2Publishing": false, - - -// Storage -/////////// - - // Filesystem publishing endpoints - // - // aptly defaults to publish to a single publish directory under `rootDir`/public. For - // a more advanced publishing strategy, you can define one or more filesystem endpoints in the - // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name - // and the following associated settings. - // - // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` - // with `endpoint-name` as the name given in the aptly configuration file. For example: - // - // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - // - "FileSystemPublishEndpoints": { - // // Endpoint Name - // "test1": { - // // Directory for publishing - // "rootDir": "/opt/srv/aptly_public", - - // // File Link Method for linking files from the internal pool to the published directory - // // * hardlink - // // * symlink - // // * copy - // "linkMethod": "hardlink", - - // // File Copare Method for comparing existing links from the internal pool to the published directory - // // Only used when "linkMethod" is set to "copy" - // // * md5 (default: compare md5 sum) - // // * size (compare file size) - // "verifyMethod": "md5" - // } - }, - - // S3 Endpoint Support - // - // cloud storage). First, publishing - // endpoints should be described in aptly configuration file. Each endpoint has name - // and associated settings. - // - // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot wheezy-main s3:test:` - // - "S3PublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Amazon region for S3 bucket - // "region": "us-east-1", - - // // Bucket name - // "bucket": "test-bucket", - - // // Endpoint (optional) - // // When using S3-compatible cloud storage, specify hostname of service endpoint here, - // // region is ignored if endpoint is set (set region to some human-readable name) - // // (should be left blank for real Amazon S3) - // "endpoint": "", - - // // Prefix (optional) - // // publishing under specified prefix in the bucket, defaults to - // // no prefix (bucket root) - // "prefix": "", - - // // Default ACLs (optional) - // // assign ACL to published files (one of the canned ACLs in Amazon - // // terminology). Useful values: `private` (default), `public-read` (public - // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), - // // for private repositories special apt S3 transport is required. - // "acl": "private", - - // // Credentials (optional) - // // Amazon credentials to access S3 bucket. If not supplied, - // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - // // are used. - // "awsAccessKeyID": "", - // "awsSecretAccessKey": "", - - // // Storage Class (optional) - // // Amazon S3 storage class, defaults to `STANDARD`. Other values - // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - // "storageClass": "STANDARD", - - // // Encryption Method (optional) - // // Server-side encryption method, defaults to none. Currently - // // the only available encryption method is `AES256` - // "encryptionMethod": "none", - - // // Plus Workaround (optional) - // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - // // creating two copies of package files with `+` in filename: one original - // // and another one with spaces instead of plus signs - // // With `plusWorkaround` enabled, package files with plus sign - // // would be stored twice. aptly might not cleanup files with spaces when published - // // repository is dropped or updated (switched) to new version of repository (snapshot) - // "plusWorkaround": false, - - // // Disable MultiDel (optional) - // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, - // // enable this setting (file deletion would be slower with this setting enabled) - // "disableMultiDel": false, - - // // ForceSig2 (optional) - // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores - // // which do not support SigV4, shouldn't be enabled for AWS - // "forceSigV2": false, - - // // ForceVirtualHostedStyle (optional) - // // Disable path style visit, useful with non-AWS S3-compatible object stores - // // which only support virtual hosted style - // "forceVirtualHostedStyle": false, - - // // Debug (optional) - // // Enables detailed request/response dump for each S3 operation - // "debug": false - // } - }, - - // Swift Endpoint Support - // - // aptly could be configured to publish repository directly to OpenStack Swift. First, - // publishing endpoints should be described in aptly configuration file. Each endpoint - // has name and associated settings. - // - // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot jessie-main swift:test:` - // - "SwiftPublishEndpoints": { - // Endpoint Name - // "test": { - - // // Container Name - // "container": "taylor1", - - // // Prefix (optional) - // // Publish under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials (optional) - // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - // "osname": "", - // "password": "", - - // // Tenant (optional) - // // OpenStack tenant name and id (in order to use v2 authentication) - // "tenant": "", - // "tenantid": "", - - // // Auth URL (optional) - // // Full url of Keystone server (including port, and version). - // // Example `http://identity.example.com:5000/v2.0` - // "authurl": "" - // } - }, - - // Azure Endpoint Support - // - // aptly can be configured to publish repositories directly to Microsoft Azure Blob - // Storage. First, publishing endpoints should be described in the aptly - // configuration file. Each endpoint has its name and associated settings. - "AzurePublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Container Name - // "container": "container1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } - }, - - // Package Pool - // Location for storing downloaded packages - // Type must be one of: - // * local - // * azure - "packagePoolStorage": { - // Local Pool - "type": "local", - // Local Pool Path - // empty path defaults to `rootDir`/pool - "path": "" - - // // Azure Azure Blob Storage Pool - // "type": "azure", - // "container": "pool1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - } - -// End of config -} diff --git a/utils/config.go b/utils/config.go index 5b8aedd0..611f549e 100644 --- a/utils/config.go +++ b/utils/config.go @@ -2,6 +2,7 @@ package utils import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -81,6 +82,9 @@ type PackagePoolStorage struct { Azure *AzureEndpoint } +var AZURE = "azure" +var LOCAL = "local" + func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { var discriminator struct { Type string `json:"type"` @@ -91,10 +95,10 @@ func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { } switch discriminator.Type { - case "azure": + case AZURE: pool.Azure = &AzureEndpoint{} return json.Unmarshal(data, &pool.Azure) - case "local", "": + case LOCAL, "": pool.Local = &LocalPoolStorage{} return json.Unmarshal(data, &pool.Local) default: @@ -111,10 +115,10 @@ func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) } switch discriminator.Type { - case "azure": + case AZURE: pool.Azure = &AzureEndpoint{} return unmarshal(&pool.Azure) - case "local", "": + case LOCAL, "": pool.Local = &LocalPoolStorage{} return unmarshal(&pool.Local) default: @@ -250,17 +254,13 @@ func LoadConfig(filename string, config *ConfigStructure) error { } defer f.Close() - dec_json := json.NewDecoder(JsonConfigReader.New(f)) - if err = dec_json.Decode(&config); err != nil { + decJSON := json.NewDecoder(JsonConfigReader.New(f)) + if err = decJSON.Decode(&config); err != nil { f.Seek(0, 0) - dec_yaml := yaml.NewDecoder(f) - if err = dec_yaml.Decode(&config); err != nil { - fmt.Errorf("config file %s is not valid yaml or json\n", filename) - } else { - fmt.Printf("config file %s format is yaml\n", filename) + decYAML := yaml.NewDecoder(f) + if err = decYAML.Decode(&config); err != nil { + err = errors.New("not valid yaml or json") } - } else { - fmt.Printf("config file %s format is json\n", filename) } return err } diff --git a/utils/config_test.go b/utils/config_test.go index ddfa4ba9..d88944e2 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -63,93 +63,93 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { f.Read(buf) c.Check(string(buf), Equals, ""+ - "{\n"+ - " \"rootDir\": \"/tmp/aptly\",\n"+ - " \"downloadConcurrency\": 5,\n"+ - " \"downloadSpeedLimit\": 0,\n"+ - " \"downloadRetries\": 0,\n"+ - " \"downloader\": \"\",\n"+ - " \"databaseOpenAttempts\": 5,\n"+ - " \"architectures\": null,\n"+ - " \"dependencyFollowSuggests\": false,\n"+ - " \"dependencyFollowRecommends\": false,\n"+ - " \"dependencyFollowAllVariants\": false,\n"+ - " \"dependencyFollowSource\": false,\n"+ - " \"dependencyVerboseResolve\": false,\n"+ - " \"gpgDisableSign\": false,\n"+ - " \"gpgDisableVerify\": false,\n"+ - " \"gpgProvider\": \"gpg\",\n"+ - " \"downloadSourcePackages\": false,\n"+ - " \"packagePoolStorage\": {\n"+ - " \"type\": \"local\",\n"+ - " \"path\": \"/tmp/aptly-pool\"\n"+ - " },\n"+ - " \"skipLegacyPool\": false,\n"+ - " \"ppaDistributorID\": \"\",\n"+ - " \"ppaCodename\": \"\",\n"+ - " \"skipContentsPublishing\": false,\n"+ - " \"skipBz2Publishing\": false,\n"+ - " \"FileSystemPublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"rootDir\": \"/opt/aptly-publish\",\n"+ - " \"linkMethod\": \"\",\n"+ - " \"verifyMethod\": \"\"\n"+ - " }\n"+ - " },\n"+ - " \"S3PublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"region\": \"us-east-1\",\n"+ - " \"bucket\": \"repo\",\n"+ - " \"endpoint\": \"\",\n"+ - " \"awsAccessKeyID\": \"\",\n"+ - " \"awsSecretAccessKey\": \"\",\n"+ - " \"awsSessionToken\": \"\",\n"+ - " \"prefix\": \"\",\n"+ - " \"acl\": \"\",\n"+ - " \"storageClass\": \"\",\n"+ - " \"encryptionMethod\": \"\",\n"+ - " \"plusWorkaround\": false,\n"+ - " \"disableMultiDel\": false,\n"+ - " \"forceSigV2\": false,\n"+ - " \"forceVirtualHostedStyle\": false,\n"+ - " \"debug\": false\n"+ - " }\n"+ - " },\n"+ - " \"SwiftPublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"osname\": \"\",\n"+ - " \"password\": \"\",\n"+ - " \"authurl\": \"\",\n"+ - " \"tenant\": \"\",\n"+ - " \"tenantid\": \"\",\n"+ - " \"domain\": \"\",\n"+ - " \"domainid\": \"\",\n"+ - " \"tenantdomain\": \"\",\n"+ - " \"tenantdomainid\": \"\",\n"+ - " \"prefix\": \"\",\n"+ - " \"container\": \"repo\"\n"+ - " }\n"+ - " },\n"+ - " \"AzurePublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"accountName\": \"\",\n"+ - " \"accountKey\": \"\",\n"+ - " \"container\": \"repo\",\n"+ - " \"prefix\": \"\",\n"+ - " \"endpoint\": \"\"\n"+ - " }\n"+ - " },\n"+ - " \"AsyncAPI\": false,\n"+ - " \"enableMetricsEndpoint\": false,\n"+ - " \"logLevel\": \"info\",\n"+ - " \"logFormat\": \"json\",\n"+ - " \"serveInAPIMode\": false,\n"+ - " \"databaseBackend\": {\n"+ - " \"type\": \"\",\n"+ - " \"url\": \"\",\n"+ - " \"dbPath\": \"\"\n" + - " },\n"+ - " \"enableSwaggerEndpoint\": false\n" + + "{\n" + + " \"rootDir\": \"/tmp/aptly\",\n" + + " \"logLevel\": \"info\",\n" + + " \"logFormat\": \"json\",\n" + + " \"databaseOpenAttempts\": 5,\n" + + " \"architectures\": null,\n" + + " \"skipLegacyPool\": false,\n" + + " \"dependencyFollowSuggests\": false,\n" + + " \"dependencyFollowRecommends\": false,\n" + + " \"dependencyFollowAllVariants\": false,\n" + + " \"dependencyFollowSource\": false,\n" + + " \"dependencyVerboseResolve\": false,\n" + + " \"ppaDistributorID\": \"\",\n" + + " \"ppaCodename\": \"\",\n" + + " \"serveInAPIMode\": false,\n" + + " \"enableMetricsEndpoint\": false,\n" + + " \"enableSwaggerEndpoint\": false,\n" + + " \"AsyncAPI\": false,\n" + + " \"databaseBackend\": {\n" + + " \"type\": \"\",\n" + + " \"dbPath\": \"\",\n" + + " \"url\": \"\"\n" + + " },\n" + + " \"downloader\": \"\",\n" + + " \"downloadConcurrency\": 5,\n" + + " \"downloadSpeedLimit\": 0,\n" + + " \"downloadRetries\": 0,\n" + + " \"downloadSourcePackages\": false,\n" + + " \"gpgProvider\": \"gpg\",\n" + + " \"gpgDisableSign\": false,\n" + + " \"gpgDisableVerify\": false,\n" + + " \"skipContentsPublishing\": false,\n" + + " \"skipBz2Publishing\": false,\n" + + " \"FileSystemPublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"rootDir\": \"/opt/aptly-publish\",\n" + + " \"linkMethod\": \"\",\n" + + " \"verifyMethod\": \"\"\n" + + " }\n" + + " },\n" + + " \"S3PublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"region\": \"us-east-1\",\n" + + " \"bucket\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"acl\": \"\",\n" + + " \"awsAccessKeyID\": \"\",\n" + + " \"awsSecretAccessKey\": \"\",\n" + + " \"awsSessionToken\": \"\",\n" + + " \"endpoint\": \"\",\n" + + " \"storageClass\": \"\",\n" + + " \"encryptionMethod\": \"\",\n" + + " \"plusWorkaround\": false,\n" + + " \"disableMultiDel\": false,\n" + + " \"forceSigV2\": false,\n" + + " \"forceVirtualHostedStyle\": false,\n" + + " \"debug\": false\n" + + " }\n" + + " },\n" + + " \"SwiftPublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"container\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"osname\": \"\",\n" + + " \"password\": \"\",\n" + + " \"tenant\": \"\",\n" + + " \"tenantid\": \"\",\n" + + " \"domain\": \"\",\n" + + " \"domainid\": \"\",\n" + + " \"tenantdomain\": \"\",\n" + + " \"tenantdomainid\": \"\",\n" + + " \"authurl\": \"\"\n" + + " }\n" + + " },\n" + + " \"AzurePublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"container\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"accountName\": \"\",\n" + + " \"accountKey\": \"\",\n" + + " \"endpoint\": \"\"\n" + + " }\n" + + " },\n" + + " \"packagePoolStorage\": {\n" + + " \"type\": \"local\",\n" + + " \"path\": \"/tmp/aptly-pool\"\n" + + " }\n" + "}") } From 22cfa4c8c718c7e60d7444abe5245f5d517f3f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 00:26:19 +0100 Subject: [PATCH 56/64] add yaml example --- debian/aptly.conf.yaml | 339 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 debian/aptly.conf.yaml diff --git a/debian/aptly.conf.yaml b/debian/aptly.conf.yaml new file mode 100644 index 00000000..24bbbecf --- /dev/null +++ b/debian/aptly.conf.yaml @@ -0,0 +1,339 @@ +# Aptly Configuration File +########################### + +# Aptly storage directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly + +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info + +# Log Format +# * default (text) +# * json +log_format: default + +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 + +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 + +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true + + +# Dependency following +####################### + +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false + +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false + +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false + +# Follow dependency from binary package to source package +dep_follow_source: false + +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false + + +# PPA +###### + +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu + +# Codename for short PPA url expansion +ppa_codename: "" + + +# Aptly Server +############### + +# Serve published repos as well as API +serve_in_api_mode: false + +# Enable metrics for Prometheus client +enable_metrics_endpoint: false + +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + From a15d8a3f7bd9de4222a6bdb400f70eb5ffbcdff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 3 Dec 2024 16:04:31 +0100 Subject: [PATCH 57/64] add test --- system/t02_config/ConfigShowYAMLTest_gold | 37 ++++ system/t02_config/config.py | 8 + utils/config.go | 17 ++ utils/config_test.go | 205 +++++++++++++++++++++- 4 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 system/t02_config/ConfigShowYAMLTest_gold diff --git a/system/t02_config/ConfigShowYAMLTest_gold b/system/t02_config/ConfigShowYAMLTest_gold new file mode 100644 index 00000000..615982b0 --- /dev/null +++ b/system/t02_config/ConfigShowYAMLTest_gold @@ -0,0 +1,37 @@ +root_dir: ${HOME}/.aptly +log_level: debug +log_format: default +database_open_attempts: 10 +architectures: [] +skip_legacy_pool: false +dep_follow_suggests: false +dep_follow_recommends: false +dep_follow_all_variants: false +dep_follow_source: false +dep_verboseresolve: false +ppa_distributor_id: ubuntu +ppa_codename: "" +serve_in_api_mode: true +enable_metrics_endpoint: true +enable_swagger_endpoint: false +async_api: false +database_backend: + type: "" + db_path: "" + url: "" +downloader: default +download_concurrency: 4 +download_limit: 0 +download_retries: 5 +download_sourcepackages: false +gpg_provider: gpg +gpg_disable_sign: false +gpg_disable_verify: false +skip_contents_publishing: false +skip_bz2_publishing: false +filesystem_publish_endpoints: {} +s3_publish_endpoints: {} +swift_publish_endpoints: {} +azure_publish_endpoints: {} +packagepool_storage: {} + diff --git a/system/t02_config/config.py b/system/t02_config/config.py index 240e5944..4bee72d8 100644 --- a/system/t02_config/config.py +++ b/system/t02_config/config.py @@ -64,3 +64,11 @@ class ConfigShowTest(BaseTest): """ runCmd = ["aptly", "config", "show"] gold_processor = BaseTest.expand_environ + + +class ConfigShowYAMLTest(BaseTest): + """ + config showing + """ + runCmd = ["aptly", "config", "show", "-yaml"] + gold_processor = BaseTest.expand_environ diff --git a/utils/config.go b/utils/config.go index 611f549e..fa542ac4 100644 --- a/utils/config.go +++ b/utils/config.go @@ -294,6 +294,23 @@ func SaveConfigRaw(filename string, conf []byte) error { return err } +// SaveConfigYAML write configuration to yaml file +func SaveConfigYAML(filename string, config *ConfigStructure) error { + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + yamlData, err := yaml.Marshal(&config) + if err != nil { + return fmt.Errorf("error marshaling to YAML: %s", err) + } + + _, err = f.Write(yamlData) + return err +} + // GetRootDir returns the RootDir with expanded ~ as home directory func (conf *ConfigStructure) GetRootDir() string { return strings.Replace(conf.RootDir, "~", os.Getenv("HOME"), 1) diff --git a/utils/config_test.go b/utils/config_test.go index d88944e2..087ad258 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -14,11 +14,14 @@ type ConfigSuite struct { var _ = Suite(&ConfigSuite{}) func (s *ConfigSuite) TestLoadConfig(c *C) { - configname := filepath.Join(c.MkDir(), "aptly.json") + configname := filepath.Join(c.MkDir(), "aptly.json1") f, _ := os.Create(configname) f.WriteString(configFile) f.Close() + // start with empty config + s.config = ConfigStructure{} + err := LoadConfig(configname, &s.config) c.Assert(err, IsNil) c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/") @@ -27,7 +30,10 @@ func (s *ConfigSuite) TestLoadConfig(c *C) { } func (s *ConfigSuite) TestSaveConfig(c *C) { - configname := filepath.Join(c.MkDir(), "aptly.json") + configname := filepath.Join(c.MkDir(), "aptly.json2") + + // start with empty config + s.config = ConfigStructure{} s.config.RootDir = "/tmp/aptly" s.config.DownloadConcurrency = 5 @@ -153,4 +159,199 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { "}") } +func (s *ConfigSuite) TestLoadYAMLConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml1") + f, _ := os.Create(configname) + f.WriteString(configFileYAML) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err, IsNil) + c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/") + c.Check(s.config.DownloadConcurrency, Equals, 40) + c.Check(s.config.DatabaseOpenAttempts, Equals, 10) +} + +func (s *ConfigSuite) TestSaveYAMLConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml2") + f, _ := os.Create(configname) + f.WriteString(configFileYAML) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err, IsNil) + + err = SaveConfigYAML(configname, &s.config) + c.Assert(err, IsNil) + + f, _ = os.Open(configname) + defer f.Close() + + st, _ := f.Stat() + buf := make([]byte, st.Size()) + f.Read(buf) + + c.Check(string(buf), Equals, configFileYAML) +} + +func (s *ConfigSuite) TestSaveYAML2Config(c *C) { + // start with empty config + s.config = ConfigStructure{} + + s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"} + s.config.PackagePoolStorage.Azure = nil + + configname := filepath.Join(c.MkDir(), "aptly.yaml3") + err := SaveConfigYAML(configname, &s.config) + c.Assert(err, IsNil) + + f, _ := os.Open(configname) + defer f.Close() + + st, _ := f.Stat() + buf := make([]byte, st.Size()) + f.Read(buf) + + c.Check(string(buf), Equals, "" + + "root_dir: \"\"\n" + + "log_level: \"\"\n" + + "log_format: \"\"\n" + + "database_open_attempts: 0\n" + + "architectures: []\n" + + "skip_legacy_pool: false\n" + + "dep_follow_suggests: false\n" + + "dep_follow_recommends: false\n" + + "dep_follow_all_variants: false\n" + + "dep_follow_source: false\n" + + "dep_verboseresolve: false\n" + + "ppa_distributor_id: \"\"\n" + + "ppa_codename: \"\"\n" + + "serve_in_api_mode: false\n" + + "enable_metrics_endpoint: false\n" + + "enable_swagger_endpoint: false\n" + + "async_api: false\n" + + "database_backend:\n" + + " type: \"\"\n" + + " db_path: \"\"\n" + + " url: \"\"\n" + + "downloader: \"\"\n" + + "download_concurrency: 0\n" + + "download_limit: 0\n" + + "download_retries: 0\n" + + "download_sourcepackages: false\n" + + "gpg_provider: \"\"\n" + + "gpg_disable_sign: false\n" + + "gpg_disable_verify: false\n" + + "skip_contents_publishing: false\n" + + "skip_bz2_publishing: false\n" + + "filesystem_publish_endpoints: {}\n" + + "s3_publish_endpoints: {}\n" + + "swift_publish_endpoints: {}\n" + + "azure_publish_endpoints: {}\n" + + "packagepool_storage:\n" + + " type: local\n" + + " path: /tmp/aptly-pool\n") +} + +func (s *ConfigSuite) TestLoadEmptyConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml4") + f, _ := os.Create(configname) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err.Error(), Equals, "not valid yaml or json") +} + const configFile = `{"rootDir": "/opt/aptly/", "downloadConcurrency": 33, "databaseOpenAttempts": 33}` +const configFileYAML = `root_dir: /opt/aptly/ +log_level: error +log_format: json +database_open_attempts: 10 +architectures: + - amd64 + - arm64 +skip_legacy_pool: true +dep_follow_suggests: true +dep_follow_recommends: true +dep_follow_all_variants: true +dep_follow_source: true +dep_verboseresolve: true +ppa_distributor_id: Ubuntu +ppa_codename: short +serve_in_api_mode: true +enable_metrics_endpoint: true +enable_swagger_endpoint: true +async_api: true +database_backend: + type: etcd + db_path: "" + url: 127.0.0.1:2379 +downloader: grab +download_concurrency: 40 +download_limit: 100 +download_retries: 10 +download_sourcepackages: true +gpg_provider: gpg +gpg_disable_sign: true +gpg_disable_verify: true +skip_contents_publishing: true +skip_bz2_publishing: true +filesystem_publish_endpoints: + test1: + root_dir: /opt/srv/aptly_public + link_method: hardlink + verify_method: md5 +s3_publish_endpoints: + test: + region: us-east-1 + bucket: test-bucket + prefix: "" + acl: public-read + access_key_id: "2" + secret_access_key: secret + session_token: none + endpoint: endpoint + storage_class: STANDARD + encryption_method: AES256 + plus_workaround: true + disable_multidel: true + force_sigv2: true + force_virtualhosted_style: true + debug: true +swift_publish_endpoints: + test: + container: c1 + prefix: pre + username: user + password: pass + tenant: t + tenant_id: "2" + domain: pop + domain_id: "1" + tenant_domain: td + tenant_domain_id: "3" + auth_url: http://auth.url +azure_publish_endpoints: + test: + container: container1 + prefix: pre2 + account_name: aname + account_key: akey + endpoint: https://end.point +packagepool_storage: + type: azure + container: test-pool1 + prefix: pre3 + account_name: a name + account_key: a key + endpoint: "" +` From eb6dd4d69e3ad3e5d5cb40c1374fcb7d2827ceb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 11:40:18 +0100 Subject: [PATCH 58/64] swagger: fix docker serve --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 58af3b69..ba282293 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ bench: serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf - sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf + sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages From a7d6782176bf51df0f3f431f58ca53c28ee91693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 12:14:00 +0100 Subject: [PATCH 59/64] add test and improve config error messages --- context/context_test.go | 2 +- system/t02_config/BadConfigTest_gold | 2 +- utils/config.go | 9 +++++---- utils/config_test.go | 24 ++++++++++++++++++++---- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/context/context_test.go b/context/context_test.go index b2e42fe8..16ecbb2b 100644 --- a/context/context_test.go +++ b/context/context_test.go @@ -84,6 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { // storage never exists. c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, FatalErrorPanicMatches, - &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: not valid yaml or json", + &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)", os.Getenv("HOME"))}) } diff --git a/system/t02_config/BadConfigTest_gold b/system/t02_config/BadConfigTest_gold index 0b0ca447..f6864d06 100644 --- a/system/t02_config/BadConfigTest_gold +++ b/system/t02_config/BadConfigTest_gold @@ -1 +1 @@ -ERROR: error loading config file ${HOME}/.aptly.conf: not valid yaml or json +ERROR: error loading config file ${HOME}/.aptly.conf: invalid yaml (yaml: line 1: did not find expected ',' or '}') or json (invalid character 's' looking for beginning of object key string) diff --git a/utils/config.go b/utils/config.go index fa542ac4..c0cc12d6 100644 --- a/utils/config.go +++ b/utils/config.go @@ -2,7 +2,6 @@ package utils import ( "encoding/json" - "errors" "fmt" "os" "path/filepath" @@ -258,9 +257,11 @@ func LoadConfig(filename string, config *ConfigStructure) error { if err = decJSON.Decode(&config); err != nil { f.Seek(0, 0) decYAML := yaml.NewDecoder(f) - if err = decYAML.Decode(&config); err != nil { - err = errors.New("not valid yaml or json") - } + if err2 := decYAML.Decode(&config); err2 != nil { + err = fmt.Errorf("invalid yaml (%s) or json (%s)", err2, err) + } else { + err = nil + } } return err } diff --git a/utils/config_test.go b/utils/config_test.go index 087ad258..3317f05d 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -175,9 +175,22 @@ func (s *ConfigSuite) TestLoadYAMLConfig(c *C) { c.Check(s.config.DatabaseOpenAttempts, Equals, 10) } -func (s *ConfigSuite) TestSaveYAMLConfig(c *C) { +func (s *ConfigSuite) TestLoadYAMLErrorConfig(c *C) { configname := filepath.Join(c.MkDir(), "aptly.yaml2") f, _ := os.Create(configname) + f.WriteString(configFileYAMLError) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err.Error(), Equals, "invalid yaml (unknown pool storage type: invalid) or json (invalid character 'p' looking for beginning of value)") +} + +func (s *ConfigSuite) TestSaveYAMLConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml3") + f, _ := os.Create(configname) f.WriteString(configFileYAML) f.Close() @@ -207,7 +220,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) { s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"} s.config.PackagePoolStorage.Azure = nil - configname := filepath.Join(c.MkDir(), "aptly.yaml3") + configname := filepath.Join(c.MkDir(), "aptly.yaml4") err := SaveConfigYAML(configname, &s.config) c.Assert(err, IsNil) @@ -260,7 +273,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) { } func (s *ConfigSuite) TestLoadEmptyConfig(c *C) { - configname := filepath.Join(c.MkDir(), "aptly.yaml4") + configname := filepath.Join(c.MkDir(), "aptly.yaml5") f, _ := os.Create(configname) f.Close() @@ -268,7 +281,7 @@ func (s *ConfigSuite) TestLoadEmptyConfig(c *C) { s.config = ConfigStructure{} err := LoadConfig(configname, &s.config) - c.Assert(err.Error(), Equals, "not valid yaml or json") + c.Assert(err.Error(), Equals, "invalid yaml (EOF) or json (EOF)") } const configFile = `{"rootDir": "/opt/aptly/", "downloadConcurrency": 33, "databaseOpenAttempts": 33}` @@ -355,3 +368,6 @@ packagepool_storage: account_key: a key endpoint: "" ` +const configFileYAMLError = `packagepool_storage: + type: invalid +` From e50a5e175fc49140892a7cadbf1308919d01ce8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 12:41:16 +0100 Subject: [PATCH 60/64] update documentation and man page --- debian/aptly.conf | 6 +- man/aptly.1 | 799 ++++++++++++++---------- man/aptly.1.ronn.tmpl | 665 +++++++++++--------- system/t02_config/CreateConfigTest_gold | 6 +- utils/config.go | 2 +- utils/config_test.go | 6 +- 6 files changed, 853 insertions(+), 631 deletions(-) diff --git a/debian/aptly.conf b/debian/aptly.conf index 24bbbecf..38a8a56a 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,7 +1,11 @@ # Aptly Configuration File ########################### +# vim: : filetype=yaml -# Aptly storage directory for: +# aptly 1.6.0 supports yaml configuraiton files with inline documentation and examples. +# Legacy json config files are still supported, and may be converted to yaml with `aptly config show -yaml` + +# Root directory for: # - downloaded packages (`rootDir`/pool) # - database (`rootDir`/db) # - published repositories (`rootDir`/public) diff --git a/man/aptly.1 b/man/aptly.1 index d49d2f6f..523440f8 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "APTLY" "1" "October 2024" "" "" +.TH "APTLY" "1" "December 2024" "" "" . .SH "NAME" \fBaptly\fR \- Debian repository management tool @@ -28,330 +28,390 @@ aptly\(cqs goal is to establish repeatability and controlled changes in a packag aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/usr/local/etc/aptly\.conf\fR and \fB/etc/aptly\.conf\fR\. If no config file found (or they are not readable), a new one is created in the home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly/\fR, it will be created if missing\. . .P -Configuration file is stored in JSON format (default values shown below): +With aptly version 1\.6\.0, yaml configuration with inline documentation is supported and recommended (see \fBdebian/aptly\.conf\fR)\. +. +.P +The legacy json configuration is still supported: . .IP "" 4 . .nf +// vim: : filetype=json +// json configuration file with comments +// validate with: sed \(cq/\e/\e//d\(cq aptly\.conf | json_pp { - "rootDir": "$HOME/\.aptly", - "databaseBackend": { - "type": "", - "url": "" - }, - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", - "databaseOpenAttempts": 10, + +// Aptly Configuration File +//////////////////////////// + + // Root directory for: + // \- downloaded packages (`rootDir`/pool) + // \- database (`rootDir`/db) + // \- published repositories (`rootDir`/public) + "rootDir": "~/\.aptly", + + // Number of attempts to open database if it\(cqs locked by other instance + // * \-1 (no retry) + "databaseOpenAttempts": \-1, + + // Log Level + // * debug + // * info + // * warning + // * error + "logLevel": "info", + + // Log Format + // * default (text) + // * json + "logFormat": "default", + + // Default Architectures + // empty array defaults to all available architectures "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package "dependencyFollowRecommends": false, + + // When dependency looks like `package\-a | package\-b`, follow both variants always "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": { - "path": "$ROOTDIR/pool", - "azure": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "" - } - }, - "skipLegacyPool": true, + + // Specifies paramaters for short PPA url expansion + // empty defaults to output of `lsb_release` command "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion "ppaCodename": "", + + // OBSOLETE + // in aptly up to version 1\.0\.0, package files were stored in internal package pool + // with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + + +// Aptly Server +//////////////// + + // Serve published repos as well as API + "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + +// Database +//////////// + + // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd + "databaseBackend": { + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db + "dbPath": "" + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127\.0\.0\.1:2379" + }, + + +// Mirroring +///////////// + + // Downloader + // * "default" + // * "grab" (more robust) + "downloader": "default", + + // Number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // Limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // Number of retries for download attempts + "downloadRetries": 0, + + // Download source packages per default + "downloadSourcePackages": false, + + +// Signing +/////////// + + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) + "gpgProvider": "gpg", + + // Disable signing of published repositories + "gpgDisableSign": false, + + // Disable signature verification of remote repositories + "gpgDisableVerify": false, + + +// Publishing +////////////// + + // Do not publish Contents files "skipContentsPublishing": false, + + // Do not create bz2 files + "skipBz2Publishing": false, + + +// Storage +/////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public\. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file\. Each endpoint has a name + // and the following associated settings\. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint\-name` + // with `endpoint\-name` as the name given in the aptly configuration file\. For example: + // + // `aptly publish snapshot wheezy\-main filesystem:test1:wheezy/daily` + // "FileSystemPublishEndpoints": { - "test1": { - "rootDir": "/opt/srv1/aptly_public", - "linkMethod": "symlink" - }, - "test2": { - "rootDir": "/opt/srv2/aptly_public", - "linkMethod": "copy", - "verifyMethod": "md5" - }, - "test3": { - "rootDir": "/opt/srv3/aptly_public", - "linkMethod": "hardlink" - } + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (compare file size) + // "verifyMethod": "md5" + // } }, + + // S3 Endpoint Support + // + // cloud storage)\. First, publishing + // endpoints should be described in aptly configuration file\. Each endpoint has name + // and associated settings\. + // + // In order to publish to S3, specify endpoint as `s3:endpoint\-name:` before + // publishing prefix on the command line, e\.g\.: + // + // `aptly publish snapshot wheezy\-main s3:test:` + // "S3PublishEndpoints": { - "test": { - "region": "us\-east\-1", - "bucket": "repo", - "endpoint": "", - "awsAccessKeyID": "", - "awsSecretAccessKey": "", - "prefix": "", - "acl": "public\-read", - "storageClass": "", - "encryptionMethod": "", - "plusWorkaround": false, - "disableMultiDel": false, - "forceSigV2": false, - "forceVirtualHostedStyle": true, - "debug": false - } + // // Endpoint Name + // "test": { + + // // Amazon region for S3 bucket + // "region": "us\-east\-1", + + // // Bucket name + // "bucket": "test\-bucket", + + // // Endpoint (optional) + // // When using S3\-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human\-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology)\. Useful values: `private` (default), `public\-read` (public + // // repository) or `none` (don\(cqt set ACL)\. Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required\. + // "acl": "private", + + // // Credentials (optional) + // // Amazon credentials to access S3 bucket\. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used\. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`\. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + + // // Encryption Method (optional) + // // Server\-side encryption method, defaults to none\. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice\. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + + // // Disable MultiDel (optional) + // // For S3\-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non\-AWS S3\-compatible object stores + // // which do not support SigV4, shouldn\(cqt be enabled for AWS + // "forceSigV2": false, + + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non\-AWS S3\-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } }, + + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift\. First, + // publishing endpoints should be described in aptly configuration file\. Each endpoint + // has name and associated settings\. + // + // In order to publish to Swift, specify endpoint as `swift:endpoint\-name:` before + // publishing prefix on the command line, e\.g\.: + // + // `aptly publish snapshot jessie\-main swift:test:` + // "SwiftPublishEndpoints": { - "test": { - "container": "repo", - "osname": "", - "password": "", - "prefix": "", - "authurl": "", - "tenant": "", - "tenantid": "" - } + // Endpoint Name + // "test": { + + // // Container Name + // "container": "taylor1", + + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone\. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version)\. + // // Example `http://identity\.example\.com:5000/v2\.0` + // "authurl": "" + // } }, + + // Azure Endpoint Support + // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage\. First, publishing endpoints should be described in the aptly + // configuration file\. Each endpoint has its name and associated settings\. "AzurePublishEndpoints": { - "test": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "blob\.core\.windows\.net" - } + // // Endpoint Name + // "test": { + + // // Container Name + // "container": "container1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string + // // defaults to "https://\.blob\.core\.windows\.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool + "type": "local", + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string + // // defaults to "https://\.blob\.core\.windows\.net" + // "endpoint": "" } + +// End of config } . .fi . .IP "" 0 . -.P -Options: -. -.IP "\[ci]" 4 -\fBrootDir\fR: is root of directory storage to store database (\fBrootDir\fR/db), the default for downloaded packages (\fBrootDir\fR/pool) and the default for published repositories (\fBrootDir\fR/public) -. -.IP "\[ci]" 4 -\fBdatabaseBackend\fR: the database config; if this config is empty, use levledb backend by default -. -.IP "\[ci]" 4 -\fBdownloadConcurrency\fR: is a number of parallel download threads to use when downloading packages -. -.IP "\[ci]" 4 -\fBdownloadSpeedLimit\fR: limit in kbytes/sec on download speed while mirroring remote repositories -. -.IP "\[ci]" 4 -\fBdownloadRetries\fR: number of retries for download attempts -. -.IP "\[ci]" 4 -\fBdatabaseOpenAttempts\fR: number of attempts to open DB if it\(cqs locked by other instance; could be overridden with option \fB\-db\-open\-attempts\fR -. -.IP "\[ci]" 4 -\fBarchitectures\fR: is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR -. -.IP "\[ci]" 4 -\fBdependencyFollowSuggests\fR: follow contents of \fBSuggests:\fR field when processing dependencies for the package -. -.IP "\[ci]" 4 -\fBdependencyFollowRecommends\fR: follow contents of \fBRecommends:\fR field when processing dependencies for the package -. -.IP "\[ci]" 4 -\fBdependencyFollowAllVariants\fR: when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants always -. -.IP "\[ci]" 4 -\fBdependencyFollowSource\fR: follow dependency from binary package to source package -. -.IP "\[ci]" 4 -\fBdependencyVerboseResolve\fR: print additional details while resolving dependencies (useful for debugging) -. -.IP "\[ci]" 4 -\fBgpgDisableSign\fR: don\(cqt sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing -. -.IP "\[ci]" 4 -\fBgpgDisableVerify\fR: don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors -. -.IP "\[ci]" 4 -\fBgpgProvider\fR: implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation; \fBgpg1\fR might be used to force use of GnuPG 1\.x, \fBgpg2\fR enables GnuPG 2\.x only; default is to use GnuPG 1\.x if available and GnuPG 2\.x otherwise -. -.IP "\[ci]" 4 -\fBdownloadSourcePackages\fR: if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag -. -.IP "\[ci]" 4 -\fBpackagePoolStorage\fR: configures the location to store downloaded packages (defaults to the path \fB$ROOTDIR/pool\fR), by setting the value of the \fBtype\fR: -. -.IP "\[ci]" 4 -\fBpath\fR: store the packages in the given path -. -.IP "\[ci]" 4 -\fBazure\fR: store the packages in the given Azure Blob Storage container (see the section on Azure publishing below for information on the configuration) -. -.IP "" 0 - -. -.IP "\[ci]" 4 -\fBskipLegacyPool\fR: in aptly up to version 1\.0\.0, package files were stored in internal package pool with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; if option is enabled, aptly stops checking for legacy paths; by default option is enabled for new aptly installations and disabled when upgrading from older versions -. -.IP "\[ci]" 4 -\fBppaDistributorID\fR, \fBppaCodename\fR: specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command -. -.IP "\[ci]" 4 -\fBFileSystemPublishEndpoints\fR: configuration of local filesystem publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBS3PublishEndpoints\fR: configuration of Amazon S3 publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBSwiftPublishEndpoints\fR: configuration of OpenStack Swift publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBAzurePublishEndpoints\fR: configuration of Azure publishing endpoints (see below) -. -.IP "" 0 -. -.SH "CUSTOM PACKAGE POOLS" -aptly defaults to storing downloaded packages at \fBrootDir/\fRpool\. In order to change this, you can set the \fBtype\fR key within \fBpackagePoolStorage\fR to one of two values: -. -.IP "\[ci]" 4 -\fBlocal\fR: Store the package pool locally (the default)\. In order to change the path, additionally set the \fBpath\fR key within \fBpackagePoolStorage\fR to the desired location\. -. -.IP "\[ci]" 4 -\fBazure\fR: Store the package pool in an Azure Blob Storage container\. Any keys in the below section on Azure publishing may be set on the \fBpackagePoolStorage\fR object in order to configure the Azure connection\. -. -.IP "" 0 -. -.SH "FILESYSTEM PUBLISHING ENDPOINTS" -aptly defaults to publish to a single publish directory under \fBrootDir\fR/public\. For a more advanced publishing strategy, you can define one or more filesystem endpoints in the \fBFileSystemPublishEndpoints\fR list of the aptly configuration file\. Each endpoint has a name and the following associated settings: -. -.TP -\fBrootDir\fR -The publish directory, e\.g\., \fB/opt/srv/aptly_public\fR\. -. -.TP -\fBlinkMethod\fR -This is one of \fBhardlink\fR, \fBsymlink\fR or \fBcopy\fR\. It specifies how aptly links the files from the internal pool to the published directory\. If not specified, empty or wrong, this defaults to \fBhardlink\fR\. -. -.TP -\fBverifyMethod\fR -This is used only when setting the \fBlinkMethod\fR to \fBcopy\fR\. Possible values are \fBmd5\fR and \fBsize\fR\. It specifies how aptly compares existing links from the internal pool to the published directory\. The \fBsize\fR method compares only the file sizes, whereas the \fBmd5\fR method calculates the md5 checksum of the found file and compares it to the desired one\. If not specified, empty or wrong, this defaults to \fBmd5\fR\. -. -.P -In order to publish to such an endpoint, specify the endpoint as \fBfilesystem:endpoint\-name\fR with \fBendpoint\-name\fR as the name given in the aptly configuration file\. For example: -. -.P -\fBaptly publish snapshot wheezy\-main filesystem:test1:wheezy/daily\fR -. -.SH "S3 PUBLISHING ENDPOINTS" -aptly could be configured to publish repository directly to Amazon S3 (or S3\-compatible cloud storage)\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings: -. -.TP -\fBregion\fR -Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR) -. -.TP -\fBbucket\fR -bucket name -. -.TP -\fBendpoint\fR -(optional) when using S3\-compatible cloud storage, specify hostname of service endpoint here, region is ignored if endpoint is set (set region to some human\-readable name) (should be left blank for real Amazon S3) -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root) -. -.TP -\fBacl\fR -(optional) assign ACL to published files (one of the canned ACLs in Amazon terminology)\. Useful values: \fBprivate\fR (default), \fBpublic\-read\fR (public repository) or \fBnone\fR (don\(cqt set ACL)\. Public repositories could be consumed by \fBapt\fR using HTTP endpoint (Amazon bucket should be configured for "website hosting"), for private repositories special apt S3 transport is required\. -. -.TP -\fBawsAccessKeyID\fR, \fBawsSecretAccessKey\fR -(optional) Amazon credentials to access S3 bucket\. If not supplied, environment variables \fBAWS_ACCESS_KEY_ID\fR and \fBAWS_SECRET_ACCESS_KEY\fR are used\. -. -.TP -\fBstorageClass\fR -(optional) Amazon S3 storage class, defaults to \fBSTANDARD\fR\. Other values available: \fBREDUCED_REDUNDANCY\fR (lower price, lower redundancy) -. -.TP -\fBencryptionMethod\fR -(optional) server\-side encryption method, defaults to none\. Currently the only available encryption method is \fBAES256\fR -. -.TP -\fBplusWorkaround\fR -(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot) -. -.TP -\fBdisableMultiDel\fR -(optional) for S3\-compatible cloud storages which do not support \fBMultiDel\fR S3 API, enable this setting (file deletion would be slower with this setting enabled) -. -.TP -\fBforceSigV2\fR -(optional) disable Signature V4 support, useful with non\-AWS S3\-compatible object stores which do not support SigV4, shouldn\(cqt be enabled for AWS -. -.TP -\fBforceVirtualHostedStyle\fR -(optional) disable path style visit, useful with non\-AWS S3\-compatible object stores which only support virtual hosted style -. -.TP -\fBdebug\fR -(optional) enables detailed request/response dump for each S3 operation -. -.P -In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.: -. -.P -\fBaptly publish snapshot wheezy\-main s3:test:\fR -. -.SH "OPENSTACK SWIFT PUBLISHING ENDPOINTS" -aptly could be configured to publish repository directly to OpenStack Swift\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings: -. -.TP -\fBcontainer\fR -container name -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the container, defaults to no prefix (container root) -. -.TP -\fBosname\fR, \fBpassword\fR -(optional) OpenStack credentials to access Keystone\. If not supplied, environment variables \fBOS_USERNAME\fR and \fBOS_PASSWORD\fR are used\. -. -.TP -\fBtenant\fR, \fBtenantid\fR -(optional) OpenStack tenant name and id (in order to use v2 authentication)\. -. -.TP -\fBauthurl\fR -(optional) the full url of Keystone server (including port, and version)\. example \fBhttp://identity\.example\.com:5000/v2\.0\fR -. -.P -In order to publish to Swift, specify endpoint as \fBswift:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.: -. -.P -\fBaptly publish snapshot jessie\-main swift:test:\fR -. -.SH "AZURE PUBLISHING ENDPOINTS" -aptly can be configured to publish repositories directly to Microsoft Azure Blob Storage\. First, publishing endpoints should be described in the aptly configuration file\. Each endpoint has its name and associated settings: -. -.TP -\fBcontainer\fR -container name -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the container, defaults to no prefix (container root) -. -.TP -\fBaccountName\fR, \fBaccountKey\fR -Azure storage account access key to access blob storage -. -.TP -\fBendpoint\fR -endpoint URL to connect to, as described in the Azure documentation \fIhttps://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string\fR; defaults to \fBhttps://$accountName\.blob\.core\.windows\.net\fR -. .SH "PACKAGE QUERY" Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms: . @@ -771,7 +831,7 @@ custom format for result printing include dependencies into search results . .SH "ADD PACKAGES TO LOCAL REPOSITORY" -\fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR +\fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR \fB(|)\|\.\|\.\|\.\fR . .P Command adds packages to local repository from \.deb, \.udeb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.[u]deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\. @@ -1057,7 +1117,7 @@ custom format for result printing include dependencies into search results . .SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES" -\fBaptly\fR \fBrepo\fR \fBinclude\fR +\fBaptly\fR \fBrepo\fR \fBinclude\fR \fB(|)\|\.\|\.\|\.\fR . .P Command include looks for \.changes files in list of arguments or specified directories\. Each \.changes file is verified, parsed, referenced files are put into separate temporary directory and added into local repository\. Successfully imported files are removed by default\. @@ -1103,7 +1163,7 @@ which repo should files go to, defaults to Distribution field of \.changes file path to uploaders\.json file . .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS" -\fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR +\fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fB(from\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty)\fR . .P Command create \fIname\fR from mirror makes persistent immutable snapshot of remote repository mirror\. Snapshot could be published or further modified using merge, pull and other aptly features\. @@ -1706,11 +1766,14 @@ don\(cqt sign Release files with GPG \-\fBsuite\fR= suite to publish (defaults to distribution) . -.SH "ADD SOURCE TO STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +.SH "ADD SOURCE COMPONENTS TO A PUBLISHED REPO" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBadd\fR \fIdistribution\fR \fIsource\fR . .P -The command adds sources to the staged source list of the published repository\. +The command adds components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: @@ -1719,7 +1782,7 @@ The flag \-component is mandatory\. Use a comma\-separated list of components, i . .nf -aptly publish add \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +aptly publish source add \-component=main,contrib wheezy wheezy\-main wheezy\-contrib . .fi . @@ -1732,7 +1795,7 @@ Example: . .nf -$ aptly publish add \-component=contrib wheezy ppa wheezy\-contrib +$ aptly publish source add \-component=contrib wheezy ppa wheezy\-contrib . .fi . @@ -1752,11 +1815,11 @@ component names to add (for multi\-component publishing, separate components wit \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "DROPS STAGED SOURCE CHANGES OF PUBLISHED REPOSITORY" +.SH "DROP PENDING SOURCE COMPONENT CHANGES OF A PUBLISHED REPOSITORY" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBdrop\fR \fIdistribution\fR . .P -Command drops the staged source changes of the published repository\. +Remove all pending changes what would be applied with a subsequent \(cqaptly publish update\(cq\. . .P Example: @@ -1816,11 +1879,14 @@ display record in JSON format \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "REMOVE SOURCE FROM STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +.SH "REMOVE SOURCE COMPONENTS FROM A PUBLISHED REPO" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBremove\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fIsource\fR . .P -The command removes sources from the staged source list of the published repository\. +The command removes source components (snapshot / local repo) from a published repository\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be removed, e\.g\.: @@ -1832,7 +1898,7 @@ Example: . .nf -$ aptly publish remove \-component=contrib,non\-free wheezy filesystem:symlink:debian +$ aptly publish source remove \-component=contrib,non\-free wheezy filesystem:symlink:debian . .fi . @@ -1849,11 +1915,14 @@ component names to remove (for multi\-component publishing, separate components \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "UPDATE SOURCE IN STAGED SOURCE LIST OF PUBLISHED REPOSITORY" -\fBaptly\fR \fBpublish\fR \fBsource\fR \fBupdate\fR \fIdistribution\fR \fIsource\fR +.SH "REPLACE THE SOURCE COMPONENTS OF A PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBreplace\fR \fIdistribution\fR \fIsource\fR . .P -The command updates sources in the staged source list of the published repository\. +The command replaces the source components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: @@ -1862,7 +1931,7 @@ The flag \-component is mandatory\. Use a comma\-separated list of components, i . .nf -aptly publish update \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +aptly publish source replace \-component=main,contrib wheezy wheezy\-main wheezy\-contrib . .fi . @@ -1875,7 +1944,53 @@ Example: . .nf -$ aptly publish update \-component=contrib wheezy ppa wheezy\-contrib +$ aptly publish source replace \-component=contrib wheezy ppa wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "UPDATE THE SOURCE COMPONENTS OF A PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBupdate\fR \fIdistribution\fR \fIsource\fR +. +.P +The command updates the source components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. +. +.P +The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: +. +.IP "" 4 +. +.nf + +aptly publish source update \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish source update \-component=contrib wheezy ppa wheezy\-contrib . .fi . @@ -1982,11 +2097,40 @@ don\(cqt generate Contents indexes \-\fBskip\-signing\fR don\(cqt sign Release files with GPG . -.SH "UPDATE PUBLISHED LOCAL REPOSITORY" +.SH "UPDATE PUBLISHED REPOSITORY" \fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] . .P -Command re\-publishes (updates) published local repository\. \fIdistribution\fR and \fIprefix\fR should be occupied with local repository published using command aptly publish repo\. Update happens in\-place with minimum possible downtime for published repository\. +The command updates updates a published repository after applying pending changes to the sources\. +. +.P +For published local repositories: +. +.IP "" 4 +. +.nf + +* update to match local repository contents +. +.fi +. +.IP "" 0 +. +.P +For published snapshots: +. +.IP "" 4 +. +.nf + +* switch components to new snapshot +. +.fi +. +.IP "" 0 +. +.P +The update happens in\-place with minimum possible downtime for published repository\. . .P For multiple component published repositories, all local repositories are updated\. @@ -2230,8 +2374,15 @@ Example: .P $ aptly config show . +.P +Options: +. +.TP +\-\fByaml\fR +show yaml config +. .SH "RUN APTLY TASKS" -\fBaptly\fR \fBtask\fR \fBrun\fR \-filename=\fIfilename\fR \fB|\fR \fIcommand1\fR, \fIcommand2\fR, \fB\|\.\|\.\|\.\fR +\fBaptly\fR \fBtask\fR \fBrun\fR (\-filename=\fIfilename\fR \fB|\fR \fIcommands\fR\|\.\|\.\|\.) . .P Command helps organise multiple aptly commands in one single aptly task, running as single thread\. @@ -2273,6 +2424,13 @@ Example: .P $ aptly config show . +.P +Options: +. +.TP +\-\fByaml\fR +show yaml config +. .SH "ENVIRONMENT" If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\. . @@ -2484,7 +2642,16 @@ Golf Hu (https://github\.com/hudeng\-go) Cookie Fei (https://github\.com/wuhuang26) . .IP "\[ci]" 4 +Andrey Loukhnov (https://github\.com/aol\-nnov) +. +.IP "\[ci]" 4 Christoph Fiehe (https://github\.com/cfiehe) . +.IP "\[ci]" 4 +Blake Kostner (https://github\.com/btkostner) +. +.IP "\[ci]" 4 +Leigh London (https://github\.com/leighlondon) +. .IP "" 0 diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index a04ac52d..b948aaf4 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -18,337 +18,384 @@ aptly has integrated help that matches contents of this manual page, to get help ## CONFIGURATION -aptly looks for configuration file first in `~/.aptly.conf` then -in `/usr/local/etc/aptly.conf` and `/etc/aptly.conf`. If no config file found (or they are not readable), a new one is created in the -home directory. If `-config=` flag is specified, aptly would use config file at specified -location. Also aptly needs root directory for database, package and published repository storage. -If not specified, directory defaults to `~/.aptly/`, it will be created if missing. +aptly looks for configuration file first in `~/.aptly.conf` then in `/usr/local/etc/aptly.conf` and `/etc/aptly.conf`. If no config file found (or they are not readable), a new one is created in the +home directory. If `-config=` flag is specified, aptly would use config file at specified location. Also aptly needs root directory for database, package and published repository storage. If not specified, directory defaults to `~/.aptly/`, it will be created if missing. -Configuration file is stored in JSON format (default values shown below): +With aptly version 1.6.0, yaml configuration with inline documentation is supported and recommended (see `debian/aptly.conf`). +The legacy json configuration is still supported: + + // vim: : filetype=json + // json configuration file with comments + // validate with: sed '/\/\//d' aptly.conf | json_pp { - "rootDir": "$HOME/.aptly", - "databaseBackend": { - "type": "", - "url": "" - }, - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", - "databaseOpenAttempts": 10, + + // Aptly Configuration File + //////////////////////////// + + // Root directory for: + // - downloaded packages (`rootDir`/pool) + // - database (`rootDir`/db) + // - published repositories (`rootDir`/public) + "rootDir": "~/.aptly", + + // Number of attempts to open database if it's locked by other instance + // * -1 (no retry) + "databaseOpenAttempts": -1, + + // Log Level + // * debug + // * info + // * warning + // * error + "logLevel": "info", + + // Log Format + // * default (text) + // * json + "logFormat": "default", + + // Default Architectures + // empty array defaults to all available architectures "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package "dependencyFollowRecommends": false, + + // When dependency looks like `package-a | package-b`, follow both variants always "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": { - "type": "local", - "path": "$ROOTDIR/pool", - "azure": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "" - } - }, - "skipLegacyPool": true, + + // Specifies paramaters for short PPA url expansion + // empty defaults to output of `lsb_release` command "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion "ppaCodename": "", + + // OBSOLETE + // in aptly up to version 1.0.0, package files were stored in internal package pool + // with MD5-dervied path, since 1.1.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + + + // Aptly Server + //////////////// + + // Serve published repos as well as API + "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + + // Database + //////////// + + // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd + "databaseBackend": { + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db + "dbPath": "" + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127.0.0.1:2379" + }, + + + // Mirroring + ///////////// + + // Downloader + // * "default" + // * "grab" (more robust) + "downloader": "default", + + // Number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // Limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // Number of retries for download attempts + "downloadRetries": 0, + + // Download source packages per default + "downloadSourcePackages": false, + + + // Signing + /////////// + + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) + "gpgProvider": "gpg", + + // Disable signing of published repositories + "gpgDisableSign": false, + + // Disable signature verification of remote repositories + "gpgDisableVerify": false, + + + // Publishing + ////////////// + + // Do not publish Contents files "skipContentsPublishing": false, + + // Do not create bz2 files + "skipBz2Publishing": false, + + + // Storage + /////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name + // and the following associated settings. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` + // with `endpoint-name` as the name given in the aptly configuration file. For example: + // + // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` + // "FileSystemPublishEndpoints": { - "test1": { - "rootDir": "/opt/srv1/aptly_public", - "linkMethod": "symlink" - }, - "test2": { - "rootDir": "/opt/srv2/aptly_public", - "linkMethod": "copy", - "verifyMethod": "md5" - }, - "test3": { - "rootDir": "/opt/srv3/aptly_public", - "linkMethod": "hardlink" - } + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (compare file size) + // "verifyMethod": "md5" + // } }, + + // S3 Endpoint Support + // + // cloud storage). First, publishing + // endpoints should be described in aptly configuration file. Each endpoint has name + // and associated settings. + // + // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main s3:test:` + // "S3PublishEndpoints": { - "test": { - "region": "us-east-1", - "bucket": "repo", - "endpoint": "", - "awsAccessKeyID": "", - "awsSecretAccessKey": "", - "prefix": "", - "acl": "public-read", - "storageClass": "", - "encryptionMethod": "", - "plusWorkaround": false, - "disableMultiDel": false, - "forceSigV2": false, - "forceVirtualHostedStyle": true, - "debug": false - } + // // Endpoint Name + // "test": { + + // // Amazon region for S3 bucket + // "region": "us-east-1", + + // // Bucket name + // "bucket": "test-bucket", + + // // Endpoint (optional) + // // When using S3-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology). Useful values: `private` (default), `public-read` (public + // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required. + // "acl": "private", + + // // Credentials (optional) + // // Amazon credentials to access S3 bucket. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + + // // Encryption Method (optional) + // // Server-side encryption method, defaults to none. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + + // // Disable MultiDel (optional) + // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores + // // which do not support SigV4, shouldn't be enabled for AWS + // "forceSigV2": false, + + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non-AWS S3-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } }, + + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift. First, + // publishing endpoints should be described in aptly configuration file. Each endpoint + // has name and associated settings. + // + // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot jessie-main swift:test:` + // "SwiftPublishEndpoints": { - "test": { - "container": "repo", - "osname": "", - "password": "", - "prefix": "", - "authurl": "", - "tenant": "", - "tenantid": "" - } + // Endpoint Name + // "test": { + + // // Container Name + // "container": "taylor1", + + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version). + // // Example `http://identity.example.com:5000/v2.0` + // "authurl": "" + // } }, + + // Azure Endpoint Support + // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage. First, publishing endpoints should be described in the aptly + // configuration file. Each endpoint has its name and associated settings. "AzurePublishEndpoints": { - "test": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "blob.core.windows.net" - } + // // Endpoint Name + // "test": { + + // // Container Name + // "container": "container1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool + "type": "local", + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" } + + // End of config } -Options: - - * `rootDir`: - is root of directory storage to store database (`rootDir`/db), - the default for downloaded packages (`rootDir`/pool) and - the default for published repositories (`rootDir`/public) and - skeleton files (`rootDir`/skel) - - * `databaseBackend`: - the database config; if this config is empty, use levledb backend by default - - * `downloadConcurrency`: - is a number of parallel download threads to use when downloading packages - - * `downloadSpeedLimit`: - limit in kbytes/sec on download speed while mirroring remote repositories - - * `downloadRetries`: - number of retries for download attempts - - * `databaseOpenAttempts`: - number of attempts to open DB if it's locked by other instance; could be overridden with option - `-db-open-attempts` - - * `architectures`: - is a list of architectures to process; if left empty defaults to all available architectures; could be - overridden with option `-architectures` - - * `dependencyFollowSuggests`: - follow contents of `Suggests:` field when processing dependencies for the package - - * `dependencyFollowRecommends`: - follow contents of `Recommends:` field when processing dependencies for the package - - * `dependencyFollowAllVariants`: - when dependency looks like `package-a | package-b`, follow both variants always - - * `dependencyFollowSource`: - follow dependency from binary package to source package - - * `dependencyVerboseResolve`: - print additional details while resolving dependencies (useful for debugging) - - * `gpgDisableSign`: - don't sign published repositories with gpg(1), also can be disabled on - per-repo basis using `-skip-signing` flag when publishing - - * `gpgDisableVerify`: - don't verify remote mirrors with gpg(1), also can be disabled on - per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors - - * `gpgProvider`: - implementation of PGP signing/validation - `gpg` for external `gpg` utility or - `internal` to use Go internal implementation; `gpg1` might be used to force use - of GnuPG 1.x, `gpg2` enables GnuPG 2.x only; default is to use GnuPG 1.x if - available and GnuPG 2.x otherwise - - * `downloadSourcePackages`: - if enabled, all mirrors created would have flag set to download source packages; - this setting could be controlled on per-mirror basis with `-with-sources` flag - - * `packagePoolStorage`: - configures the location to store downloaded packages (defaults to the - path `$ROOTDIR/pool`), by setting the value of the `type`: - * `path`: store the packages in the given path - * `azure`: store the packages in the given Azure Blob Storage container - (see the section on Azure publishing below for information on the - configuration) - - * `skipLegacyPool`: - in aptly up to version 1.0.0, package files were stored in internal package pool - with MD5-dervied path, since 1.1.0 package pool layout was changed; - if option is enabled, aptly stops checking for legacy paths; - by default option is enabled for new aptly installations and disabled when - upgrading from older versions - - * `ppaDistributorID`, `ppaCodename`: - specifies paramaters for short PPA url expansion, if left blank they default - to output of `lsb_release` command - - * `FileSystemPublishEndpoints`: - configuration of local filesystem publishing endpoints (see below) - - * `S3PublishEndpoints`: - configuration of Amazon S3 publishing endpoints (see below) - - * `SwiftPublishEndpoints`: - configuration of OpenStack Swift publishing endpoints (see below) - - * `AzurePublishEndpoints`: - configuration of Azure publishing endpoints (see below) - -## CUSTOM PACKAGE POOLS - -aptly defaults to storing downloaded packages at `rootDir/`pool. In order to -change this, you can set the `type` key within `packagePoolStorage` to one of -two values: - - * `local`: Store the package pool locally (the default). In order to change - the path, additionally set the `path` key within `packagePoolStorage` to - the desired location. - * `azure`: Store the package pool in an Azure Blob Storage container. Any - keys in the below section on Azure publishing may be set on the - `packagePoolStorage` object in order to configure the Azure connection. - -## FILESYSTEM PUBLISHING ENDPOINTS - -aptly defaults to publish to a single publish directory under `rootDir`/public. For -a more advanced publishing strategy, you can define one or more filesystem endpoints in the -`FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name -and the following associated settings: - - * `rootDir`: - The publish directory, e.g., `/opt/srv/aptly_public`. - * `linkMethod`: - This is one of `hardlink`, `symlink` or `copy`. It specifies how aptly links the - files from the internal pool to the published directory. - If not specified, empty or wrong, this defaults to `hardlink`. - * `verifyMethod`: - This is used only when setting the `linkMethod` to `copy`. Possible values are - `md5` and `size`. It specifies how aptly compares existing links from the - internal pool to the published directory. The `size` method compares only the - file sizes, whereas the `md5` method calculates the md5 checksum of the found - file and compares it to the desired one. - If not specified, empty or wrong, this defaults to `md5`. - -In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` -with `endpoint-name` as the name given in the aptly configuration file. For example: - - `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - -## S3 PUBLISHING ENDPOINTS - -aptly could be configured to publish repository directly to Amazon S3 (or S3-compatible -cloud storage). First, publishing -endpoints should be described in aptly configuration file. Each endpoint has name -and associated settings: - - * `region`: - Amazon region for S3 bucket (e.g. `us-east-1`) - * `bucket`: - bucket name - * `endpoint`: - (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here, - region is ignored if endpoint is set (set region to some human-readable name) - (should be left blank for real Amazon S3) - * `prefix`: - (optional) do publishing under specified prefix in the bucket, defaults to - no prefix (bucket root) - * `acl`: - (optional) assign ACL to published files (one of the canned ACLs in Amazon - terminology). Useful values: `private` (default), `public-read` (public - repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - HTTP endpoint (Amazon bucket should be configured for "website hosting"), - for private repositories special apt S3 transport is required. - * `awsAccessKeyID`, `awsSecretAccessKey`: - (optional) Amazon credentials to access S3 bucket. If not supplied, - environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - are used. - * `storageClass`: - (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values - available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - * `encryptionMethod`: - (optional) server-side encryption method, defaults to none. Currently - the only available encryption method is `AES256` - * `plusWorkaround`: - (optional) workaround misbehavior in apt and Amazon S3 - for files with `+` in filename by - creating two copies of package files with `+` in filename: one original - and another one with spaces instead of plus signs - With `plusWorkaround` enabled, package files with plus sign - would be stored twice. aptly might not cleanup files with spaces when published - repository is dropped or updated (switched) to new version of repository (snapshot) - * `disableMultiDel`: - (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API, - enable this setting (file deletion would be slower with this setting enabled) - * `forceSigV2`: - (optional) disable Signature V4 support, useful with non-AWS S3-compatible object stores - which do not support SigV4, shouldn't be enabled for AWS - * `forceVirtualHostedStyle`: - (optional) disable path style visit, useful with non-AWS S3-compatible object stores - which only support virtual hosted style - * `debug`: - (optional) enables detailed request/response dump for each S3 operation - -In order to publish to S3, specify endpoint as `s3:endpoint-name:` before -publishing prefix on the command line, e.g.: - - `aptly publish snapshot wheezy-main s3:test:` - -## OPENSTACK SWIFT PUBLISHING ENDPOINTS - -aptly could be configured to publish repository directly to OpenStack Swift. First, -publishing endpoints should be described in aptly configuration file. Each endpoint -has name and associated settings: - - * `container`: - container name - * `prefix`: - (optional) do publishing under specified prefix in the container, defaults to - no prefix (container root) - * `osname`, `password`: - (optional) OpenStack credentials to access Keystone. If not supplied, - environment variables `OS_USERNAME` and `OS_PASSWORD` are used. - * `tenant`, `tenantid`: - (optional) OpenStack tenant name and id (in order to use v2 authentication). - * `authurl`: - (optional) the full url of Keystone server (including port, and version). - example `http://identity.example.com:5000/v2.0` - -In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before -publishing prefix on the command line, e.g.: - - `aptly publish snapshot jessie-main swift:test:` - -## AZURE PUBLISHING ENDPOINTS - -aptly can be configured to publish repositories directly to Microsoft Azure Blob -Storage. First, publishing endpoints should be described in the aptly -configuration file. Each endpoint has its name and associated settings: - - * `container`: - container name - * `prefix`: - (optional) do publishing under specified prefix in the container, defaults to - no prefix (container root) - * `accountName`, `accountKey`: - Azure storage account access key to access blob storage - * `endpoint`: - endpoint URL to connect to, as described in - [the Azure documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string); - defaults to `https://$accountName.blob.core.windows.net` ## PACKAGE QUERY diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 24bbbecf..38a8a56a 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -1,7 +1,11 @@ # Aptly Configuration File ########################### +# vim: : filetype=yaml -# Aptly storage directory for: +# aptly 1.6.0 supports yaml configuraiton files with inline documentation and examples. +# Legacy json config files are still supported, and may be converted to yaml with `aptly config show -yaml` + +# Root directory for: # - downloaded packages (`rootDir`/pool) # - database (`rootDir`/db) # - published repositories (`rootDir`/public) diff --git a/utils/config.go b/utils/config.go index c0cc12d6..da7dfcd6 100644 --- a/utils/config.go +++ b/utils/config.go @@ -261,7 +261,7 @@ func LoadConfig(filename string, config *ConfigStructure) error { err = fmt.Errorf("invalid yaml (%s) or json (%s)", err2, err) } else { err = nil - } + } } return err } diff --git a/utils/config_test.go b/utils/config_test.go index 3317f05d..66f1ad03 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -299,7 +299,7 @@ dep_follow_all_variants: true dep_follow_source: true dep_verboseresolve: true ppa_distributor_id: Ubuntu -ppa_codename: short +ppa_codename: code serve_in_api_mode: true enable_metrics_endpoint: true enable_swagger_endpoint: true @@ -327,7 +327,7 @@ s3_publish_endpoints: test: region: us-east-1 bucket: test-bucket - prefix: "" + prefix: prfx acl: public-read access_key_id: "2" secret_access_key: secret @@ -366,7 +366,7 @@ packagepool_storage: prefix: pre3 account_name: a name account_key: a key - endpoint: "" + endpoint: ep ` const configFileYAMLError = `packagepool_storage: type: invalid From b14595cb2d6eadab27c44517918f21c70f466021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 13:49:20 +0100 Subject: [PATCH 61/64] cleanup makefile --- Makefile | 6 +- debian/aptly.conf.yaml | 339 ----------------------------------------- 2 files changed, 3 insertions(+), 342 deletions(-) delete mode 100644 debian/aptly.conf.yaml diff --git a/Makefile b/Makefile index ba282293..1794a6b2 100644 --- a/Makefile +++ b/Makefile @@ -50,11 +50,11 @@ swagger-install: echo "// @version $(VERSION)" >> docs/swagger.conf azurite-start: - azurite & \ + azurite -l /tmp/aptly-azurite & \ echo $$! > ~/.azurite.pid azurite-stop: - kill `cat ~/.azurite.pid` + @kill `cat ~/.azurite.pid` swagger: swagger-install # Generate swagger docs @@ -210,7 +210,7 @@ man: ## Create man pages clean: ## remove local build and module cache # Clean all generated and build files - find .go/ -type d ! -perm -u=w -exec chmod u+w {} \; + test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \; rm -rf .go/ rm -rf build/ obj-*-linux-gnu* tmp/ rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf diff --git a/debian/aptly.conf.yaml b/debian/aptly.conf.yaml deleted file mode 100644 index 24bbbecf..00000000 --- a/debian/aptly.conf.yaml +++ /dev/null @@ -1,339 +0,0 @@ -# Aptly Configuration File -########################### - -# Aptly storage directory for: -# - downloaded packages (`rootDir`/pool) -# - database (`rootDir`/db) -# - published repositories (`rootDir`/public) -root_dir: ~/.aptly - -# Log Level -# * debug -# * info -# * warning -# * error -log_level: info - -# Log Format -# * default (text) -# * json -log_format: default - -# Number of attempts to open database if it's locked by other instance -# * -1 (no retry) -database_open_attempts: -1 - -# Default Architectures -# empty list defaults to all available architectures -architectures: -# - amd64 - -# OBSOLETE -# in aptly up to version 1.0.0, package files were stored in internal package pool -# with MD5-dervied path, since 1.1.0 package pool layout was changed; -# if option is enabled, aptly stops checking for legacy paths; -# by default option is enabled for new aptly installations and disabled when -# upgrading from older versions -skip_legacy_pool: true - - -# Dependency following -####################### - -# Follow contents of `Suggests:` field when processing dependencies for the package -dep_follow_suggests: false - -# Follow contents of `Recommends:` field when processing dependencies for the package -dep_follow_recommends: false - -# When dependency looks like `package-a | package-b`, follow both variants always -dep_follow_allvariants: false - -# Follow dependency from binary package to source package -dep_follow_source: false - -# Log additional details while resolving dependencies (useful for debugging) -dep_verbose_resolve: false - - -# PPA -###### - -# Specify paramaters for short PPA url expansion -# empty defaults to output of `lsb_release` command -ppa_distributor_id: ubuntu - -# Codename for short PPA url expansion -ppa_codename: "" - - -# Aptly Server -############### - -# Serve published repos as well as API -serve_in_api_mode: false - -# Enable metrics for Prometheus client -enable_metrics_endpoint: false - -# Enable API documentation on /docs -enable_swagger_endpoint: false - -# OBSOLETE: use via url param ?_async=true -async_api: false - - -# Database -########### - -# Database backend -# Type must be one of: -# * leveldb (default) -# * etcd -database_backend: - type: leveldb - # Path to leveldb files - # empty dbPath defaults to `rootDir`/db - db_path: "" - - # type: etcd - # # URL to db server - # url: "127.0.0.1:2379" - - -# Mirroring -############ - -# Downloader -# * "default" -# * "grab" (more robust) -downloader: default - -# Number of parallel download threads to use when downloading packages -download_concurrency: 4 - -# Limit in kbytes/sec on download speed while mirroring remote repositories -download_limit: 0 - -# Number of retries for download attempts -download_retries: 0 - -# Download source packages per default -download_sourcepackages: false - - -# Signing -########## - -# GPG Provider -# * "internal" (Go internal implementation) -# * "gpg" (External `gpg` utility) -gpg_provider: gpg - -# Disable signing of published repositories -gpg_disable_sign: false - -# Disable signature verification of remote repositories -gpg_disable_verify: false - - -# Publishing -############# - -# Do not publish Contents files -skip_contents_publishing: false - -# Do not create bz2 files -skip_bz2_publishing: false - - -# Storage -########## - -# Filesystem publishing endpoints -# -# aptly defaults to publish to a single publish directory under `rootDir`/public. For -# a more advanced publishing strategy, you can define one or more filesystem endpoints in the -# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name -# and the following associated settings. -# -# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` -# with `endpoint-name` as the name given in the aptly configuration file. For example: -# -# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` -# -filesystem_publish_endpoints: - # # Endpoint Name - # test1: - # # Directory for publishing - # root_dir: /opt/srv/aptly_public - # # File Link Method for linking files from the internal pool to the published directory - # # * hardlink - # # * symlink - # # * copy - # link_method: hardlink - # # File Copare Method for comparing existing links from the internal pool to the published directory - # # Only used when "linkMethod" is set to "copy" - # # * md5 (default: compare md5 sum) - # # * size (compare file size) - # verify_method: md5 - -# S3 Endpoint Support -# -# cloud storage). First, publishing -# endpoints should be described in aptly configuration file. Each endpoint has name -# and associated settings. -# -# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before -# publishing prefix on the command line, e.g.: -# -# `aptly publish snapshot wheezy-main s3:test:` -# -s3_publish_endpoints: - # # Endpoint Name - # test: - # # Amazon region for S3 bucket - # region: us-east-1 - # # Bucket name - # bucket: test-bucket - # # Prefix (optional) - # # publishing under specified prefix in the bucket, defaults to - # # no prefix (bucket root) - # prefix: "" - # # Default ACLs (optional) - # # assign ACL to published files: - # # * private (default, for use with apt S3 transport) - # # * public-read (public repository) - # # * none (don't set ACL) - # acl: private - # # Credentials (optional) - # # Amazon credentials to access S3 bucket. If not supplied, environment variables - # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used - # access_key_id: "" - # secret_access_key: "" - # session_token: "" - # # Endpoint (optional) - # # When using S3-compatible cloud storage, specify hostname of service endpoint here, - # # region is ignored if endpoint is set (set region to some human-readable name) - # # (should be left blank for real Amazon S3) - # endpoint: "" - # # Storage Class (optional) - # # Amazon S3 storage class, defaults to `STANDARD`. Other values - # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - # storage_class: STANDARD - # # Encryption Method (optional) - # # Server-side encryption method, defaults to none. Currently - # # the only available encryption method is `AES256` - # encryption_method: none - # # Plus Workaround (optional) - # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - # # creating two copies of package files with `+` in filename: one original - # # and another one with spaces instead of plus signs - # # With `plusWorkaround` enabled, package files with plus sign - # # would be stored twice. aptly might not cleanup files with spaces when published - # # repository is dropped or updated (switched) to new version of repository (snapshot) - # plus_workaround: false - # # Disable MultiDel (optional) - # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, - # # enable this setting (file deletion would be slower with this setting enabled) - # disable_multidel: false - # # Force Signature v2 (optional) - # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores - # # which do not support SigV4, shouldn't be enabled for AWS - # force_sigv2: false - # # Force VirtualHosted Style (optional) - # # Disable path style visit, useful with non-AWS S3-compatible object stores - # # which only support virtual hosted style - # force_virtualhosted_style: false - # # Debug (optional) - # # Enables detailed request/response dump for each S3 operation - # debug: false - -# Swift Endpoint Support -# -# aptly can publish a repository directly to OpenStack Swift. -# Each endpoint has name and associated settings. -# -# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before -# publishing prefix on the command line, e.g.: -# -# `aptly publish snapshot jessie-main swift:test:` -# -swift_publish_endpoints: - # # Endpoint Name - # test: - # # Container Name - # container: taylor1 - # # Prefix (optional) - # # Publish under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials (optional) - # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - # username: "" - # password: "" - # # Domain (optional) - # # OpenStack domain - # domain: "" - # domain_id: "" - # # Tenant (optional) - # # OpenStack tenant (in order to use v2 authentication) - # tenant: "" - # tenant_id: "" - # tenant_domain: "" - # tenant_domain_id: "" - # # Auth URL (optional) - # # Full url of Keystone server (including port, and version). - # # Example `http://identity.example.com:5000/v2.0` - # auth_url: "" - -# Azure Endpoint Support -# -# aptly can be configured to publish repositories directly to Microsoft Azure Blob -# Storage. First, publishing endpoints should be described in the aptly -# configuration file. Each endpoint has its name and associated settings. -azure_publish_endpoints: - # # Endpoint Name - # test: - # # Container Name - # container: container1 - # # Prefix (optional) - # # Publishing under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials - # # Azure storage account access key to access blob storage - # account_name: "" - # account_key: "" - # # Endpoint URL - # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - # # defaults to "https://.blob.core.windows.net" - # endpoint: "" - -# Package Pool -# -# Location for storing downloaded packages -# Type must be one of: -# * local -# * azure -packagepool_storage: - # Local Pool - type: local - # Local Pool Path - # empty path defaults to `rootDir`/pool - path: - - # # Azure Azure Blob Storage Pool - # type: azure - # # Container Name - # container: pool1 - # # Prefix (optional) - # # Publishing under specified prefix in the container, defaults to no prefix (container root) - # prefix: "" - # # Credentials - # # Azure storage account access key to access blob storage - # account_name: "" - # account_key: "" - # # Endpoint URL - # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - # # defaults to "https://.blob.core.windows.net" - # endpoint: "" - From 0d90ff96b9fdf63efbd8f66bed2b4e6ea4e0cfcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 4 Dec 2024 23:21:15 +0100 Subject: [PATCH 62/64] debian: add build dependency for yaml --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index f7ed7f09..101dbcf1 100644 --- a/debian/control +++ b/debian/control @@ -77,6 +77,7 @@ Build-Depends: bash-completion, golang-go.uber-multierr-dev, golang-go.uber-zap-dev, golang-etcd-server-dev (>= 3.5.15-7), + golang-gopkg-yaml.v3-dev, git Standards-Version: 4.7.0 Homepage: https://www.aptly.info From 35ad56ff7f137b56f79229aa2f4e475e074e6650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 11 Dec 2024 05:57:45 +0100 Subject: [PATCH 63/64] Revert "debian: do not conflict with gnupg1" This reverts commit 2f540a8026e2bb0901f8aa266ce9e21448c88c98. --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 101dbcf1..b6c4d8de 100644 --- a/debian/control +++ b/debian/control @@ -90,6 +90,7 @@ Package: aptly Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, bzip2, xz-utils, gpgv, gpg Suggests: graphviz +Conflicts: gnupg1, gpgv1 Built-Using: ${misc:Static-Built-Using}, ${misc:Built-Using} Description: Swiss army knife for Debian repository management - main package It offers several features making it easy to manage Debian package From 76d3b278422efc0e49e2ea62bd261a46ebd97071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 11 Dec 2024 06:10:33 +0100 Subject: [PATCH 64/64] update changelog --- debian/changelog | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 47d58eb1..2ea00ddc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,27 @@ -aptly (1.6.0~rc1) stable; urgency=medium +aptly (1.6.0) stable; urgency=medium - * release candidate + * fix mirroring source packages + * support yaml config per default + * provide swagger API documentation + * provide API for querying aptly storage usage + * provide snapshot pull via API + * support creating repos from snapshots + * fix mirroring flat remote repos + * support skeleton files for publishing + * use new azure sdk + * support updating the components of a published repo + * support publishing multiple distributions (-multi-dist) + * support etcd database + * allow slash (/) in distribution names + * support for storing the "local" pool on Azure + * provide copy package API + * fix publish concurrency and improve performance + * improved mirroring + * fix download throttling + * fix resuming package downloads + * fix ignoring signatures + * fix packages dependency resolution (Virtual Packages, version numbers in Provides) + * improved S3 support and performance + * fix race condition with goleveldb -- André Roth Sat, 16 Nov 2024 12:44:06 +0100