From b28daa84174acd6e1a949d13267998e29b77bca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 18 Jun 2026 13:52:09 +0200 Subject: [PATCH] initial commit for JFrog support # Conflicts: # api/api_test.go # api/router.go # go.mod # go.sum # utils/config_test.go --- .github/workflows/ci.yml | 3 + api/api_test.go | 18 +++++- debian/aptly.conf | 23 +++++++ docs/Publish.md | 2 +- man/aptly.1.ronn.tmpl | 18 ++++++ system/jfrog_lib.py | 99 ++++++++++++++++++++++++++++++ system/t06_publish/jfrog.py | 116 ++++++++++++++++++++++++++++++++++++ 7 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 system/jfrog_lib.py create mode 100644 system/t06_publish/jfrog.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4c6efb1..a2cc14a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,9 @@ jobs: AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + JFROG_URL: ${{ secrets.JFROG_URL }} + JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }} + JFROG_PASSWORD: ${{ secrets.JFROG_PASSWORD }} run: | sudo mkdir -p /srv ; sudo chown runner /srv mkdir -p out/coverage diff --git a/api/api_test.go b/api/api_test.go index 835361bb..b6a5b11f 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -51,8 +51,10 @@ func createTestConfig() *os.File { "GcsPublishEndpoints": map[string]map[string]string{ "test-gcs": { "bucket": "bucket-gcs", - "JFrogPublishEndpoints": gin.H{ - "test-jfrog": gin.H{ + }, + }, + "JFrogPublishEndpoints": map[string]map[string]string{ + "test-jfrog": { "url": "http://jfrog.example.com", }, }, @@ -189,6 +191,18 @@ func (s *APISuite) TestTruthy(c *C) { c.Check(truthy(gin.H{}), Equals, true) } +func (s *APISuite) TestGetJFrogEndpoints(c *C) { + response, err := s.HTTPRequest("GET", "/api/jfrog", nil) + c.Assert(err, IsNil) + c.Check(response.Code, Equals, 200) + + var endpoints []string + err = json.Unmarshal(response.Body.Bytes(), &endpoints) + c.Assert(err, IsNil) + sort.Strings(endpoints) + c.Check(endpoints, DeepEquals, []string{"test-jfrog"}) +} + func (s *APISuite) TestGetS3Endpoints(c *C) { response, err := s.HTTPRequest("GET", "/api/s3", nil) c.Assert(err, IsNil) diff --git a/debian/aptly.conf b/debian/aptly.conf index 93177314..98e75bce 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -196,6 +196,29 @@ filesystem_publish_endpoints: # # `aptly publish snapshot wheezy-main s3:test:` # + +# JFrog Artifactory Endpoint Support +# +# aptly can be configured to publish repositories directly to JFrog Artifactory. First, +# publishing endpoints should be described in the aptly configuration file. +# +# In order to publish to JFrog, specify endpoint as `jfrog:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main jfrog:test:` +# +jfrog_publish_endpoints: + # # Endpoint Name + # test: + # # JFrog URL + # url: "https://artifactory.example.com/artifactory/" + # # Repository + # repository: apt-local + # # Username + # username: admin + # # Password + # password: password + s3_publish_endpoints: # # Endpoint Name # test: diff --git a/docs/Publish.md b/docs/Publish.md index d6a8fd75..36b0edc4 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -5,7 +5,7 @@ Publish snapshot or local repo as Debian repository to be used as APT source on The published repository is signed with the user's GnuPG key. -Repositories can be published to local directories, Amazon S3 buckets, Azure or Swift Storage. +Repositories can be published to local directories, Amazon S3 buckets, Azure, Swift, or JFrog Artifactory Storage. #### GPG Keys diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 04364310..acfcc612 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -356,6 +356,24 @@ The legacy json configuration is still supported (and also supports comments): // } }, + // JFrog Artifactory Endpoint Support + // aptly could be configured to publish repository directly to JFrog Artifactory. First, + // endpoints should be described in aptly.conf: + // + // In order to publish to JFrog, specify endpoint as `jfrog:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main jfrog:test:` + // + "JFrogPublishEndpoints": { + "test": { + "url": "https://artifactory.example.com/artifactory/", + "repository": "apt-local", + "username": "admin", + "password": "password" + } + } + // Swift Endpoint Support // // aptly could be configured to publish repository directly to OpenStack Swift. First, diff --git a/system/jfrog_lib.py b/system/jfrog_lib.py new file mode 100644 index 00000000..6dc1f5dc --- /dev/null +++ b/system/jfrog_lib.py @@ -0,0 +1,99 @@ +from lib import BaseTest +import uuid +import os + +try: + import requests + + if 'JFROG_URL' in os.environ and 'JFROG_USERNAME' in os.environ and \ + os.environ['JFROG_URL'] != "" and os.environ['JFROG_USERNAME'] != "": + jfrog_ready = True + else: + print("JFrog tests disabled: JFrog creds not found in the environment (JFROG_URL, JFROG_USERNAME, JFROG_PASSWORD)") + jfrog_ready = False +except ImportError as e: + print("JFrog tests disabled: can't import requests", e) + jfrog_ready = False + + +class JFrogTest(BaseTest): + """ + BaseTest + support for JFrog + """ + + jfrogOverrides = {} + + def fixture_available(self): + return super(JFrogTest, self).fixture_available() and jfrog_ready + + def prepare(self): + self.repository = "aptly-sys-test-" + str(uuid.uuid1()) + self.jfrog_url = os.environ["JFROG_URL"] + self.jfrog_username = os.environ["JFROG_USERNAME"] + self.jfrog_password = os.environ["JFROG_PASSWORD"] + + # Create repository via REST API + auth = (self.jfrog_username, self.jfrog_password) + create_url = f"{self.jfrog_url}/api/repositories/{self.repository}" + payload = { + "key": self.repository, + "rclass": "local", + "packageType": "generic" + } + res = requests.put(create_url, json=payload, auth=auth) + if res.status_code >= 400: + raise Exception(f"Failed to create JFrog repository: {res.text}") + + self.configOverride = {"JFrogPublishEndpoints": { + "test1": { + "url": self.jfrog_url, + "repository": self.repository, + "username": self.jfrog_username, + "password": self.jfrog_password + } + }} + + self.configOverride["JFrogPublishEndpoints"]["test1"].update(**self.jfrogOverrides) + + super(JFrogTest, self).prepare() + + def shutdown(self): + if hasattr(self, "repository"): + auth = (self.jfrog_username, self.jfrog_password) + delete_url = f"{self.jfrog_url}/api/repositories/{self.repository}" + requests.delete(delete_url, auth=auth) + + super(JFrogTest, self).shutdown() + + + def check_path(self, path): + if path.startswith("public/"): + path = path[7:] + + # Check against JFrog Artifactory API + auth = (self.jfrog_username, self.jfrog_password) + check_url = f"{self.jfrog_url}/api/storage/{self.repository}/{path}" + res = requests.head(check_url, auth=auth) + if res.status_code == 200: + return True + return False + + def check_exists(self, path): + if not self.check_path(path): + raise Exception("path %s doesn't exist" % (path, )) + + def check_not_exists(self, path): + if self.check_path(path): + raise Exception("path %s exists" % (path, )) + + def read_file(self, path, mode=''): + assert not mode + if path.startswith("public/"): + path = path[7:] + + auth = (self.jfrog_username, self.jfrog_password) + get_url = f"{self.jfrog_url}/{self.repository}/{path}" + res = requests.get(get_url, auth=auth) + res.raise_for_status() + return res.text + diff --git a/system/t06_publish/jfrog.py b/system/t06_publish/jfrog.py new file mode 100644 index 00000000..fd319c11 --- /dev/null +++ b/system/t06_publish/jfrog.py @@ -0,0 +1,116 @@ +from jfrog_lib import JFrogTest + + +def strip_processor(output): + return '\n'.join( + [ + l + for l in output.split('\n') + if not l.startswith(' ') and not l.startswith('Date:') + ] + ) + + +class JFrogPublish1Test(JFrogTest): + """ + publish to JFrog: from repo + """ + + fixtureCmds = [ + 'aptly repo create -distribution=maverick local-repo', + 'aptly repo add local-repo ${files}', + 'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386', + ] + runCmd = 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo jfrog:test1:' + + def check(self): + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/source/Sources') + self.check_exists('public/dists/maverick/main/source/Sources.gz') + self.check_exists('public/dists/maverick/main/source/Sources.bz2') + + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + ) + + # verify contents except sums/date chunks + self.check_file_contents( + 'public/dists/maverick/Release', 'release', match_prepare=strip_processor + ) + self.check_file_contents( + 'public/dists/maverick/main/source/Sources', + 'sources', + match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))), + ) + self.check_file_contents( + 'public/dists/maverick/main/binary-i386/Packages', + 'binary', + match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))), + ) + + +class JFrogPublish2Test(JFrogTest): + """ + publish to JFrog: update after removing package from repo + """ + + fixtureCmds = [ + 'aptly repo create -distribution=maverick local-repo', + 'aptly repo add local-repo ${files}/', + 'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo jfrog:test1:', + 'aptly repo remove local-repo pyspi', + ] + runCmd = 'aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick jfrog:test1:' + + def check(self): + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + ) + + +class JFrogPublish3Test(JFrogTest): + """ + publish to JFrog: publish drop performs cleanup + """ + + fixtureCmds = [ + 'aptly repo create local1', + 'aptly repo create local2', + 'aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb', + 'aptly repo add local2 ${files}', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 jfrog:test1:', + 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 jfrog:test1:', + ] + runCmd = 'aptly publish drop sq2 jfrog:test1:' + + def check(self): + self.check_exists('public/dists/sq1') + self.check_not_exists('public/dists/sq2') + self.check_exists('public/pool/main/') + + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') + self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc') + self.check_exists( + 'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb' + )