initial commit for JFrog support

# Conflicts:
#	api/api_test.go
#	api/router.go
#	go.mod
#	go.sum
#	utils/config_test.go

.github/workflows/ci.yml

@@ -100,6 +100,9 @@ jobs:
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          JFROG_URL: ${{ secrets.JFROG_URL }}
+          JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }}
+          JFROG_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
         run: |
           sudo mkdir -p /srv ; sudo chown runner /srv
           mkdir -p out/coverage

api/api_test.go

@@ -51,8 +51,10 @@ func createTestConfig() *os.File {
 		"GcsPublishEndpoints": map[string]map[string]string{
 			"test-gcs": {
 				"bucket": "bucket-gcs",
-		"JFrogPublishEndpoints": gin.H{
+			},
-			"test-jfrog": gin.H{
+		},
+		"JFrogPublishEndpoints": map[string]map[string]string{
+			"test-jfrog": {
 				"url": "http://jfrog.example.com",
 			},
 		},
@@ -189,6 +191,18 @@ func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy(gin.H{}), Equals, true)
 }
 
+func (s *APISuite) TestGetJFrogEndpoints(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/jfrog", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+
+	var endpoints []string
+	err = json.Unmarshal(response.Body.Bytes(), &endpoints)
+	c.Assert(err, IsNil)
+	sort.Strings(endpoints)
+	c.Check(endpoints, DeepEquals, []string{"test-jfrog"})
+}
+
 func (s *APISuite) TestGetS3Endpoints(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/s3", nil)
 	c.Assert(err, IsNil)

debian/aptly.conf

@@ -196,6 +196,29 @@ filesystem_publish_endpoints:
 #
 #   `aptly publish snapshot wheezy-main s3:test:`
 #
+
+# JFrog Artifactory Endpoint Support
+#
+# aptly can be configured to publish repositories directly to JFrog Artifactory. First,
+# publishing endpoints should be described in the aptly configuration file.
+#
+# In order to publish to JFrog, specify endpoint as `jfrog:endpoint-name:` before
+# publishing prefix on the command line, e.g.:
+#
+#   `aptly publish snapshot wheezy-main jfrog:test:`
+#
+jfrog_publish_endpoints:
+    # # Endpoint Name
+    # test:
+    #     # JFrog URL
+    #     url: "https://artifactory.example.com/artifactory/"
+    #     # Repository
+    #     repository: apt-local
+    #     # Username
+    #     username: admin
+    #     # Password
+    #     password: password
+
 s3_publish_endpoints:
     # # Endpoint Name
     # test:

docs/Publish.md

@@ -5,7 +5,7 @@ Publish snapshot or local repo as Debian repository to be used as APT source on
 
 The published repository is signed with the user's GnuPG key.
 
-Repositories can be published to local directories, Amazon S3 buckets, Azure or Swift Storage.
+Repositories can be published to local directories, Amazon S3 buckets, Azure, Swift, or JFrog Artifactory Storage.
 
 #### GPG Keys
 

man/aptly.1.ronn.tmpl

@@ -356,6 +356,24 @@ The legacy json configuration is still supported (and also supports comments):
         // }
       },
 
+      // JFrog Artifactory Endpoint Support
+      // aptly could be configured to publish repository directly to JFrog Artifactory. First,
+      // endpoints should be described in aptly.conf:
+      //
+      // In order to publish to JFrog, specify endpoint as `jfrog:endpoint-name:` before
+      // publishing prefix on the command line, e.g.:
+      //
+      //   `aptly publish snapshot wheezy-main jfrog:test:`
+      //
+      "JFrogPublishEndpoints": {
+        "test": {
+          "url": "https://artifactory.example.com/artifactory/",
+          "repository": "apt-local",
+          "username": "admin",
+          "password": "password"
+        }
+      }
+
       // Swift Endpoint Support
       //
       // aptly could be configured to publish repository directly to OpenStack Swift. First,

system/jfrog_lib.py

@@ -0,0 +1,99 @@
+from lib import BaseTest
+import uuid
+import os
+
+try:
+    import requests
+
+    if 'JFROG_URL' in os.environ and 'JFROG_USERNAME' in os.environ and \
+       os.environ['JFROG_URL'] != "" and os.environ['JFROG_USERNAME'] != "":
+        jfrog_ready = True
+    else:
+        print("JFrog tests disabled: JFrog creds not found in the environment (JFROG_URL, JFROG_USERNAME, JFROG_PASSWORD)")
+        jfrog_ready = False
+except ImportError as e:
+    print("JFrog tests disabled: can't import requests", e)
+    jfrog_ready = False
+
+
+class JFrogTest(BaseTest):
+    """
+    BaseTest + support for JFrog
+    """
+
+    jfrogOverrides = {}
+
+    def fixture_available(self):
+        return super(JFrogTest, self).fixture_available() and jfrog_ready
+
+    def prepare(self):
+        self.repository = "aptly-sys-test-" + str(uuid.uuid1())
+        self.jfrog_url = os.environ["JFROG_URL"]
+        self.jfrog_username = os.environ["JFROG_USERNAME"]
+        self.jfrog_password = os.environ["JFROG_PASSWORD"]
+        
+        # Create repository via REST API
+        auth = (self.jfrog_username, self.jfrog_password)
+        create_url = f"{self.jfrog_url}/api/repositories/{self.repository}"
+        payload = {
+            "key": self.repository,
+            "rclass": "local",
+            "packageType": "generic"
+        }
+        res = requests.put(create_url, json=payload, auth=auth)
+        if res.status_code >= 400:
+            raise Exception(f"Failed to create JFrog repository: {res.text}")
+
+        self.configOverride = {"JFrogPublishEndpoints": {
+            "test1": {
+                "url": self.jfrog_url,
+                "repository": self.repository,
+                "username": self.jfrog_username,
+                "password": self.jfrog_password
+            }
+        }}
+
+        self.configOverride["JFrogPublishEndpoints"]["test1"].update(**self.jfrogOverrides)
+
+        super(JFrogTest, self).prepare()
+
+    def shutdown(self):
+        if hasattr(self, "repository"):
+            auth = (self.jfrog_username, self.jfrog_password)
+            delete_url = f"{self.jfrog_url}/api/repositories/{self.repository}"
+            requests.delete(delete_url, auth=auth)
+
+        super(JFrogTest, self).shutdown()
+
+
+    def check_path(self, path):
+        if path.startswith("public/"):
+            path = path[7:]
+        
+        # Check against JFrog Artifactory API
+        auth = (self.jfrog_username, self.jfrog_password)
+        check_url = f"{self.jfrog_url}/api/storage/{self.repository}/{path}"
+        res = requests.head(check_url, auth=auth)
+        if res.status_code == 200:
+            return True
+        return False
+
+    def check_exists(self, path):
+        if not self.check_path(path):
+            raise Exception("path %s doesn't exist" % (path, ))
+
+    def check_not_exists(self, path):
+        if self.check_path(path):
+            raise Exception("path %s exists" % (path, ))
+
+    def read_file(self, path, mode=''):
+        assert not mode
+        if path.startswith("public/"):
+            path = path[7:]
+        
+        auth = (self.jfrog_username, self.jfrog_password)
+        get_url = f"{self.jfrog_url}/{self.repository}/{path}"
+        res = requests.get(get_url, auth=auth)
+        res.raise_for_status()
+        return res.text
+

system/t06_publish/jfrog.py

@@ -0,0 +1,116 @@
+from jfrog_lib import JFrogTest
+
+
+def strip_processor(output):
+    return '\n'.join(
+        [
+            l
+            for l in output.split('\n')
+            if not l.startswith(' ') and not l.startswith('Date:')
+        ]
+    )
+
+
+class JFrogPublish1Test(JFrogTest):
+    """
+    publish to JFrog: from repo
+    """
+
+    fixtureCmds = [
+        'aptly repo create -distribution=maverick local-repo',
+        'aptly repo add local-repo ${files}',
+        'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
+    ]
+    runCmd = 'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo jfrog:test1:'
+
+    def check(self):
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists(
+            'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
+        )
+
+        # verify contents except sums/date chunks
+        self.check_file_contents(
+            'public/dists/maverick/Release', 'release', match_prepare=strip_processor
+        )
+        self.check_file_contents(
+            'public/dists/maverick/main/source/Sources',
+            'sources',
+            match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
+        )
+        self.check_file_contents(
+            'public/dists/maverick/main/binary-i386/Packages',
+            'binary',
+            match_prepare=lambda s: '\n'.join(sorted(s.split('\n'))),
+        )
+
+
+class JFrogPublish2Test(JFrogTest):
+    """
+    publish to JFrog: update after removing package from repo
+    """
+
+    fixtureCmds = [
+        'aptly repo create -distribution=maverick local-repo',
+        'aptly repo add local-repo ${files}/',
+        'aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386',
+        'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo jfrog:test1:',
+        'aptly repo remove local-repo pyspi',
+    ]
+    runCmd = 'aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick jfrog:test1:'
+
+    def check(self):
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists(
+            'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
+        )
+
+
+class JFrogPublish3Test(JFrogTest):
+    """
+    publish to JFrog: publish drop performs cleanup
+    """
+
+    fixtureCmds = [
+        'aptly repo create local1',
+        'aptly repo create local2',
+        'aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb',
+        'aptly repo add local2 ${files}',
+        'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 jfrog:test1:',
+        'aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 jfrog:test1:',
+    ]
+    runCmd = 'aptly publish drop sq2 jfrog:test1:'
+
+    def check(self):
+        self.check_exists('public/dists/sq1')
+        self.check_not_exists('public/dists/sq2')
+        self.check_exists('public/pool/main/')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists(
+            'public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb'
+        )