initial commit for JFrog support

This commit is contained in:
Pierig Le Saux
2026-04-07 11:07:27 -04:00
committed by André Roth
parent 8ebe80f066
commit 4b8f0c42ac
9 changed files with 415 additions and 226 deletions
+3
View File
@@ -51,6 +51,9 @@ func createTestConfig() *os.File {
"GcsPublishEndpoints": map[string]map[string]string{
"test-gcs": {
"bucket": "bucket-gcs",
"JFrogPublishEndpoints": gin.H{
"test-jfrog": gin.H{
"url": "http://jfrog.example.com",
},
},
})
+21
View File
@@ -0,0 +1,21 @@
package api
import (
"github.com/gin-gonic/gin"
)
// @Summary JFrog repositories
// @Description **Get list of JFrog repositories**
// @Description
// @Description List configured JFrog publish endpoints.
// @Tags Status
// @Produce json
// @Success 200 {array} string "List of JFrog publish endpoints"
// @Router /api/jfrog [get]
func apiJFrogList(c *gin.Context) {
keys := []string{}
for k := range context.Config().JFrogPublishRoots {
keys = append(keys, k)
}
c.JSON(200, keys)
}
+1
View File
@@ -178,6 +178,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
{
api.GET("/s3", apiS3List)
api.GET("/gcs", apiGCSList)
api.GET("/jfrog", apiJFrogList)
}
{
+13
View File
@@ -25,6 +25,7 @@ import (
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/gcs"
"github.com/aptly-dev/aptly/http"
"github.com/aptly-dev/aptly/jfrog"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/s3"
"github.com/aptly-dev/aptly/swift"
@@ -476,6 +477,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
if err != nil {
Fatal(err)
}
} else if strings.HasPrefix(name, "jfrog:") {
params, ok := context.config().JFrogPublishRoots[name[6:]]
if !ok {
Fatal(fmt.Errorf("published JFrog storage %v not configured", name[6:]))
}
var err error
publishedStorage, err = jfrog.NewPublishedStorage(
name[6:], params)
if err != nil {
Fatal(err)
}
} else {
Fatal(fmt.Errorf("unknown published storage format: %v", name))
}
+2
View File
@@ -0,0 +1,2 @@
// Package jfrog handles publishing to JFrog Artifactory
package jfrog
+238
View File
@@ -0,0 +1,238 @@
package jfrog
import (
"fmt"
"path/filepath"
"strings"
"github.com/aptly-dev/aptly/aptly"
aptly_utils "github.com/aptly-dev/aptly/utils"
"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/artifactory/auth"
"github.com/jfrog/jfrog-client-go/artifactory/services"
"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
"github.com/jfrog/jfrog-client-go/config"
"github.com/pkg/errors"
)
// PublishedStorage represents published repository on JFrog Artifactory
type PublishedStorage struct {
manager artifactory.ArtifactoryServicesManager
repository string
prefix string
plusWorkaround bool
pathCache map[string]string
}
// Check interface
var (
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)
// NewPublishedStorageRaw creates jfrog PublishedStorage from raw connection specs
func NewPublishedStorageRaw(
repository, url, user, password, apiKey, accessToken, prefix string,
plusWorkaround, debug bool,
) (*PublishedStorage, error) {
artDetails := auth.NewArtifactoryDetails()
artDetails.SetUrl(url)
if user != "" && password != "" {
artDetails.SetUser(user)
artDetails.SetPassword(password)
} else if apiKey != "" {
artDetails.SetApiKey(apiKey)
} else if accessToken != "" {
artDetails.SetAccessToken(accessToken)
}
serviceConfig, err := config.NewConfigBuilder().
SetServiceDetails(artDetails).
SetDryRun(false).
Build()
if err != nil {
return nil, errors.Wrap(err, "error building jfrog client config")
}
manager, err := artifactory.New(serviceConfig)
if err != nil {
return nil, errors.Wrap(err, "error creating jfrog manager")
}
return &PublishedStorage{
manager: manager,
repository: repository,
prefix: prefix,
plusWorkaround: plusWorkaround,
}, nil
}
// NewPublishedStorage creates published storage from aptly configuration struct
func NewPublishedStorage(
account string, root aptly_utils.JFrogPublishRoot,
) (*PublishedStorage, error) {
return NewPublishedStorageRaw(
root.Repository, root.Url, root.User, root.Password, root.ApiKey, root.AccessToken,
root.Prefix, root.PlusWorkaround, root.Debug)
}
func (storage *PublishedStorage) String() string {
return fmt.Sprintf("jfrog:%s:%s", storage.repository, storage.prefix)
}
func (storage *PublishedStorage) MkDir(path string) error {
return nil
}
func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
targetPath := filepath.Join(storage.repository, storage.prefix, path)
if storage.plusWorkaround {
targetPath = strings.Replace(targetPath, "+", "%2B", -1)
}
params := services.NewUploadParams()
params.Pattern = sourceFilename
params.Target = targetPath
params.Flat = true
_, _, err := storage.manager.UploadFiles(artifactory.UploadServiceOptions{}, params)
return err
}
func (storage *PublishedStorage) Remove(path string) error {
targetPath := filepath.Join(storage.repository, storage.prefix, path)
if storage.plusWorkaround {
targetPath = strings.Replace(targetPath, "+", "%2B", -1)
}
deleteParams := services.NewDeleteParams()
deleteParams.SetPattern(targetPath)
res, err := storage.manager.GetPathsToDelete(deleteParams)
if err != nil {
return err
}
defer res.Close()
_, err = storage.manager.DeleteFiles(res)
return err
}
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
return storage.Remove(path)
}
func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceMD5 aptly_utils.ChecksumInfo, force bool) error {
return storage.PutFile(filepath.Join(publishedPrefix, publishedRelPath, fileName), sourcePath)
}
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
searchPattern := filepath.Join(storage.repository, storage.prefix, prefix, "*")
params := services.NewSearchParams()
params.Pattern = searchPattern
reader, err := storage.manager.SearchFiles(params)
if err != nil {
return nil, err
}
defer reader.Close()
var paths []string
var md5s []string
for element := new(utils.ResultItem); reader.NextRecord(element) == nil; element = new(utils.ResultItem) {
path := element.Path + "/" + element.Name
relPath := strings.TrimPrefix(path, storage.repository + "/" + storage.prefix + "/")
if storage.plusWorkaround {
relPath = strings.Replace(relPath, "%2B", "+", -1)
}
paths = append(paths, relPath)
md5s = append(md5s, element.Actual_Md5)
}
return paths, nil
}
func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
oldTarget := filepath.Join(storage.repository, storage.prefix, oldName)
newTarget := filepath.Join(storage.repository, storage.prefix, newName)
if storage.plusWorkaround {
oldTarget = strings.Replace(oldTarget, "+", "%2B", -1)
newTarget = strings.Replace(newTarget, "+", "%2B", -1)
}
params := services.NewMoveCopyParams()
params.Pattern = oldTarget
params.Target = newTarget
params.Flat = true
_, _, err := storage.manager.Move(params)
return err
}
func (storage *PublishedStorage) SymLink(src string, dst string) error {
oldTarget := filepath.Join(storage.repository, storage.prefix, src)
newTarget := filepath.Join(storage.repository, storage.prefix, dst)
if storage.plusWorkaround {
oldTarget = strings.Replace(oldTarget, "+", "%2B", -1)
newTarget = strings.Replace(newTarget, "+", "%2B", -1)
}
params := services.NewMoveCopyParams()
params.Pattern = oldTarget
params.Target = newTarget
params.Flat = true
props := utils.NewProperties()
props.AddProperty("SymLink", src)
params.SetTargetProps(props)
_, _, err := storage.manager.Copy(params)
return err
}
func (storage *PublishedStorage) HardLink(src string, dst string) error {
return storage.SymLink(src, dst)
}
func (storage *PublishedStorage) FileExists(path string) (bool, error) {
targetPath := filepath.Join(storage.repository, storage.prefix, path)
if storage.plusWorkaround {
targetPath = strings.Replace(targetPath, "+", "%2B", -1)
}
params := services.NewSearchParams()
params.Pattern = targetPath
reader, err := storage.manager.SearchFiles(params)
if err != nil {
return false, err
}
defer reader.Close()
length, err := reader.Length()
isEmpty := length == 0
return !isEmpty, err
}
func (storage *PublishedStorage) ReadLink(path string) (string, error) {
targetPath := filepath.Join(storage.repository, storage.prefix, path)
if storage.plusWorkaround {
targetPath = strings.Replace(targetPath, "+", "%2B", -1)
}
props, err := storage.manager.GetItemProps(targetPath)
if err != nil {
return "", nil
}
for k, v := range props.Properties {
if k == "SymLink" && len(v) > 0 {
return v[0], nil
}
}
return "", nil
}
@@ -32,6 +32,7 @@ gpg_keys: []
skip_contents_publishing: false
skip_bz2_publishing: false
filesystem_publish_endpoints: {}
jfrog_publish_endpoints: {}
s3_publish_endpoints: {}
gcs_publish_endpoints: {}
swift_publish_endpoints: {}
+14
View File
@@ -61,6 +61,7 @@ type ConfigStructure struct { // nolint: maligned
// Storage
FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints" yaml:"filesystem_publish_endpoints"`
JFrogPublishRoots map[string]JFrogPublishRoot `json:"JFrogPublishEndpoints" yaml:"jfrog_publish_endpoints"`
S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints" yaml:"s3_publish_endpoints"`
GCSPublishRoots map[string]GCSPublishRoot `json:"GcsPublishEndpoints" yaml:"gcs_publish_endpoints"`
SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints" yaml:"swift_publish_endpoints"`
@@ -172,6 +173,19 @@ type FileSystemPublishRoot struct {
}
// S3PublishRoot describes single S3 publishing entry point
type JFrogPublishRoot struct {
Repository string `json:"repository" yaml:"repository"`
Url string `json:"url" yaml:"url"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
ApiKey string `json:"apiKey" yaml:"api_key"`
AccessToken string `json:"accessToken" yaml:"access_token"`
Prefix string `json:"prefix" yaml:"prefix"`
PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"`
Debug bool `json:"debug" yaml:"debug"`
}
type S3PublishRoot struct {
Region string `json:"region" yaml:"region"`
Bucket string `json:"bucket" yaml:"bucket"`
+122 -226
View File
@@ -45,6 +45,10 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
s.config.FileSystemPublishRoots = map[string]FileSystemPublishRoot{"test": {
RootDir: "/opt/aptly-publish"}}
s.config.JFrogPublishRoots = map[string]JFrogPublishRoot{"test": {
Repository: "repo",
Url: "jfrog.example.com"}}
s.config.S3PublishRoots = map[string]S3PublishRoot{"test": {
Region: "us-east-1",
Bucket: "repo"}}
@@ -73,232 +77,124 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, ""+
"{\n" +
" \"rootDir\": \"/tmp/aptly\",\n" +
" \"logLevel\": \"info\",\n" +
" \"logFormat\": \"json\",\n" +
" \"databaseOpenAttempts\": 5,\n" +
" \"architectures\": null,\n" +
" \"skipLegacyPool\": false,\n" +
" \"dependencyFollowSuggests\": false,\n" +
" \"dependencyFollowRecommends\": false,\n" +
" \"dependencyFollowAllVariants\": false,\n" +
" \"dependencyFollowSource\": false,\n" +
" \"dependencyVerboseResolve\": false,\n" +
" \"ppaDistributorID\": \"\",\n" +
" \"ppaCodename\": \"\",\n" +
" \"ppaBaseURL\": \"\",\n" +
" \"serveInAPIMode\": false,\n" +
" \"enableMetricsEndpoint\": false,\n" +
" \"enableSwaggerEndpoint\": false,\n" +
" \"AsyncAPI\": false,\n" +
" \"databaseBackend\": {\n" +
" \"type\": \"\",\n" +
" \"dbPath\": \"\",\n" +
" \"url\": \"\"\n" +
" },\n" +
" \"downloader\": \"\",\n" +
" \"downloadConcurrency\": 5,\n" +
" \"downloadSpeedLimit\": 0,\n" +
" \"downloadRetries\": 0,\n" +
" \"downloadSourcePackages\": false,\n" +
" \"gpgProvider\": \"gpg\",\n" +
" \"gpgDisableSign\": false,\n" +
" \"gpgDisableVerify\": false,\n" +
" \"gpgKeys\": null,\n" +
" \"skipContentsPublishing\": false,\n" +
" \"skipBz2Publishing\": false,\n" +
" \"FileSystemPublishEndpoints\": {\n" +
" \"test\": {\n" +
" \"rootDir\": \"/opt/aptly-publish\",\n" +
" \"linkMethod\": \"\",\n" +
" \"verifyMethod\": \"\"\n" +
" }\n" +
" },\n" +
" \"S3PublishEndpoints\": {\n" +
" \"test\": {\n" +
" \"region\": \"us-east-1\",\n" +
" \"bucket\": \"repo\",\n" +
" \"prefix\": \"\",\n" +
" \"acl\": \"\",\n" +
" \"awsAccessKeyID\": \"\",\n" +
" \"awsSecretAccessKey\": \"\",\n" +
" \"awsSessionToken\": \"\",\n" +
" \"endpoint\": \"\",\n" +
" \"storageClass\": \"\",\n" +
" \"encryptionMethod\": \"\",\n" +
" \"plusWorkaround\": false,\n" +
" \"disableMultiDel\": false,\n" +
" \"forceSigV2\": false,\n" +
" \"forceVirtualHostedStyle\": false,\n" +
" \"debug\": false\n" +
" }\n" +
" },\n" +
" \"GcsPublishEndpoints\": {\n" +
" \"test\": {\n" +
" \"bucket\": \"repo\",\n" +
" \"prefix\": \"\",\n" +
" \"credentialsFile\": \"\",\n" +
" \"serviceAccountJSON\": \"\",\n" +
" \"project\": \"\",\n" +
" \"endpoint\": \"\",\n" +
" \"acl\": \"\",\n" +
" \"storageClass\": \"\",\n" +
" \"encryptionKey\": \"\",\n" +
" \"disableMultiDel\": false,\n" +
" \"debug\": false\n" +
" }\n" +
" },\n" +
" \"SwiftPublishEndpoints\": {\n" +
" \"test\": {\n" +
" \"container\": \"repo\",\n" +
" \"prefix\": \"\",\n" +
" \"osname\": \"\",\n" +
" \"password\": \"\",\n" +
" \"tenant\": \"\",\n" +
" \"tenantid\": \"\",\n" +
" \"domain\": \"\",\n" +
" \"domainid\": \"\",\n" +
" \"tenantdomain\": \"\",\n" +
" \"tenantdomainid\": \"\",\n" +
" \"authurl\": \"\"\n" +
" }\n" +
" },\n" +
" \"AzurePublishEndpoints\": {\n" +
" \"test\": {\n" +
" \"container\": \"repo\",\n" +
" \"prefix\": \"\",\n" +
" \"accountName\": \"\",\n" +
" \"accountKey\": \"\",\n" +
" \"endpoint\": \"\"\n" +
" }\n" +
" },\n" +
" \"packagePoolStorage\": {\n" +
" \"type\": \"local\",\n" +
" \"path\": \"/tmp/aptly-pool\"\n" +
" }\n" +
"}")
}
func (s *ConfigSuite) TestLoadYAMLConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml1")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAML)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err, IsNil)
c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/")
c.Check(s.config.DownloadConcurrency, Equals, 40)
c.Check(s.config.DatabaseOpenAttempts, Equals, 10)
}
func (s *ConfigSuite) TestLoadYAMLErrorConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml2")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAMLError)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err.Error(), Equals, "invalid yaml (unknown pool storage type: invalid) or json (invalid character 'p' looking for beginning of value)")
}
func (s *ConfigSuite) TestSaveYAMLConfig(c *C) {
configname := filepath.Join(c.MkDir(), "aptly.yaml3")
f, _ := os.Create(configname)
_, _ = f.WriteString(configFileYAML)
_ = f.Close()
// start with empty config
s.config = ConfigStructure{}
err := LoadConfig(configname, &s.config)
c.Assert(err, IsNil)
err = SaveConfigYAML(configname, &s.config)
c.Assert(err, IsNil)
f, _ = os.Open(configname)
defer func() {
_ = f.Close()
}()
st, _ := f.Stat()
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, configFileYAML)
}
func (s *ConfigSuite) TestSaveYAML2Config(c *C) {
// start with empty config
s.config = ConfigStructure{}
s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"}
s.config.PackagePoolStorage.Azure = nil
configname := filepath.Join(c.MkDir(), "aptly.yaml4")
err := SaveConfigYAML(configname, &s.config)
c.Assert(err, IsNil)
f, _ := os.Open(configname)
defer func() {
_ = f.Close()
}()
st, _ := f.Stat()
buf := make([]byte, st.Size())
_, _ = f.Read(buf)
c.Check(string(buf), Equals, "" +
"root_dir: \"\"\n" +
"log_level: \"\"\n" +
"log_format: \"\"\n" +
"database_open_attempts: 0\n" +
"architectures: []\n" +
"skip_legacy_pool: false\n" +
"dep_follow_suggests: false\n" +
"dep_follow_recommends: false\n" +
"dep_follow_all_variants: false\n" +
"dep_follow_source: false\n" +
"dep_verboseresolve: false\n" +
"ppa_distributor_id: \"\"\n" +
"ppa_codename: \"\"\n" +
"ppa_baseurl: \"\"\n" +
"serve_in_api_mode: false\n" +
"enable_metrics_endpoint: false\n" +
"enable_swagger_endpoint: false\n" +
"async_api: false\n" +
"database_backend:\n" +
" type: \"\"\n" +
" db_path: \"\"\n" +
" url: \"\"\n" +
"downloader: \"\"\n" +
"download_concurrency: 0\n" +
"download_limit: 0\n" +
"download_retries: 0\n" +
"download_sourcepackages: false\n" +
"gpg_provider: \"\"\n" +
"gpg_disable_sign: false\n" +
"gpg_disable_verify: false\n" +
"gpg_keys: []\n" +
"skip_contents_publishing: false\n" +
"skip_bz2_publishing: false\n" +
"filesystem_publish_endpoints: {}\n" +
"s3_publish_endpoints: {}\n" +
"gcs_publish_endpoints: {}\n" +
"swift_publish_endpoints: {}\n" +
"azure_publish_endpoints: {}\n" +
"packagepool_storage:\n" +
" type: local\n" +
" path: /tmp/aptly-pool\n")
c.Check(string(buf), Equals, `{
"rootDir": "/tmp/aptly",
"logLevel": "info",
"logFormat": "json",
"databaseOpenAttempts": 5,
"architectures": null,
"skipLegacyPool": false,
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"ppaDistributorID": "",
"ppaCodename": "",
"ppaBaseURL": "",
"serveInAPIMode": false,
"enableMetricsEndpoint": false,
"enableSwaggerEndpoint": false,
"AsyncAPI": false,
"databaseBackend": {
"type": "",
"dbPath": "",
"url": ""
},
"downloader": "",
"downloadConcurrency": 5,
"downloadSpeedLimit": 0,
"downloadRetries": 0,
"downloadSourcePackages": false,
"gpgProvider": "gpg",
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgKeys": null,
"skipContentsPublishing": false,
"skipBz2Publishing": false,
"FileSystemPublishEndpoints": {
"test": {
"rootDir": "/opt/aptly-publish",
"linkMethod": "",
"verifyMethod": ""
}
},
"JFrogPublishEndpoints": {
"test": {
"repository": "repo",
"url": "jfrog.example.com",
"user": "",
"password": "",
"apiKey": "",
"accessToken": "",
"prefix": "",
"plusWorkaround": false,
"debug": false
}
},
"S3PublishEndpoints": {
"test": {
"region": "us-east-1",
"bucket": "repo",
"prefix": "",
"acl": "",
"awsAccessKeyID": "",
"awsSecretAccessKey": "",
"awsSessionToken": "",
"endpoint": "",
"storageClass": "",
"encryptionMethod": "",
"plusWorkaround": false,
"disableMultiDel": false,
"forceSigV2": false,
"forceVirtualHostedStyle": false,
"debug": false
}
},
"GcsPublishEndpoints": {
"test": {
"bucket": "repo",
"prefix": "",
"credentialsFile": "",
"serviceAccountJSON": "",
"project": "",
"endpoint": "",
"acl": "",
"storageClass": "",
"encryptionKey": "",
"disableMultiDel": false,
"debug": false
}
},
"SwiftPublishEndpoints": {
"test": {
"container": "repo",
"prefix": "",
"osname": "",
"password": "",
"tenant": "",
"tenantid": "",
"domain": "",
"domainid": "",
"tenantdomain": "",
"tenantdomainid": "",
"authurl": ""
}
},
"AzurePublishEndpoints": {
"test": {
"container": "repo",
"prefix": "",
"accountName": "",
"accountKey": "",
"endpoint": ""
}
},
"packagePoolStorage": {
"type": "local",
"path": "/tmp/aptly-pool"
}
}`)
}
func (s *ConfigSuite) TestLoadEmptyConfig(c *C) {