vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-11 03:11:50 +00:00
Code Issues Projects Releases Wiki Activity

add endpoint for listing repos while serving in api mode and add more metrics

Browse Source
This commit is contained in:
Markus Muellner
2023-03-20 17:14:45 +01:00
committed by Benj Fassbind
parent 0fdba29d51
commit 9c6f896666
7 changed files with 189 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/files.go
View File

@@ -109,6 +109,7 @@ func apiFilesUpload(c *gin.Context) {
}
}
apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
c.JSON(200, stored)
}

43
api/metrics.go
View File

@@ -1,12 +1,15 @@
package api
import (
"fmt"
"runtime"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/rs/zerolog/log"
)
var (
@@ -52,6 +55,20 @@ var (
},
[]string{"version", "goversion"},
)
apiFilesUploadedCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_files_uploaded_total",
Help: "Total number of uploaded files labeled by upload directory.",
},
[]string{"directory"},
)
apiReposPackageCountGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_repos_package_count",
Help: "Current number of published packages labeled by source, distribution and component.",
},
[]string{"source", "distribution", "component"},
)
)
type metricsCollectorRegistrar struct {
@@ -71,3 +88,29 @@ func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
}
var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
func countPackagesByRepos() {
err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
if err != nil {
msg := fmt.Sprintf(
"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
err, repo.StoragePrefix(), repo.Distribution, repo.Components())
log.Warn().Msg(msg)
return err
}
components := repo.Components()
for _, c := range components {
count := float64(len(repo.RefList(c).Refs))
apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
}
return nil
})
if err != nil {
msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
log.Warn().Msg(msg)
}
}

32
api/repos.go
View File

@@ -17,6 +17,38 @@ import (
"github.com/gin-gonic/gin"
)
// GET /repos
func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
c.Writer.Flush()
c.Writer.WriteString("<pre>\n")
if len(localRepos) == 0 {
c.Writer.WriteString("<a href=\"-/\">default</a>\n")
}
for publishPrefix := range localRepos {
c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
}
c.Writer.WriteString("</pre>")
c.Writer.Flush()
}
}
// GET /repos/:storage/*pkgPath
func reposServeInAPIMode(c *gin.Context) {
pkgpath := c.Param("pkgPath")
storage := c.Param("storage")
if storage == "-" {
storage = ""
} else {
storage = "filesystem:" + storage
}
publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
c.FileFromFS(pkgpath, http.Dir(publicPath))
}
// GET /api/repos
func apiReposList(c *gin.Context) {
result := []*deb.LocalRepo{}

14
api/router.go
View File

@@ -17,6 +17,7 @@ var context *ctx.AptlyContext
func apiMetricsGet() gin.HandlerFunc {
return func(c *gin.Context) {
countPackagesByRepos()
promhttp.Handler().ServeHTTP(c.Writer, c.Request)
}
}
@@ -52,17 +53,8 @@ func Router(c *ctx.AptlyContext) http.Handler {
}
if c.Config().ServeInAPIMode {
router.GET("/repos/:storage/*pkgPath", func(c *gin.Context) {
pkgpath := c.Param("pkgPath")
storage := c.Param("storage")
if storage == "-" {
storage = ""
}
publicPath := context.GetPublishedStorage("filesystem:" + storage).(aptly.FileSystemPublishedStorage).PublicPath()
c.FileFromFS(pkgpath, http.Dir(publicPath))
})
router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
}
api := router.Group("/api")

23
deb/publish.go
View File

@@ -423,6 +423,29 @@ func (p *PublishedRepo) Components() []string {
return result
}
// Components returns sorted list of published repo source names
func (p *PublishedRepo) SourceNames() []string {
var sources = []string{}
for _, component := range p.Components() {
var source string
item := p.sourceItems[component]
if item.snapshot != nil {
source = item.snapshot.Name
} else if item.localRepo != nil {
source = item.localRepo.Name
} else {
panic("no snapshot/localRepo")
}
sources = append(sources, fmt.Sprintf("%s:%s", source, component))
}
sort.Strings(sources)
return sources
}
// UpdateLocalRepo updates content from local repo in component
func (p *PublishedRepo) UpdateLocalRepo(component string) {
if p.SourceKind != SourceLocalRepo {

41
system/t12_api/metrics.py
View File

@@ -7,6 +7,35 @@ class MetricsEnabledAPITest(APITest):
"""
def check(self):
d = "libboost-program-options-dev_1.62.0.1"
r = "foo"
f = "libboost-program-options-dev_1.62.0.1_i386.deb"
self.check_equal(self.upload("/api/files/" + d, f).status_code, 200)
self.check_equal(self.post("/api/repos", json={
"Name": r,
"Comment": "test repo",
"DefaultDistribution": r,
"DefaultComponent": "main"
}).status_code, 201)
self.check_equal(self.post(f"/api/repos/{r}/file/{d}").status_code, 200)
self.check_equal(self.post("/api/publish/filesystem:apiandserve:", json={
"SourceKind": "local",
"Sources": [
{
"Component": "main",
"Name": r
}
],
"Distribution": r,
"Signing": {
"Skip": True
}
}).status_code, 201)
resp = self.get("/api/metrics")
self.check_equal(resp.status_code, 200)
@@ -27,3 +56,15 @@ class MetricsEnabledAPITest(APITest):
apiBuildInfoGauge = "# TYPE aptly_build_info gauge"
self.check_in(apiBuildInfoGauge, resp.text)
apiFilesUploadedCounter = "# TYPE aptly_api_files_uploaded_total counter"
self.check_in(apiFilesUploadedCounter, resp.text)
apiFilesUploadedCounterValue = "aptly_api_files_uploaded_total{directory=\"libboost-program-options-dev_1.62.0.1\"} 1"
self.check_in(apiFilesUploadedCounterValue, resp.text)
apiReposPackageCountGauge = "# TYPE aptly_repos_package_count gauge"
self.check_in(apiReposPackageCountGauge, resp.text)
apiReposPackageCountGaugeValue = "aptly_repos_package_count{component=\"main\",distribution=\"foo\",source=\"[foo:main]\"} 1"
self.check_in(apiReposPackageCountGaugeValue, resp.text)

47
system/t12_api/publish.py
View File

@@ -619,6 +619,51 @@ class PublishSwitchAPISkipCleanupTestRepo(APITest):
self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
class ServePublishedListTestRepo(APITest):
"""
GET /repos
"""
def check(self):
d = "libboost-program-options-dev_1.62.0.1"
r = "bar"
f = "libboost-program-options-dev_1.62.0.1_i386.deb"
self.check_equal(self.upload("/api/files/" + d, f).status_code, 200)
self.check_equal(self.post("/api/repos", json={
"Name": r,
"Comment": "test repo",
"DefaultDistribution": r,
"DefaultComponent": "main"
}).status_code, 201)
self.check_equal(self.post(f"/api/repos/{r}/file/{d}").status_code, 200)
self.check_equal(self.post("/api/publish/filesystem:apiandserve:", json={
"SourceKind": "local",
"Sources": [
{
"Component": "main",
"Name": r
}
],
"Distribution": r,
"Signing": {
"Skip": True
}
}).status_code, 201)
get = self.get("/repos")
expected_content_type = "text/html; charset=utf-8"
if get.headers['content-type'] != expected_content_type:
raise Exception(f"Received content-type {get.headers['content-type']} was not: {expected_content_type}")
excepted_content = b'<pre>\n<a href="apiandserve/">apiandserve</a>\n</pre>'
if excepted_content != get.content:
raise Exception(f"Expected content {excepted_content} was not: {get.content}")
class ServePublishedTestRepo(APITest):
"""
GET /repos/:storage/*pkgPath
@@ -664,7 +709,7 @@ class ServePublishedTestRepo(APITest):
raise Exception(f"Received content-type {get.headers['content-type']} not one of expected: {deb_content_types}")
if len(get.content) != 3428:
raise Exception(f"Expected file size 3428 bytes != {get.status_code} bytes")
raise Exception(f"Expected file size 3428 bytes != {len(get.content)} bytes")
class ServePublishedNotFoundTestRepo(APITest):