add ready and healthy probe endpoints

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
 TAG="$(shell git describe --tags --always)"
-VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g' | tr -d '\n')
 PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python3
 TESTS?=

api/api.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
@@ -27,6 +28,23 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+// GET /api/ready
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+	}
+}
+
+// GET /api/healthy
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
+}
+
 type dbRequestKind int
 
 const (

api/api_test.go

@@ -6,8 +6,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
+	"github.com/aptly-dev/aptly/aptly"
 	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/gin-gonic/gin"
 
@@ -36,6 +38,7 @@ func createTestConfig() *os.File {
 	}
 	jsonString, err := json.Marshal(gin.H{
 		"architectures": []string{},
+		"enableMetricsEndpoint": true,
 	})
 	if err != nil {
 		return nil
@@ -45,6 +48,7 @@ func createTestConfig() *os.File {
 }
 
 func (s *ApiSuite) SetUpSuite(c *C) {
+	aptly.Version = "testVersion"
 	file := createTestConfig()
 	c.Assert(file, NotNil)
 	s.configFile = file
@@ -89,7 +93,35 @@ func (s *ApiSuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, ".*Version.*")
+	c.Check(response.Body.String(), Matches, "{\"Version\":\"" + aptly.Version + "\"}")
+}
+
+func (s *ApiSuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *ApiSuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *ApiSuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
 }
 
 func (s *ApiSuite) TestTruthy(c *C) {

api/metrics.go

@@ -0,0 +1,73 @@
+package api
+
+import (
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}

api/middleware.go

@@ -9,45 +9,6 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	apiRequestsInFlightGauge = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "aptly_api_http_requests_in_flight",
-			Help: "Number of concurrent HTTP api requests currently handled.",
-		},
-		[]string{"method", "path"},
-	)
-	apiRequestsTotalCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "aptly_api_http_requests_total",
-			Help: "Total number of api requests.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_size_bytes",
-			Help: "Api HTTP request size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiResponseSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_response_size_bytes",
-			Help: "Api HTTP response size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestsDurationSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_duration_seconds",
-			Help: "Duration of api requests in seconds.",
-		},
-		[]string{"code", "method", "path"},
-	)
 )
 
 // Only use base path as label value (e.g.: /api/repos) because of time series cardinality

api/router.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"net/http"
+	"sync/atomic"
 
 	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/gin-gonic/gin"
@@ -25,11 +26,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	router.Use(gin.ErrorLogger())
 
 	if c.Config().EnableMetricsEndpoint {
-		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		MetricsCollectorRegistrar.Register(router)
 	}
 
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
@@ -71,6 +68,12 @@ func Router(c *ctx.AptlyContext) http.Handler {
 			root.GET("/metrics", apiMetricsGet())
 		}
 		root.GET("/version", apiVersion)
+
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		root.GET("/ready", apiReady(isReady))
+		root.GET("/healthy", apiHealthy)
 	}
 
 	{

system/t12_api/metrics.py

@@ -24,3 +24,6 @@ class MetricsEnabledAPITest(APITest):
 
         apiRequestsDurationSummary = "# TYPE aptly_api_http_request_duration_seconds summary"
         self.check_in(apiRequestsDurationSummary, resp.text)
+
+        apiBuildInfoGauge = "# TYPE aptly_build_info gauge"
+        self.check_in(apiBuildInfoGauge, resp.text)

system/t12_api/probes.py

@@ -0,0 +1,27 @@
+from api_lib import APITest
+
+
+class ReadyAPITest(APITest):
+    """
+    GET /ready
+    """
+
+    def check(self):
+        resp = self.get("/api/ready")
+        self.check_equal(resp.status_code, 200)
+
+        readyStatus = "{\"Status\":\"Aptly is ready\"}"
+        self.check_equal(readyStatus, resp.text)
+
+
+class HealthyAPITest(APITest):
+    """
+    GET /healthy
+    """
+
+    def check(self):
+        resp = self.get("/api/healthy")
+        self.check_equal(resp.status_code, 200)
+
+        healthyStatus = "{\"Status\":\"Aptly is healthy\"}"
+        self.check_equal(healthyStatus, resp.text)