From 38cb6bd13319c4027f47d894f3fa0d6e69713933 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Mon, 12 Jan 2015 10:26:36 +0100 Subject: [PATCH 1/5] Graph REST API #116 --- api/graph.go | 56 +++++++++++++++++++++++ api/router.go | 4 ++ cmd/graph.go | 122 ++----------------------------------------------- graph/graph.go | 121 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 117 deletions(-) create mode 100644 api/graph.go create mode 100644 graph/graph.go diff --git a/api/graph.go b/api/graph.go new file mode 100644 index 00000000..15213fef --- /dev/null +++ b/api/graph.go @@ -0,0 +1,56 @@ +package api + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "github.com/gin-gonic/gin" + "github.com/smira/aptly/graph" +) + +// GET /api/graph +func apiGraph(c *gin.Context) { + var ( + err error + output []byte + ) + + graph, err := graph.BuildGraph(context) + if err != nil { + c.JSON(500, err) + return + } + + buf := bytes.NewBufferString(graph.String()) + + command := exec.Command("dot", "-Tpng") + command.Stderr = os.Stderr + + stdin, err := command.StdinPipe() + if err != nil { + c.Fail(500, err) + return + } + + _, err = io.Copy(stdin, buf) + if err != nil { + c.Fail(500, err) + return + } + + err = stdin.Close() + if err != nil { + c.Fail(500, err) + return + } + + output, err = command.Output() + if err != nil { + c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err)) + return + } + + c.Data(200, "image/png", output) +} diff --git a/api/router.go b/api/router.go index 3e5b5efd..4c43d7a2 100644 --- a/api/router.go +++ b/api/router.go @@ -47,5 +47,9 @@ func Router(c *ctx.AptlyContext) http.Handler { root.DELETE("/publish/:prefix/:distribution", apiPublishDrop) } + { + root.GET("/graph", apiGraph) + } + return router } diff --git a/cmd/graph.go b/cmd/graph.go index 28bbc969..91f2e1a0 100644 --- a/cmd/graph.go +++ b/cmd/graph.go @@ -2,15 +2,13 @@ package cmd import ( "bytes" - "code.google.com/p/gographviz" "fmt" - "github.com/smira/aptly/deb" "github.com/smira/commander" + "github.com/smira/aptly/graph" "io" "io/ioutil" "os" "os/exec" - "strings" ) func aptlyGraph(cmd *commander.Command, args []string) error { @@ -21,121 +19,11 @@ func aptlyGraph(cmd *commander.Command, args []string) error { return commander.ErrCommandError } - graph := gographviz.NewEscape() - graph.SetDir(true) - graph.SetName("aptly") - - existingNodes := map[string]bool{} - - fmt.Printf("Loading mirrors...\n") - - err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { - err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) - if err != nil { - return err - } - - graph.AddNode("aptly", repo.UUID, map[string]string{ - "shape": "Mrecord", - "style": "filled", - "fillcolor": "darkgoldenrod1", - "label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}", - repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "), - strings.Join(repo.Architectures, ", "), repo.NumPackages()), - }) - existingNodes[repo.UUID] = true - return nil - }) - - if err != nil { - return err - } - - fmt.Printf("Loading local repos...\n") - - err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { - err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) - if err != nil { - return err - } - - graph.AddNode("aptly", repo.UUID, map[string]string{ - "shape": "Mrecord", - "style": "filled", - "fillcolor": "mediumseagreen", - "label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}", - repo.Name, repo.Comment, repo.NumPackages()), - }) - existingNodes[repo.UUID] = true - return nil - }) - - if err != nil { - return err - } - - fmt.Printf("Loading snapshots...\n") - - context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { - existingNodes[snapshot.UUID] = true - return nil - }) - - err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { - err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) - if err != nil { - return err - } - - description := snapshot.Description - if snapshot.SourceKind == "repo" { - description = "Snapshot from repo" - } - - graph.AddNode("aptly", snapshot.UUID, map[string]string{ - "shape": "Mrecord", - "style": "filled", - "fillcolor": "cadetblue1", - "label": fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()), - }) - - if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" { - for _, uuid := range snapshot.SourceIDs { - _, exists := existingNodes[uuid] - if exists { - graph.AddEdge(uuid, snapshot.UUID, true, nil) - } - } - } - return nil - }) - - if err != nil { - return err - } - - fmt.Printf("Loading published repos...\n") - - context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error { - graph.AddNode("aptly", repo.UUID, map[string]string{ - "shape": "Mrecord", - "style": "filled", - "fillcolor": "darkolivegreen1", - "label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, - strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")), - }) - - for _, uuid := range repo.Sources { - _, exists := existingNodes[uuid] - if exists { - graph.AddEdge(uuid, repo.UUID, true, nil) - } - } - - return nil - }) - fmt.Printf("Generating graph...\n") + graph, err := graph.BuildGraph(context) + if err != nil { + return err + } buf := bytes.NewBufferString(graph.String()) diff --git a/graph/graph.go b/graph/graph.go new file mode 100644 index 00000000..7f9e9daf --- /dev/null +++ b/graph/graph.go @@ -0,0 +1,121 @@ +package graph + +import ( + "code.google.com/p/gographviz" + "fmt" + "strings" + "github.com/smira/aptly/context" + "github.com/smira/aptly/deb" +) + +func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { + var err error + + graph := gographviz.NewEscape() + graph.SetDir(true) + graph.SetName("aptly") + + existingNodes := map[string]bool{} + + err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { + err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) + if err != nil { + return err + } + + graph.AddNode("aptly", repo.UUID, map[string]string{ + "shape": "Mrecord", + "style": "filled", + "fillcolor": "darkgoldenrod1", + "label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}", + repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "), + strings.Join(repo.Architectures, ", "), repo.NumPackages()), + }) + existingNodes[repo.UUID] = true + return nil + }) + + if err != nil { + return nil, err + } + + err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { + err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) + if err != nil { + return err + } + + graph.AddNode("aptly", repo.UUID, map[string]string{ + "shape": "Mrecord", + "style": "filled", + "fillcolor": "mediumseagreen", + "label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}", + repo.Name, repo.Comment, repo.NumPackages()), + }) + existingNodes[repo.UUID] = true + return nil + }) + + if err != nil { + return nil, err + } + + context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { + existingNodes[snapshot.UUID] = true + return nil + }) + + err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { + err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) + if err != nil { + return err + } + + description := snapshot.Description + if snapshot.SourceKind == "repo" { + description = "Snapshot from repo" + } + + graph.AddNode("aptly", snapshot.UUID, map[string]string{ + "shape": "Mrecord", + "style": "filled", + "fillcolor": "cadetblue1", + "label": fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()), + }) + + if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" { + for _, uuid := range snapshot.SourceIDs { + _, exists := existingNodes[uuid] + if exists { + graph.AddEdge(uuid, snapshot.UUID, true, nil) + } + } + } + return nil + }) + + if err != nil { + return nil, err + } + + context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error { + graph.AddNode("aptly", repo.UUID, map[string]string{ + "shape": "Mrecord", + "style": "filled", + "fillcolor": "darkolivegreen1", + "label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, + strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")), + }) + + for _, uuid := range repo.Sources { + _, exists := existingNodes[uuid] + if exists { + graph.AddEdge(uuid, repo.UUID, true, nil) + } + } + + return nil + }) + + return graph, nil +} \ No newline at end of file From 427c42f4b85056733849dd1ea24774cb16d1e687 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 13 Jan 2015 19:10:39 +0300 Subject: [PATCH 2/5] Move graph into deb/ package, passing collection factory. #169 --- api/graph.go | 8 ++++---- cmd/graph.go | 4 ++-- {graph => deb}/graph.go | 24 +++++++++++------------- 3 files changed, 17 insertions(+), 19 deletions(-) rename {graph => deb}/graph.go (72%) diff --git a/api/graph.go b/api/graph.go index 15213fef..fcf39d29 100644 --- a/api/graph.go +++ b/api/graph.go @@ -3,21 +3,21 @@ package api import ( "bytes" "fmt" + "github.com/gin-gonic/gin" + "github.com/smira/aptly/deb" "io" "os" "os/exec" - "github.com/gin-gonic/gin" - "github.com/smira/aptly/graph" ) // GET /api/graph func apiGraph(c *gin.Context) { var ( - err error + err error output []byte ) - graph, err := graph.BuildGraph(context) + graph, err := deb.BuildGraph(context.CollectionFactory()) if err != nil { c.JSON(500, err) return diff --git a/cmd/graph.go b/cmd/graph.go index 91f2e1a0..552066da 100644 --- a/cmd/graph.go +++ b/cmd/graph.go @@ -3,8 +3,8 @@ package cmd import ( "bytes" "fmt" + "github.com/smira/aptly/deb" "github.com/smira/commander" - "github.com/smira/aptly/graph" "io" "io/ioutil" "os" @@ -20,7 +20,7 @@ func aptlyGraph(cmd *commander.Command, args []string) error { } fmt.Printf("Generating graph...\n") - graph, err := graph.BuildGraph(context) + graph, err := deb.BuildGraph(context.CollectionFactory()) if err != nil { return err } diff --git a/graph/graph.go b/deb/graph.go similarity index 72% rename from graph/graph.go rename to deb/graph.go index 7f9e9daf..108b6d95 100644 --- a/graph/graph.go +++ b/deb/graph.go @@ -1,14 +1,12 @@ -package graph +package deb import ( "code.google.com/p/gographviz" "fmt" "strings" - "github.com/smira/aptly/context" - "github.com/smira/aptly/deb" ) -func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { +func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) { var err error graph := gographviz.NewEscape() @@ -17,8 +15,8 @@ func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { existingNodes := map[string]bool{} - err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { - err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) + err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error { + err := collectionFactory.RemoteRepoCollection().LoadComplete(repo) if err != nil { return err } @@ -39,8 +37,8 @@ func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { return nil, err } - err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { - err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) + err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error { + err := collectionFactory.LocalRepoCollection().LoadComplete(repo) if err != nil { return err } @@ -60,13 +58,13 @@ func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { return nil, err } - context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { + collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error { existingNodes[snapshot.UUID] = true return nil }) - err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { - err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) + err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error { + err := collectionFactory.SnapshotCollection().LoadComplete(snapshot) if err != nil { return err } @@ -98,7 +96,7 @@ func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { return nil, err } - context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error { + collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error { graph.AddNode("aptly", repo.UUID, map[string]string{ "shape": "Mrecord", "style": "filled", @@ -118,4 +116,4 @@ func BuildGraph(context *context.AptlyContext) (gographviz.Interface, error) { }) return graph, nil -} \ No newline at end of file +} From 67ce828eeb2064d1546196c2ee97e2367656a469 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 13 Jan 2015 19:17:19 +0300 Subject: [PATCH 3/5] Lock collections before building graph. #169 --- api/graph.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/graph.go b/api/graph.go index fcf39d29..4db0a6c6 100644 --- a/api/graph.go +++ b/api/graph.go @@ -17,7 +17,18 @@ func apiGraph(c *gin.Context) { output []byte ) - graph, err := deb.BuildGraph(context.CollectionFactory()) + factory := context.CollectionFactory() + + factory.RemoteRepoCollection().RLock() + defer factory.RemoteRepoCollection().RUnlock() + factory.LocalRepoCollection().RLock() + defer factory.LocalRepoCollection().RUnlock() + factory.SnapshotCollection().RLock() + defer factory.LocalRepoCollection().RUnlock() + factory.PublishedRepoCollection().RLock() + defer factory.PublishedRepoCollection().RUnlock() + + graph, err := deb.BuildGraph(factory) if err != nil { c.JSON(500, err) return From 281664780988200892412b6ec1b077358530157e Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 13 Jan 2015 19:22:24 +0300 Subject: [PATCH 4/5] Allow to generate graph in formats supported by dot. #169 --- api/graph.go | 16 ++++++++++++---- api/router.go | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/graph.go b/api/graph.go index 4db0a6c6..66ed4444 100644 --- a/api/graph.go +++ b/api/graph.go @@ -6,17 +6,20 @@ import ( "github.com/gin-gonic/gin" "github.com/smira/aptly/deb" "io" + "mime" "os" "os/exec" ) -// GET /api/graph +// GET /api/graph.:ext func apiGraph(c *gin.Context) { var ( err error output []byte ) + ext := c.Params.ByName("ext") + factory := context.CollectionFactory() factory.RemoteRepoCollection().RLock() @@ -24,7 +27,7 @@ func apiGraph(c *gin.Context) { factory.LocalRepoCollection().RLock() defer factory.LocalRepoCollection().RUnlock() factory.SnapshotCollection().RLock() - defer factory.LocalRepoCollection().RUnlock() + defer factory.SnapshotCollection().RUnlock() factory.PublishedRepoCollection().RLock() defer factory.PublishedRepoCollection().RUnlock() @@ -36,7 +39,7 @@ func apiGraph(c *gin.Context) { buf := bytes.NewBufferString(graph.String()) - command := exec.Command("dot", "-Tpng") + command := exec.Command("dot", "-T"+ext) command.Stderr = os.Stderr stdin, err := command.StdinPipe() @@ -63,5 +66,10 @@ func apiGraph(c *gin.Context) { return } - c.Data(200, "image/png", output) + mimeType := mime.TypeByExtension("." + ext) + if mimeType == "" { + mimeType = "application/octet-stream" + } + + c.Data(200, mimeType, output) } diff --git a/api/router.go b/api/router.go index 6c4ce5c9..1b378b8b 100644 --- a/api/router.go +++ b/api/router.go @@ -53,7 +53,7 @@ func Router(c *ctx.AptlyContext) http.Handler { } { - root.GET("/graph", apiGraph) + root.GET("/graph.:ext", apiGraph) } return router From a0d7ae28bffe6b3b9ae9ebb850c705d1641888a2 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 13 Jan 2015 22:15:06 +0300 Subject: [PATCH 5/5] Simple tests for graph generation API. #169 --- system/t12_api/__init__.py | 3 ++- system/t12_api/graph.py | 17 +++++++++++++++++ system/t12_api/publish.py | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 system/t12_api/graph.py diff --git a/system/t12_api/__init__.py b/system/t12_api/__init__.py index b09d27a0..c8ba5645 100644 --- a/system/t12_api/__init__.py +++ b/system/t12_api/__init__.py @@ -5,4 +5,5 @@ Testing aptly REST API from .repos import * from .files import * from .publish import * -from .version import * \ No newline at end of file +from .version import * +from .graph import * diff --git a/system/t12_api/graph.py b/system/t12_api/graph.py new file mode 100644 index 00000000..2f2a9f10 --- /dev/null +++ b/system/t12_api/graph.py @@ -0,0 +1,17 @@ +from api_lib import APITest + + +class GraphAPITest(APITest): + """ + GET /graph.:ext + """ + + def check(self): + resp = self.get("/api/graph.png") + self.check_equal(resp.headers["Content-Type"], "image/png") + self.check_equal(resp.content[:4], '\x89PNG') + + self.check_equal(self.post("/api/repos", json={"Name": "xyz", "Comment": "fun repo"}).status_code, 201) + resp = self.get("/api/graph.svg") + self.check_equal(resp.headers["Content-Type"], "image/svg+xml") + self.check_equal(resp.content[:4], '