From 66c9bb86f5617beef835264906458926326ac5d3 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Mon, 15 Dec 2014 10:44:46 +0100 Subject: [PATCH 01/26] Move command line snapshot sorting to common snapshot code --- cmd/snapshot_list.go | 75 ++++++-------------------------------------- deb/snapshot.go | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/cmd/snapshot_list.go b/cmd/snapshot_list.go index 2362e9cc..8c5b6d56 100644 --- a/cmd/snapshot_list.go +++ b/cmd/snapshot_list.go @@ -4,49 +4,8 @@ import ( "fmt" "github.com/smira/aptly/deb" "github.com/smira/commander" - "sort" ) -// Snapshot sorting methods -const ( - SortName = iota - SortTime -) - -type snapshotListToSort struct { - list []*deb.Snapshot - sortMethod int -} - -func parseSortMethod(sortMethod string) (int, error) { - switch sortMethod { - case "time", "Time": - return SortTime, nil - case "name", "Name": - return SortName, nil - } - - return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod) -} - -func (s snapshotListToSort) Swap(i, j int) { - s.list[i], s.list[j] = s.list[j], s.list[i] -} - -func (s snapshotListToSort) Less(i, j int) bool { - switch s.sortMethod { - case SortName: - return s.list[i].Name < s.list[j].Name - case SortTime: - return s.list[i].CreatedAt.Before(s.list[j].CreatedAt) - } - panic("unknown sort method") -} - -func (s snapshotListToSort) Len() int { - return len(s.list) -} - func aptlySnapshotList(cmd *commander.Command, args []string) error { var err error if len(args) != 0 { @@ -57,44 +16,30 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error { raw := cmd.Flag.Lookup("raw").Value.Get().(bool) sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string) - snapshotsToSort := &snapshotListToSort{} - snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len()) - snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString) - if err != nil { - return err - } - - i := 0 - context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { - snapshotsToSort.list[i] = snapshot - i++ - - return nil - }) - - context.CloseDatabase() - - sort.Sort(snapshotsToSort) + collection := context.CollectionFactory().SnapshotCollection() + collection.Sort(sortMethodString) if raw { - for _, snapshot := range snapshotsToSort.list { + collection.ForEach(func(snapshot *deb.Snapshot) error { fmt.Printf("%s\n", snapshot.Name) - } + return nil + }) } else { - if len(snapshotsToSort.list) > 0 { + if collection.Len() > 0 { fmt.Printf("List of snapshots:\n") - for _, snapshot := range snapshotsToSort.list { + collection.ForEach(func(snapshot *deb.Snapshot) error { fmt.Printf(" * %s\n", snapshot.String()) - } + return nil + }) fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show `.\n") } else { fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n") } } - return err + return err } func makeCmdSnapshotList() *commander.Command { diff --git a/deb/snapshot.go b/deb/snapshot.go index 0f85581a..e50efb95 100644 --- a/deb/snapshot.go +++ b/deb/snapshot.go @@ -9,6 +9,7 @@ import ( "github.com/smira/aptly/utils" "github.com/ugorji/go/codec" "log" + "sort" "strings" "sync" "time" @@ -327,3 +328,58 @@ func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error { return collection.db.Delete(snapshot.RefKey()) } + +// Snapshot sorting methods +const ( + SortName = iota + SortTime +) + +type snapshotListToSort struct { + list []*Snapshot + sortMethod int +} + +func parseSortMethod(sortMethod string) (int, error) { + switch sortMethod { + case "time", "Time": + return SortTime, nil + case "name", "Name": + return SortName, nil + } + + return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod) +} + +func (s snapshotListToSort) Swap(i, j int) { + s.list[i], s.list[j] = s.list[j], s.list[i] +} + +func (s snapshotListToSort) Less(i, j int) bool { + switch s.sortMethod { + case SortName: + return s.list[i].Name < s.list[j].Name + case SortTime: + return s.list[i].CreatedAt.Before(s.list[j].CreatedAt) + } + panic("unknown sort method") +} + +func (s snapshotListToSort) Len() int { + return len(s.list) +} + +func (collection *SnapshotCollection) Sort(sortMethodString string) error { + var err error + snapshotsToSort := &snapshotListToSort{} + snapshotsToSort.list = collection.list + snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString) + if err != nil { + return err + } + + sort.Sort(snapshotsToSort) + collection.list = snapshotsToSort.list + + return err +} From 64ef3421218510fd7adb377cc7e9efcc469c9923 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Mon, 15 Dec 2014 10:47:35 +0100 Subject: [PATCH 02/26] Add /snapshots/ API. #116 Implements : - CRUD - Difference between snapshots --- api/router.go | 15 +++ api/snapshot.go | 320 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 api/snapshot.go diff --git a/api/router.go b/api/router.go index 3c6badb0..67be831e 100644 --- a/api/router.go +++ b/api/router.go @@ -29,6 +29,12 @@ func Router(c *ctx.AptlyContext) http.Handler { root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile) root.POST("/repos/:name/file/:dir", apiReposPackageFromDir) + + root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository) + } + + { + root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror) } { @@ -39,5 +45,14 @@ func Router(c *ctx.AptlyContext) http.Handler { root.DELETE("/files/:dir/:name", apiFilesDeleteFile) } + { + root.GET("/snapshots", apiSnapshotsList) + root.POST("/snapshots", apiSnapshotsCreateEmpty) + root.PUT("/snapshots/:name", apiSnapshotsRename) + root.GET("/snapshots/:name", apiSnapshotsShow) + root.DELETE("/snapshots/:name", apiSnapshotsDrop) + root.POST("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) + } + return router } diff --git a/api/snapshot.go b/api/snapshot.go new file mode 100644 index 00000000..4cd5c0ce --- /dev/null +++ b/api/snapshot.go @@ -0,0 +1,320 @@ +package api + +import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/smira/aptly/deb" +) + +// GET /api/snapshots +func apiSnapshotsList(c *gin.Context) { + SortMethodString := c.Request.URL.Query().Get("sort") + + collection := context.CollectionFactory().SnapshotCollection() + collection.Sort(SortMethodString) + + result := []*deb.Snapshot{} + collection.ForEach(func(snapshot *deb.Snapshot) error { + result = append(result, snapshot) + return nil + }) + + c.JSON(200, result) +} + +// POST /api/mirros/:name/snapshots/:snapname +func apiSnapshotsCreateFromMirror(c *gin.Context) { + var ( + err error + repo *deb.RemoteRepo + snapshot *deb.Snapshot + ) + + var b struct { + Name string `binding:"required"` + } + + if !c.Bind(&b) { + return + } + + collection := context.CollectionFactory().RemoteRepoCollection() + collection.Lock() + defer collection.Unlock() + + snapshotCollection := context.CollectionFactory().SnapshotCollection() + snapshotCollection.Lock() + defer snapshotCollection.Unlock() + + repo, err = collection.ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + return + } + + err = repo.CheckLock() + if err != nil { + c.Fail(409, err) + return + } + + err = collection.LoadComplete(repo) + if err != nil { + c.Fail(500, err) + return + } + + snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo) + if err != nil { + c.Fail(500, err) + return + } + + err = snapshotCollection.Add(snapshot) + if err != nil { + c.Fail(500, err) + return + } + + c.JSON(201, snapshot) +} + +// POST /api/repos/:name/snapshots/:snapname +func apiSnapshotsCreateEmpty(c *gin.Context) { + var ( + err error + snapshot *deb.Snapshot + ) + + var b struct { + Name string `binding:"required"` + } + + if !c.Bind(&b) { + return + } + + snapshotCollection := context.CollectionFactory().SnapshotCollection() + snapshotCollection.Lock() + defer snapshotCollection.Unlock() + + packageList := deb.NewPackageList() + + snapshot = deb.NewSnapshotFromPackageList(b.Name, nil, packageList, "Created as empty") + + err = snapshotCollection.Add(snapshot) + if err != nil { + c.Fail(500, err) + return + } + + c.JSON(201, snapshot) +} + +// POST /api/repos/:name/snapshots/:snapname +func apiSnapshotsCreateFromRepository(c *gin.Context) { + var ( + err error + repo *deb.LocalRepo + snapshot *deb.Snapshot + ) + + var b struct { + Name string `binding:"required"` + } + + if !c.Bind(&b) { + return + } + + collection := context.CollectionFactory().LocalRepoCollection() + collection.Lock() + defer collection.Unlock() + + snapshotCollection := context.CollectionFactory().SnapshotCollection() + snapshotCollection.Lock() + defer snapshotCollection.Unlock() + + repo, err = collection.ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + return + } + + err = collection.LoadComplete(repo) + if err != nil { + c.Fail(500, err) + return + } + + snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo) + if err != nil { + c.Fail(500, err) + return + } + + err = snapshotCollection.Add(snapshot) + if err != nil { + c.Fail(500, err) + return + } + + c.JSON(201, snapshot) +} + +// PUT /api/snapshots/:name +func apiSnapshotsRename(c *gin.Context) { + var ( + err error + snapshot *deb.Snapshot + ) + + var b struct { + Name string `binding:"required"` + } + + if !c.Bind(&b) { + return + } + + collection := context.CollectionFactory().SnapshotCollection() + collection.Lock() + defer collection.Unlock() + + snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + } + + _, err = context.CollectionFactory().SnapshotCollection().ByName(b.Name) + if err == nil { + c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)) + } + + snapshot.Name = b.Name + err = context.CollectionFactory().SnapshotCollection().Update(snapshot) + if err != nil { + c.Fail(403, err) + } + + c.JSON(200, snapshot) +} + +// GET /api/snapshots/:name +func apiSnapshotsShow(c *gin.Context) { + collection := context.CollectionFactory().SnapshotCollection() + collection.RLock() + defer collection.RUnlock() + + repo, err := collection.ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + return + } + + c.JSON(200, repo) +} + +// DELETE /api/snapshots/:name +func apiSnapshotsDrop(c *gin.Context) { + name := c.Params.ByName("name") + force := c.Request.URL.Query().Get("force") == "1" + + collection := context.CollectionFactory().LocalRepoCollection() + collection.Lock() + defer collection.Unlock() + + snapshotCollection := context.CollectionFactory().SnapshotCollection() + snapshotCollection.RLock() + defer snapshotCollection.RUnlock() + + publishedCollection := context.CollectionFactory().PublishedRepoCollection() + publishedCollection.RLock() + defer publishedCollection.RUnlock() + + snapshot, err := snapshotCollection.ByName(name) + if err != nil { + c.Fail(404, err) + return + } + + published := publishedCollection.BySnapshot(snapshot) + + if len(published) > 0 { + for _, repo := range published { + err = publishedCollection.LoadComplete(repo, context.CollectionFactory()) + if err != nil { + c.Fail(409, err) + return + } + } + + c.Fail(409, fmt.Errorf("unable to drop: snapshot is published")) + return + } + + if !force { + snapshots := snapshotCollection.BySnapshotSource(snapshot) + if len(snapshots) > 0 { + c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")) + return + } + } + + err = context.CollectionFactory().SnapshotCollection().Drop(snapshot) + if err != nil { + c.Fail(409, err) + return + } + + c.JSON(200, gin.H{}) +} + +// GET /api/snapshots/:name +func apiSnapshotsDiff(c *gin.Context) { + onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1" + + collection := context.CollectionFactory().SnapshotCollection() + collection.RLock() + defer collection.RUnlock() + + snapshotA, err := collection.ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + return + } + + snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot")) + if err != nil { + c.Fail(404, err) + return + } + + err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA) + if err != nil { + c.Fail(500, err) + } + + err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB) + if err != nil { + c.Fail(500, err) + } + + // Calculate diff + diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection()) + if err != nil { + c.Fail(500, err) + } + + result := []deb.PackageDiff{} + + for _, pdiff := range diff { + if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) { + continue + } + + result = append(result, pdiff) + } + + c.JSON(200, result) +} From e1382125931c3962e68d641dfec951e7d164ed57 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:13:16 +0100 Subject: [PATCH 03/26] Make files hash a type for proper JSON serialization --- deb/package.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deb/package.go b/deb/package.go index 43e0a39b..8e1c1d0b 100644 --- a/deb/package.go +++ b/deb/package.go @@ -9,6 +9,8 @@ import ( "strings" ) +type Hash uint64 + // Package is single instance of Debian package type Package struct { // Basic package properties @@ -27,7 +29,7 @@ type Package struct { // Is this udeb package IsUdeb bool // Hash of files section - FilesHash uint64 + FilesHash Hash // Is this >= 0.6 package? V06Plus bool // Offload fields @@ -38,6 +40,10 @@ type Package struct { collection *PackageCollection } +func (h *Hash) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("\"%08x\"", *h)), nil +} + // NewPackageFromControlFile creates Package from parsed Debian control file func NewPackageFromControlFile(input Stanza) *Package { result := &Package{ @@ -398,7 +404,7 @@ func (p *Package) Files() PackageFiles { // UpdateFiles saves new state of files func (p *Package) UpdateFiles(files PackageFiles) { p.files = &files - p.FilesHash = files.Hash() + p.FilesHash = Hash(files.Hash()) } // Stanza creates original stanza from package From c733129de984d919cc95423e66cc766963154291 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:12:13 +0100 Subject: [PATCH 04/26] Add search API for packages in snapshots --- api/router.go | 1 + api/snapshot.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/api/router.go b/api/router.go index 67be831e..b84356f7 100644 --- a/api/router.go +++ b/api/router.go @@ -50,6 +50,7 @@ func Router(c *ctx.AptlyContext) http.Handler { root.POST("/snapshots", apiSnapshotsCreateEmpty) root.PUT("/snapshots/:name", apiSnapshotsRename) root.GET("/snapshots/:name", apiSnapshotsShow) + root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages) root.DELETE("/snapshots/:name", apiSnapshotsDrop) root.POST("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) } diff --git a/api/snapshot.go b/api/snapshot.go index 4cd5c0ce..5925e4d5 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -2,8 +2,10 @@ package api import ( "fmt" + "sort" "github.com/gin-gonic/gin" "github.com/smira/aptly/deb" + "github.com/smira/aptly/query" ) // GET /api/snapshots @@ -318,3 +320,78 @@ func apiSnapshotsDiff(c *gin.Context) { c.JSON(200, result) } + +// GET /api/snapshots/:name/packages +func apiSnapshotsSearchPackages(c *gin.Context) { + collection := context.CollectionFactory().SnapshotCollection() + collection.RLock() + defer collection.RUnlock() + + snapshot, err := collection.ByName(c.Params.ByName("name")) + if err != nil { + c.Fail(404, err) + return + } + + err = collection.LoadComplete(snapshot) + if err != nil { + c.Fail(400, err) + return + } + + reflist := snapshot.RefList() + result := []*deb.Package{} + + list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress()) + if err != nil { + c.Fail(400, err) + return + } + + queryS := c.Request.URL.Query().Get("q") + if queryS != "" { + q, err := query.Parse(c.Request.URL.Query().Get("q")) + if err != nil { + c.Fail(401, err) + return + } + + withDeps := c.Request.URL.Query().Get("withDeps") == "1" + architecturesList := []string{} + + if withDeps { + if len(context.ArchitecturesList()) > 0 { + architecturesList = context.ArchitecturesList() + } else { + architecturesList = list.Architectures(false) + } + + sort.Strings(architecturesList) + + if len(architecturesList) == 0 { + c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly")) + return + } + } + + list.PrepareIndex() + + packages, err := list.Filter([]deb.PackageQuery{q}, withDeps, + nil, context.DependencyOptions(), architecturesList) + if err != nil { + c.Fail(400, fmt.Errorf("unable to search: %s", err)) + } + + packages.ForEach(func(p *deb.Package) error { + result = append(result, p) + return nil + }) + } else { + list.ForEach(func(p *deb.Package) error { + result = append(result, p) + return nil + }) + } + + c.JSON(200, result) +} From acde6ff2b21a7b8e0c3792f7bc650379359f67a2 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:18:37 +0100 Subject: [PATCH 05/26] Fix wrong methods comments --- api/snapshot.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index 5925e4d5..5d833585 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -24,7 +24,7 @@ func apiSnapshotsList(c *gin.Context) { c.JSON(200, result) } -// POST /api/mirros/:name/snapshots/:snapname +// POST /api/mirrors/:name/snapshots/ func apiSnapshotsCreateFromMirror(c *gin.Context) { var ( err error @@ -81,7 +81,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { c.JSON(201, snapshot) } -// POST /api/repos/:name/snapshots/:snapname +// POST /api/snapshots/:name func apiSnapshotsCreateEmpty(c *gin.Context) { var ( err error @@ -272,7 +272,7 @@ func apiSnapshotsDrop(c *gin.Context) { c.JSON(200, gin.H{}) } -// GET /api/snapshots/:name +// POST /api/snapshots/:name/diff/:name2 func apiSnapshotsDiff(c *gin.Context) { onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1" From 85f38cd73936d017d7eab0f06204b1bd18b54930 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:40:09 +0100 Subject: [PATCH 06/26] Allow setting description on snapshots using API --- api/router.go | 2 +- api/snapshot.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/api/router.go b/api/router.go index b84356f7..b891e7b5 100644 --- a/api/router.go +++ b/api/router.go @@ -48,7 +48,7 @@ func Router(c *ctx.AptlyContext) http.Handler { { root.GET("/snapshots", apiSnapshotsList) root.POST("/snapshots", apiSnapshotsCreateEmpty) - root.PUT("/snapshots/:name", apiSnapshotsRename) + root.PUT("/snapshots/:name", apiSnapshotsUpdate) root.GET("/snapshots/:name", apiSnapshotsShow) root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages) root.DELETE("/snapshots/:name", apiSnapshotsDrop) diff --git a/api/snapshot.go b/api/snapshot.go index 5d833585..e6e4174f 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -34,6 +34,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { var b struct { Name string `binding:"required"` + Description string } if !c.Bind(&b) { @@ -72,6 +73,10 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { return } + if b.Description != "" { + snapshot.Description = b.Description + } + err = snapshotCollection.Add(snapshot) if err != nil { c.Fail(500, err) @@ -90,19 +95,24 @@ func apiSnapshotsCreateEmpty(c *gin.Context) { var b struct { Name string `binding:"required"` + Description string } if !c.Bind(&b) { return } + if b.Description == "" { + b.Description = "Created as empty" + } + snapshotCollection := context.CollectionFactory().SnapshotCollection() snapshotCollection.Lock() defer snapshotCollection.Unlock() packageList := deb.NewPackageList() - snapshot = deb.NewSnapshotFromPackageList(b.Name, nil, packageList, "Created as empty") + snapshot = deb.NewSnapshotFromPackageList(b.Name, nil, packageList, b.Description) err = snapshotCollection.Add(snapshot) if err != nil { @@ -123,6 +133,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { var b struct { Name string `binding:"required"` + Description string } if !c.Bind(&b) { @@ -155,6 +166,10 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { return } + if b.Description != "" { + snapshot.Description = b.Description + } + err = snapshotCollection.Add(snapshot) if err != nil { c.Fail(500, err) @@ -165,14 +180,15 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { } // PUT /api/snapshots/:name -func apiSnapshotsRename(c *gin.Context) { +func apiSnapshotsUpdate(c *gin.Context) { var ( err error snapshot *deb.Snapshot ) var b struct { - Name string `binding:"required"` + Name string + Description string } if !c.Bind(&b) { @@ -193,7 +209,14 @@ func apiSnapshotsRename(c *gin.Context) { c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)) } - snapshot.Name = b.Name + if b.Name != "" { + snapshot.Name = b.Name + } + + if b.Description != "" { + snapshot.Description = b.Description + } + err = context.CollectionFactory().SnapshotCollection().Update(snapshot) if err != nil { c.Fail(403, err) From a6fc65ff4ebb2b77357dcd97e4710fa88534137c Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:41:11 +0100 Subject: [PATCH 07/26] Sanitize snapshots API return codes --- api/snapshot.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index e6e4174f..6eb2bde6 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -69,7 +69,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo) if err != nil { - c.Fail(500, err) + c.Fail(400, err) return } @@ -162,7 +162,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo) if err != nil { - c.Fail(500, err) + c.Fail(400, err) return } @@ -202,11 +202,13 @@ func apiSnapshotsUpdate(c *gin.Context) { snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(c.Params.ByName("name")) if err != nil { c.Fail(404, err) + return } _, err = context.CollectionFactory().SnapshotCollection().ByName(b.Name) if err == nil { c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)) + return } if b.Name != "" { @@ -220,6 +222,7 @@ func apiSnapshotsUpdate(c *gin.Context) { err = context.CollectionFactory().SnapshotCollection().Update(snapshot) if err != nil { c.Fail(403, err) + return } c.JSON(200, snapshot) @@ -269,7 +272,7 @@ func apiSnapshotsDrop(c *gin.Context) { for _, repo := range published { err = publishedCollection.LoadComplete(repo, context.CollectionFactory()) if err != nil { - c.Fail(409, err) + c.Fail(500, err) return } } @@ -288,7 +291,7 @@ func apiSnapshotsDrop(c *gin.Context) { err = context.CollectionFactory().SnapshotCollection().Drop(snapshot) if err != nil { - c.Fail(409, err) + c.Fail(500, err) return } @@ -318,17 +321,20 @@ func apiSnapshotsDiff(c *gin.Context) { err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA) if err != nil { c.Fail(500, err) + return } err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB) if err != nil { c.Fail(500, err) + return } // Calculate diff diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection()) if err != nil { c.Fail(500, err) + return } result := []deb.PackageDiff{} @@ -358,7 +364,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) { err = collection.LoadComplete(snapshot) if err != nil { - c.Fail(400, err) + c.Fail(500, err) return } @@ -367,7 +373,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) { list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress()) if err != nil { - c.Fail(400, err) + c.Fail(404, err) return } @@ -375,7 +381,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) { if queryS != "" { q, err := query.Parse(c.Request.URL.Query().Get("q")) if err != nil { - c.Fail(401, err) + c.Fail(400, err) return } @@ -402,7 +408,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) { packages, err := list.Filter([]deb.PackageQuery{q}, withDeps, nil, context.DependencyOptions(), architecturesList) if err != nil { - c.Fail(400, fmt.Errorf("unable to search: %s", err)) + c.Fail(500, fmt.Errorf("unable to search: %s", err)) } packages.ForEach(func(p *deb.Package) error { From d983e10d089decdc8b80202b03f4b3eae800a784 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 11:42:11 +0100 Subject: [PATCH 08/26] Add snapshots API test suite --- system/api_lib.py | 8 ++ system/lib.py | 11 +++ system/t12_api/__init__.py | 1 + system/t12_api/snapshots.py | 144 ++++++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 system/t12_api/snapshots.py diff --git a/system/api_lib.py b/system/api_lib.py index f9646c7c..c62270c7 100644 --- a/system/api_lib.py +++ b/system/api_lib.py @@ -47,6 +47,14 @@ class APITest(BaseTest): kwargs["headers"]["Content-Type"] = "application/json" return requests.post("http://%s%s" % (self.base_url, uri), *args, **kwargs) + def put(self, uri, *args, **kwargs): + if "json" in kwargs: + kwargs["data"] = json.dumps(kwargs.pop("json")) + if not "headers" in kwargs: + kwargs["headers"] = {} + kwargs["headers"]["Content-Type"] = "application/json" + return requests.put("http://%s%s" % (self.base_url, uri), *args, **kwargs) + def delete(self, uri, *args, **kwargs): if "json" in kwargs: kwargs["data"] = json.dumps(kwargs.pop("json")) diff --git a/system/lib.py b/system/lib.py index 7782729e..dc6ebd94 100644 --- a/system/lib.py +++ b/system/lib.py @@ -267,6 +267,17 @@ class BaseTest(object): if a != b: self.verify_match(a, b, match_prepare=pprint.pformat) + def check_subset(self, a, b): + diff = '' + for k, v in a.items(): + if k not in b: + diff += "unexpected key '%s'\n" % (k,) + elif b[k] != v: + diff += "wrong value '%s' for key '%s', expected '%s'\n" % (v, k, b[k]) + if diff: + raise Exception("content doesn't match:\n" + diff) + + def verify_match(self, a, b, match_prepare=None): if match_prepare is not None: a = match_prepare(a) diff --git a/system/t12_api/__init__.py b/system/t12_api/__init__.py index 51c8c87f..93ef8eff 100644 --- a/system/t12_api/__init__.py +++ b/system/t12_api/__init__.py @@ -4,3 +4,4 @@ Testing aptly REST API from .repos import * from .files import * +from .snapshots import * \ No newline at end of file diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py new file mode 100644 index 00000000..c2f3e4c6 --- /dev/null +++ b/system/t12_api/snapshots.py @@ -0,0 +1,144 @@ +from api_lib import APITest + + +class SnapshotsAPITestCreateShow(APITest): + """ + GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages + """ + def check(self): + snapshot_name = self.random_name() + snapshot_desc = {u'Description': u'fun snapshot', + u'Name': snapshot_name} + + resp = self.post("/api/snapshots", json=snapshot_desc) + self.check_subset(snapshot_desc, resp.json()) + self.check_equal(resp.status_code, 201) + + self.check_subset(snapshot_desc, self.get("/api/snapshots/" + snapshot_name).json()) + self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) + + self.check_equal(self.get("/api/snapshots/" + self.random_name()).status_code, 404) + + +class SnapshotsAPITestCreateFromRepo(APITest): + """ + POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name + """ + def check(self): + repo_name = self.random_name() + snapshot_name = self.random_name() + self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) + + resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) + self.check_equal(resp.status_code, 400) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) + self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) + + self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, + self.get("/api/snapshots/" + snapshot_name + "/packages").json()[0]) + + self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, + self.get("/api/snapshots/" + snapshot_name + "/packages", params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) + + +class SnapshotsAPITestCreateUpdate(APITest): + """ + POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name + """ + def check(self): + snapshot_name = self.random_name() + snapshot_desc = {u'Description': u'fun snapshot', + u'Name': snapshot_name} + + resp = self.post("/api/snapshots", json=snapshot_desc) + self.check_equal(resp.status_code, 201) + + new_snapshot_name = self.random_name() + resp = self.put("/api/snapshots/" + snapshot_name, json={'Name': new_snapshot_name, + 'Description': 'New description'}) + self.check_equal(resp.status_code, 200) + + resp = self.get("/api/snapshots/" + new_snapshot_name) + self.check_equal(resp.status_code, 200) + self.check_subset({"Name": new_snapshot_name, + "Description": "New description"}, resp.json()) + + +class SnapshotsAPITestCreateDelete(APITest): + """ + POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name + """ + def check(self): + snapshot_name = self.random_name() + snapshot_desc = {u'Description': u'fun snapshot', + u'Name': snapshot_name} + + resp = self.post("/api/snapshots", json=snapshot_desc) + self.check_equal(resp.status_code, 201) + + self.check_equal(self.delete("/api/snapshots/" + snapshot_name).status_code, 200) + + self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 404) + + +class SnapshotsAPITestSearch(APITest): + """ + POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name + """ + def check(self): + + repo_name = self.random_name() + self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) + + d = self.random_name() + snapshot_name = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) + self.check_equal(resp.status_code, 201) + + resp = self.get("/api/snapshots/" + snapshot_name + "/packages?q=libboost-program-options-dev") + self.check_equal(resp.status_code, 200) + + self.check_equal(len(resp.json()), 1) + self.check_equal(resp.json()[0]["Name"], "libboost-program-options-dev") + + +class SnapshotsAPITestDiff(APITest): + """ + POST /api/snapshot/:name/diff/:name2 + """ + def check(self): + repos = [ self.random_name() for x in xrange(2) ] + snapshots = [ self.random_name() for x in xrange(2) ] + + for repo_name in repos: + self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) + + resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshots[0]}) + self.check_equal(resp.status_code, 201) + + resp = self.post("/api/snapshots", json={'Name': snapshots[1]}) + self.check_equal(resp.status_code, 201) + + resp = self.post("/api/snapshots/" + snapshots[0] + "/diff/" + snapshots[1]) + + self.check_equal(resp.status_code, 200) + self.check_subset({"Right": None}, resp.json()[0]) + self.check_subset({"Name": "libboost-program-options-dev"}, resp.json()[0]["Left"]) From d847cba8704e3fd38abf7562cc4d37c5d581b846 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 18:16:35 +0100 Subject: [PATCH 09/26] Make repos and snapshots API return JSON objects for packages when asked --- api/repos.go | 30 +++++++++++++++++++----------- api/snapshot.go | 25 +++++++++++++++---------- system/t12_api/snapshots.py | 12 +++++++++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/api/repos.go b/api/repos.go index 3050f028..908beb58 100644 --- a/api/repos.go +++ b/api/repos.go @@ -178,6 +178,15 @@ func apiReposPackagesShow(c *gin.Context) { return } + list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil) + if err != nil { + c.Fail(500, err) + return + } + + list.PrepareIndex() + + result := []*deb.Package{} queryS := c.Request.URL.Query().Get("q") if queryS != "" { q, err := query.Parse(queryS) @@ -186,14 +195,6 @@ func apiReposPackagesShow(c *gin.Context) { return } - list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil) - if err != nil { - c.Fail(500, err) - return - } - - list.PrepareIndex() - withDeps := c.Request.URL.Query().Get("withDeps") == "1" architecturesList := []string{} @@ -212,16 +213,23 @@ func apiReposPackagesShow(c *gin.Context) { } } - result, err := list.Filter([]deb.PackageQuery{q}, withDeps, + list, err = list.Filter([]deb.PackageQuery{q}, withDeps, nil, context.DependencyOptions(), architecturesList) if err != nil { c.Fail(500, err) return } + } - c.JSON(200, result.Strings()) + if c.Request.URL.Query().Get("format") == "details" { + list.ForEach(func(p *deb.Package) error { + result = append(result, p) + return nil + }) + + c.JSON(200, result) } else { - c.JSON(200, repo.RefList().Strings()) + c.JSON(200, list.Strings()) } } diff --git a/api/snapshot.go b/api/snapshot.go index 6eb2bde6..ad3f0148 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -234,13 +234,19 @@ func apiSnapshotsShow(c *gin.Context) { collection.RLock() defer collection.RUnlock() - repo, err := collection.ByName(c.Params.ByName("name")) + snapshot, err := collection.ByName(c.Params.ByName("name")) if err != nil { c.Fail(404, err) return } - c.JSON(200, repo) + err = collection.LoadComplete(snapshot) + if err != nil { + c.Fail(500, err) + return + } + + c.JSON(200, snapshot) } // DELETE /api/snapshots/:name @@ -405,22 +411,21 @@ func apiSnapshotsSearchPackages(c *gin.Context) { list.PrepareIndex() - packages, err := list.Filter([]deb.PackageQuery{q}, withDeps, + list, err = list.Filter([]deb.PackageQuery{q}, withDeps, nil, context.DependencyOptions(), architecturesList) if err != nil { c.Fail(500, fmt.Errorf("unable to search: %s", err)) } + } - packages.ForEach(func(p *deb.Package) error { - result = append(result, p) - return nil - }) - } else { + if c.Request.URL.Query().Get("format") == "details" { list.ForEach(func(p *deb.Package) error { result = append(result, p) return nil }) - } - c.JSON(200, result) + c.JSON(200, result) + } else { + c.JSON(200, list.Strings()) + } } diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index c2f3e4c6..42c59d4d 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -42,10 +42,10 @@ class SnapshotsAPITestCreateFromRepo(APITest): self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, - self.get("/api/snapshots/" + snapshot_name + "/packages").json()[0]) + self.get("/api/snapshots/" + snapshot_name + "/packages?format=details").json()[0]) self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, - self.get("/api/snapshots/" + snapshot_name + "/packages", params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) + self.get("/api/snapshots/" + snapshot_name + "/packages?format=details", params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) class SnapshotsAPITestCreateUpdate(APITest): @@ -107,12 +107,18 @@ class SnapshotsAPITestSearch(APITest): resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) self.check_equal(resp.status_code, 201) - resp = self.get("/api/snapshots/" + snapshot_name + "/packages?q=libboost-program-options-dev") + resp = self.get("/api/snapshots/" + snapshot_name + "/packages?q=libboost-program-options-dev&format=details") self.check_equal(resp.status_code, 200) self.check_equal(len(resp.json()), 1) self.check_equal(resp.json()[0]["Name"], "libboost-program-options-dev") + resp = self.get("/api/snapshots/" + snapshot_name + "/packages") + self.check_equal(resp.status_code, 200) + + self.check_equal(len(resp.json()), 1) + self.check_equal(resp.json(), ["Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378"]) + class SnapshotsAPITestDiff(APITest): """ From dd9fc8e40ed329d427e6cc73e27b2fd5a5c11504 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Thu, 18 Dec 2014 18:17:43 +0100 Subject: [PATCH 10/26] Allow API creation of snapshots using package references --- api/router.go | 2 +- api/snapshot.go | 32 ++++++++++++++++++++++++++++---- system/t12_api/snapshots.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/api/router.go b/api/router.go index b891e7b5..88588722 100644 --- a/api/router.go +++ b/api/router.go @@ -47,7 +47,7 @@ func Router(c *ctx.AptlyContext) http.Handler { { root.GET("/snapshots", apiSnapshotsList) - root.POST("/snapshots", apiSnapshotsCreateEmpty) + root.POST("/snapshots", apiSnapshotsCreate) root.PUT("/snapshots/:name", apiSnapshotsUpdate) root.GET("/snapshots/:name", apiSnapshotsShow) root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages) diff --git a/api/snapshot.go b/api/snapshot.go index ad3f0148..2ae75dac 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -87,7 +87,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { } // POST /api/snapshots/:name -func apiSnapshotsCreateEmpty(c *gin.Context) { +func apiSnapshotsCreate(c *gin.Context) { var ( err error snapshot *deb.Snapshot @@ -96,6 +96,8 @@ func apiSnapshotsCreateEmpty(c *gin.Context) { var b struct { Name string `binding:"required"` Description string + SourceIDs []string + PackageRefs []string } if !c.Bind(&b) { @@ -103,16 +105,38 @@ func apiSnapshotsCreateEmpty(c *gin.Context) { } if b.Description == "" { - b.Description = "Created as empty" + if len(b.SourceIDs) + len(b.PackageRefs) == 0 { + b.Description = "Created as empty" + } } snapshotCollection := context.CollectionFactory().SnapshotCollection() snapshotCollection.Lock() defer snapshotCollection.Unlock() - packageList := deb.NewPackageList() + sources := make([]*deb.Snapshot, len(b.SourceIDs)) - snapshot = deb.NewSnapshotFromPackageList(b.Name, nil, packageList, b.Description) + for i := 0; i < len(b.SourceIDs); i++ { + sources[i], err = snapshotCollection.ByUUID(b.SourceIDs[i]) + if err != nil { + c.Fail(404, err) + return + } + + err = snapshotCollection.LoadComplete(sources[i]) + if err != nil { + c.Fail(500, err) + return + } + } + + packageRefs := make([][]byte, len(b.PackageRefs)) + for i, ref := range b.PackageRefs { + packageRefs[i] = []byte(ref) + } + + packageRefList := &deb.PackageRefList{packageRefs} + snapshot = deb.NewSnapshotFromRefList(b.Name, sources, packageRefList, b.Description) err = snapshotCollection.Add(snapshot) if err != nil { diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 42c59d4d..20d8da4a 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -20,6 +20,35 @@ class SnapshotsAPITestCreateShow(APITest): self.check_equal(self.get("/api/snapshots/" + self.random_name()).status_code, 404) +class SnapshotsAPITestCreateFromRefs(APITest): + """ + GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages + """ + def check(self): + snapshot_name = self.random_name() + snapshot_desc = {u'Description': u'fun snapshot', + u'Name': snapshot_name, + u'SourceIDs': ['123']} + + resp = self.post("/api/snapshots", json=snapshot_desc) + self.check_equal(resp.status_code, 404) + + resp = self.post("/api/snapshots", json={"Name": self.random_name()}) + self.check_equal(resp.status_code, 201) + snapshot_desc['SourceIDs'] = [resp.json()["UUID"]] + + snapshot = snapshot_desc.copy() + snapshot['PackageRefs'] = ["Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378"] + resp = self.post("/api/snapshots", json=snapshot) + self.check_equal(resp.status_code, 201) + self.check_subset(snapshot_desc, resp.json()) + + self.check_subset(snapshot_desc, self.get("/api/snapshots/" + snapshot_name).json()) + self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) + + self.check_equal(self.get("/api/snapshots/" + self.random_name()).status_code, 404) + + class SnapshotsAPITestCreateFromRepo(APITest): """ POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name From 6bc70481662d0a7908ab94309f3c6fd0e5540a3e Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Tue, 6 Jan 2015 18:00:04 +0100 Subject: [PATCH 11/26] Fix wrong method comment --- api/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/snapshot.go b/api/snapshot.go index 2ae75dac..fa1a5fc7 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -86,7 +86,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { c.JSON(201, snapshot) } -// POST /api/snapshots/:name +// POST /api/snapshots func apiSnapshotsCreate(c *gin.Context) { var ( err error From 6a1a871dda4a42d096cdb641c9aa4d4a434ebc7c Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Tue, 6 Jan 2015 18:01:11 +0100 Subject: [PATCH 12/26] Lock snapshot collection before sorting --- api/snapshot.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/snapshot.go b/api/snapshot.go index fa1a5fc7..ae39bbab 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -13,7 +13,12 @@ func apiSnapshotsList(c *gin.Context) { SortMethodString := c.Request.URL.Query().Get("sort") collection := context.CollectionFactory().SnapshotCollection() - collection.Sort(SortMethodString) + collection.RLock() + defer collection.RUnlock() + + if SortMethodString != "" { + collection.Sort(SortMethodString) + } result := []*deb.Snapshot{} collection.ForEach(func(snapshot *deb.Snapshot) error { From d828732307a126f4e2886f46d07237094e4adc6b Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 22 Jan 2015 22:01:00 +0300 Subject: [PATCH 13/26] Refactoring: make snapshot sorting non-intrusive to collection contents. #168 --- api/snapshot.go | 52 ++++++++--------- cmd/snapshot_list.go | 9 ++- deb/snapshot.go | 65 +++++++++++++--------- system/t05_snapshot/ListSnapshot7Test_gold | 1 + system/t05_snapshot/list.py | 3 + 5 files changed, 75 insertions(+), 55 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index ae39bbab..ad96cff0 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -2,10 +2,10 @@ package api import ( "fmt" - "sort" "github.com/gin-gonic/gin" "github.com/smira/aptly/deb" "github.com/smira/aptly/query" + "sort" ) // GET /api/snapshots @@ -16,12 +16,12 @@ func apiSnapshotsList(c *gin.Context) { collection.RLock() defer collection.RUnlock() - if SortMethodString != "" { - collection.Sort(SortMethodString) + if SortMethodString == "" { + SortMethodString = "name" } result := []*deb.Snapshot{} - collection.ForEach(func(snapshot *deb.Snapshot) error { + collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error { result = append(result, snapshot) return nil }) @@ -32,14 +32,14 @@ func apiSnapshotsList(c *gin.Context) { // POST /api/mirrors/:name/snapshots/ func apiSnapshotsCreateFromMirror(c *gin.Context) { var ( - err error + err error repo *deb.RemoteRepo snapshot *deb.Snapshot ) var b struct { - Name string `binding:"required"` - Description string + Name string `binding:"required"` + Description string } if !c.Bind(&b) { @@ -94,15 +94,15 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { // POST /api/snapshots func apiSnapshotsCreate(c *gin.Context) { var ( - err error + err error snapshot *deb.Snapshot ) var b struct { - Name string `binding:"required"` - Description string - SourceIDs []string - PackageRefs []string + Name string `binding:"required"` + Description string + SourceIDs []string + PackageRefs []string } if !c.Bind(&b) { @@ -110,7 +110,7 @@ func apiSnapshotsCreate(c *gin.Context) { } if b.Description == "" { - if len(b.SourceIDs) + len(b.PackageRefs) == 0 { + if len(b.SourceIDs)+len(b.PackageRefs) == 0 { b.Description = "Created as empty" } } @@ -119,23 +119,23 @@ func apiSnapshotsCreate(c *gin.Context) { snapshotCollection.Lock() defer snapshotCollection.Unlock() - sources := make([]*deb.Snapshot, len(b.SourceIDs)) + sources := make([]*deb.Snapshot, len(b.SourceIDs)) - for i := 0; i < len(b.SourceIDs); i++ { - sources[i], err = snapshotCollection.ByUUID(b.SourceIDs[i]) - if err != nil { + for i := 0; i < len(b.SourceIDs); i++ { + sources[i], err = snapshotCollection.ByUUID(b.SourceIDs[i]) + if err != nil { c.Fail(404, err) return - } + } - err = snapshotCollection.LoadComplete(sources[i]) - if err != nil { + err = snapshotCollection.LoadComplete(sources[i]) + if err != nil { c.Fail(500, err) return - } - } + } + } - packageRefs := make([][]byte, len(b.PackageRefs)) + packageRefs := make([][]byte, len(b.PackageRefs)) for i, ref := range b.PackageRefs { packageRefs[i] = []byte(ref) } @@ -155,14 +155,14 @@ func apiSnapshotsCreate(c *gin.Context) { // POST /api/repos/:name/snapshots/:snapname func apiSnapshotsCreateFromRepository(c *gin.Context) { var ( - err error + err error repo *deb.LocalRepo snapshot *deb.Snapshot ) var b struct { - Name string `binding:"required"` - Description string + Name string `binding:"required"` + Description string } if !c.Bind(&b) { diff --git a/cmd/snapshot_list.go b/cmd/snapshot_list.go index 8c5b6d56..06ba631c 100644 --- a/cmd/snapshot_list.go +++ b/cmd/snapshot_list.go @@ -17,10 +17,9 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error { sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string) collection := context.CollectionFactory().SnapshotCollection() - collection.Sort(sortMethodString) if raw { - collection.ForEach(func(snapshot *deb.Snapshot) error { + collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error { fmt.Printf("%s\n", snapshot.Name) return nil }) @@ -28,11 +27,15 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error { if collection.Len() > 0 { fmt.Printf("List of snapshots:\n") - collection.ForEach(func(snapshot *deb.Snapshot) error { + err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error { fmt.Printf(" * %s\n", snapshot.String()) return nil }) + if err != nil { + return err + } + fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show `.\n") } else { fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n") diff --git a/deb/snapshot.go b/deb/snapshot.go index e50efb95..eade4ece 100644 --- a/deb/snapshot.go +++ b/deb/snapshot.go @@ -297,6 +297,22 @@ func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) err return err } +func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error { + sorter, err := newSnapshotSorter(sortMethod, collection) + if err != nil { + return err + } + + for _, i := range sorter.list { + err = handler(collection.list[i]) + if err != nil { + return err + } + } + + return nil +} + // Len returns number of snapshots in collection // ForEach runs method for each snapshot func (collection *SnapshotCollection) Len() int { @@ -335,51 +351,48 @@ const ( SortTime ) -type snapshotListToSort struct { - list []*Snapshot +type snapshotSorter struct { + list []int + collection *SnapshotCollection sortMethod int } -func parseSortMethod(sortMethod string) (int, error) { +func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snapshotSorter, error) { + s := &snapshotSorter{collection: collection} + switch sortMethod { case "time", "Time": - return SortTime, nil + s.sortMethod = SortTime case "name", "Name": - return SortName, nil + s.sortMethod = SortName + default: + return nil, fmt.Errorf("sorting method \"%s\" unknown", sortMethod) } - return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod) + s.list = make([]int, len(collection.list)) + for i := range s.list { + s.list[i] = i + } + + sort.Sort(s) + + return s, nil } -func (s snapshotListToSort) Swap(i, j int) { +func (s *snapshotSorter) Swap(i, j int) { s.list[i], s.list[j] = s.list[j], s.list[i] } -func (s snapshotListToSort) Less(i, j int) bool { +func (s *snapshotSorter) Less(i, j int) bool { switch s.sortMethod { case SortName: - return s.list[i].Name < s.list[j].Name + return s.collection.list[s.list[i]].Name < s.collection.list[s.list[j]].Name case SortTime: - return s.list[i].CreatedAt.Before(s.list[j].CreatedAt) + return s.collection.list[s.list[i]].CreatedAt.Before(s.collection.list[s.list[j]].CreatedAt) } panic("unknown sort method") } -func (s snapshotListToSort) Len() int { +func (s *snapshotSorter) Len() int { return len(s.list) } - -func (collection *SnapshotCollection) Sort(sortMethodString string) error { - var err error - snapshotsToSort := &snapshotListToSort{} - snapshotsToSort.list = collection.list - snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString) - if err != nil { - return err - } - - sort.Sort(snapshotsToSort) - collection.list = snapshotsToSort.list - - return err -} diff --git a/system/t05_snapshot/ListSnapshot7Test_gold b/system/t05_snapshot/ListSnapshot7Test_gold index a9dfcb04..c706052d 100644 --- a/system/t05_snapshot/ListSnapshot7Test_gold +++ b/system/t05_snapshot/ListSnapshot7Test_gold @@ -1 +1,2 @@ +List of snapshots: ERROR: sorting method "planet" unknown diff --git a/system/t05_snapshot/list.py b/system/t05_snapshot/list.py index d3e576d0..49bd1f6a 100644 --- a/system/t05_snapshot/list.py +++ b/system/t05_snapshot/list.py @@ -87,5 +87,8 @@ class ListSnapshot7Test(BaseTest): """ list snapshots: wrong parameter sort """ + fixtureCmds = [ + "aptly snapshot create empty empty" + ] runCmd = "aptly -sort=planet snapshot list" expectedCode = 1 From ebea4f10a008fb1b4343f80ffe41ef09912aacba Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Sat, 24 Jan 2015 21:51:33 +0300 Subject: [PATCH 14/26] Make snapshot diff GET, not POST (as it doesn't change anything in the system). #168 --- api/router.go | 2 +- api/snapshot.go | 2 +- system/t12_api/snapshots.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/router.go b/api/router.go index 3892c86f..a049bfed 100644 --- a/api/router.go +++ b/api/router.go @@ -65,7 +65,7 @@ func Router(c *ctx.AptlyContext) http.Handler { root.GET("/snapshots/:name", apiSnapshotsShow) root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages) root.DELETE("/snapshots/:name", apiSnapshotsDrop) - root.POST("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) + root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) } { diff --git a/api/snapshot.go b/api/snapshot.go index ad96cff0..863ee026 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -333,7 +333,7 @@ func apiSnapshotsDrop(c *gin.Context) { c.JSON(200, gin.H{}) } -// POST /api/snapshots/:name/diff/:name2 +// GET /api/snapshots/:name/diff/:withSnapshot func apiSnapshotsDiff(c *gin.Context) { onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1" diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 20d8da4a..88681b5b 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -62,7 +62,7 @@ class SnapshotsAPITestCreateFromRepo(APITest): self.check_equal(resp.status_code, 400) d = self.random_name() - self.check_equal(self.upload("/api/files/" + d, + self.check_equal(self.upload("/api/files/" + d, "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) @@ -128,7 +128,7 @@ class SnapshotsAPITestSearch(APITest): d = self.random_name() snapshot_name = self.random_name() - self.check_equal(self.upload("/api/files/" + d, + self.check_equal(self.upload("/api/files/" + d, "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) @@ -161,7 +161,7 @@ class SnapshotsAPITestDiff(APITest): self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) d = self.random_name() - self.check_equal(self.upload("/api/files/" + d, + self.check_equal(self.upload("/api/files/" + d, "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) self.check_equal(self.post("/api/repos/" + repo_name + "/file/" + d).status_code, 200) @@ -172,7 +172,7 @@ class SnapshotsAPITestDiff(APITest): resp = self.post("/api/snapshots", json={'Name': snapshots[1]}) self.check_equal(resp.status_code, 201) - resp = self.post("/api/snapshots/" + snapshots[0] + "/diff/" + snapshots[1]) + resp = self.get("/api/snapshots/" + snapshots[0] + "/diff/" + snapshots[1]) self.check_equal(resp.status_code, 200) self.check_subset({"Right": None}, resp.json()[0]) From 9c60421bd663c4170775d031ab8b82802df163db Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Sat, 24 Jan 2015 21:53:06 +0300 Subject: [PATCH 15/26] Python style fixes. #168 --- system/t12_api/snapshots.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 88681b5b..f1952371 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -70,11 +70,18 @@ class SnapshotsAPITestCreateFromRepo(APITest): resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) - self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, + self.check_subset({u'Architecture': 'i386', + u'Name': 'libboost-program-options-dev', + u'Version': '1.49.0.1', + 'FilesHash': '918d2f433384e378'}, self.get("/api/snapshots/" + snapshot_name + "/packages?format=details").json()[0]) - self.check_subset({u'Architecture': 'i386', u'Name': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, - self.get("/api/snapshots/" + snapshot_name + "/packages?format=details", params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) + self.check_subset({u'Architecture': 'i386', + u'Name': 'libboost-program-options-dev', + u'Version': '1.49.0.1', + 'FilesHash': '918d2f433384e378'}, + self.get("/api/snapshots/" + snapshot_name + "/packages?format=details", + params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) class SnapshotsAPITestCreateUpdate(APITest): @@ -154,8 +161,8 @@ class SnapshotsAPITestDiff(APITest): POST /api/snapshot/:name/diff/:name2 """ def check(self): - repos = [ self.random_name() for x in xrange(2) ] - snapshots = [ self.random_name() for x in xrange(2) ] + repos = [self.random_name() for x in xrange(2)] + snapshots = [self.random_name() for x in xrange(2)] for repo_name in repos: self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) From 925047984674dcf414038a859f880215d97d2b37 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Sat, 24 Jan 2015 22:23:16 +0300 Subject: [PATCH 16/26] Extract common part of show and search packages from snapshots and repos. #168 --- api/api.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ api/repos.go | 56 +------------------------------------------- api/snapshot.go | 57 +-------------------------------------------- 3 files changed, 64 insertions(+), 111 deletions(-) diff --git a/api/api.go b/api/api.go index 30ba1a24..e86e8012 100644 --- a/api/api.go +++ b/api/api.go @@ -2,8 +2,12 @@ package api import ( + "fmt" "github.com/gin-gonic/gin" "github.com/smira/aptly/aptly" + "github.com/smira/aptly/deb" + "github.com/smira/aptly/query" + "sort" ) // Lock order acquisition (canonical): @@ -16,3 +20,61 @@ import ( func apiVersion(c *gin.Context) { c.JSON(200, gin.H{"Version": aptly.Version}) } + +// Common piece of code to show list of packages, +// with searching & details if requested +func showPackages(c *gin.Context, reflist *deb.PackageRefList) { + result := []*deb.Package{} + + list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress()) + if err != nil { + c.Fail(404, err) + return + } + + queryS := c.Request.URL.Query().Get("q") + if queryS != "" { + q, err := query.Parse(c.Request.URL.Query().Get("q")) + if err != nil { + c.Fail(400, err) + return + } + + withDeps := c.Request.URL.Query().Get("withDeps") == "1" + architecturesList := []string{} + + if withDeps { + if len(context.ArchitecturesList()) > 0 { + architecturesList = context.ArchitecturesList() + } else { + architecturesList = list.Architectures(false) + } + + sort.Strings(architecturesList) + + if len(architecturesList) == 0 { + c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly")) + return + } + } + + list.PrepareIndex() + + list, err = list.Filter([]deb.PackageQuery{q}, withDeps, + nil, context.DependencyOptions(), architecturesList) + if err != nil { + c.Fail(500, fmt.Errorf("unable to search: %s", err)) + } + } + + if c.Request.URL.Query().Get("format") == "details" { + list.ForEach(func(p *deb.Package) error { + result = append(result, p) + return nil + }) + + c.JSON(200, result) + } else { + c.JSON(200, list.Strings()) + } +} diff --git a/api/repos.go b/api/repos.go index 44e163cb..35b2b2bb 100644 --- a/api/repos.go +++ b/api/repos.go @@ -6,11 +6,9 @@ import ( "github.com/smira/aptly/aptly" "github.com/smira/aptly/database" "github.com/smira/aptly/deb" - "github.com/smira/aptly/query" "github.com/smira/aptly/utils" "os" "path/filepath" - "sort" ) // GET /api/repos @@ -178,59 +176,7 @@ func apiReposPackagesShow(c *gin.Context) { return } - list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil) - if err != nil { - c.Fail(500, err) - return - } - - list.PrepareIndex() - - result := []*deb.Package{} - queryS := c.Request.URL.Query().Get("q") - if queryS != "" { - q, err := query.Parse(queryS) - if err != nil { - c.Fail(400, err) - return - } - - withDeps := c.Request.URL.Query().Get("withDeps") == "1" - architecturesList := []string{} - - if withDeps { - if len(context.ArchitecturesList()) > 0 { - architecturesList = context.ArchitecturesList() - } else { - architecturesList = list.Architectures(false) - } - - sort.Strings(architecturesList) - - if len(architecturesList) == 0 { - c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly")) - return - } - } - - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) - if err != nil { - c.Fail(500, err) - return - } - } - - if c.Request.URL.Query().Get("format") == "details" { - list.ForEach(func(p *deb.Package) error { - result = append(result, p) - return nil - }) - - c.JSON(200, result) - } else { - c.JSON(200, list.Strings()) - } + showPackages(c, repo.RefList()) } // Handler for both add and delete diff --git a/api/snapshot.go b/api/snapshot.go index 863ee026..366c75d2 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/gin-gonic/gin" "github.com/smira/aptly/deb" - "github.com/smira/aptly/query" - "sort" ) // GET /api/snapshots @@ -403,58 +401,5 @@ func apiSnapshotsSearchPackages(c *gin.Context) { return } - reflist := snapshot.RefList() - result := []*deb.Package{} - - list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress()) - if err != nil { - c.Fail(404, err) - return - } - - queryS := c.Request.URL.Query().Get("q") - if queryS != "" { - q, err := query.Parse(c.Request.URL.Query().Get("q")) - if err != nil { - c.Fail(400, err) - return - } - - withDeps := c.Request.URL.Query().Get("withDeps") == "1" - architecturesList := []string{} - - if withDeps { - if len(context.ArchitecturesList()) > 0 { - architecturesList = context.ArchitecturesList() - } else { - architecturesList = list.Architectures(false) - } - - sort.Strings(architecturesList) - - if len(architecturesList) == 0 { - c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly")) - return - } - } - - list.PrepareIndex() - - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) - if err != nil { - c.Fail(500, fmt.Errorf("unable to search: %s", err)) - } - } - - if c.Request.URL.Query().Get("format") == "details" { - list.ForEach(func(p *deb.Package) error { - result = append(result, p) - return nil - }) - - c.JSON(200, result) - } else { - c.JSON(200, list.Strings()) - } + showPackages(c, snapshot.RefList()) } From 3f6491b8a3a86b72b44c1da1fba67961c39510c2 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 26 Jan 2015 21:06:28 +0300 Subject: [PATCH 17/26] Non-working test on format=details. --- system/t12_api/repos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system/t12_api/repos.py b/system/t12_api/repos.py index 678c5073..dbd94429 100644 --- a/system/t12_api/repos.py +++ b/system/t12_api/repos.py @@ -163,6 +163,9 @@ class ReposAPITestShowQuery(APITest): self.check_equal(sorted(self.get("/api/repos/" + repo_name + "/packages", params={"q": "Version (> 0.6.1-1.4)"}).json()), ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378', 'Psource pyspi 0.6.1-1.4 f8f1daa806004e89']) + self.check_equal(sorted(self.get("/api/repos/" + repo_name + "/packages", params={"q": "pyspi", "format": "details"}).json()), + ['Psource pyspi 0.6.1-1.3 3a8b37cbd9a3559e', 'Psource pyspi 0.6.1-1.4 f8f1daa806004e89']) + resp = self.get("/api/repos/" + repo_name + "/packages", params={"q": "pyspi)"}) self.check_equal(resp.status_code, 400) self.check_equal(resp.json()[0]["error"], u'parsing failed: unexpected token ): expecting end of query') From 8e8ff8ba65b3660d8b256dd066a4e27c9a9b8faa Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 00:27:14 +0300 Subject: [PATCH 18/26] Revert "Make files hash a type for proper JSON serialization" This reverts commit e1382125931c3962e68d641dfec951e7d164ed57. --- deb/package.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/deb/package.go b/deb/package.go index 8e1c1d0b..43e0a39b 100644 --- a/deb/package.go +++ b/deb/package.go @@ -9,8 +9,6 @@ import ( "strings" ) -type Hash uint64 - // Package is single instance of Debian package type Package struct { // Basic package properties @@ -29,7 +27,7 @@ type Package struct { // Is this udeb package IsUdeb bool // Hash of files section - FilesHash Hash + FilesHash uint64 // Is this >= 0.6 package? V06Plus bool // Offload fields @@ -40,10 +38,6 @@ type Package struct { collection *PackageCollection } -func (h *Hash) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("\"%08x\"", *h)), nil -} - // NewPackageFromControlFile creates Package from parsed Debian control file func NewPackageFromControlFile(input Stanza) *Package { result := &Package{ @@ -404,7 +398,7 @@ func (p *Package) Files() PackageFiles { // UpdateFiles saves new state of files func (p *Package) UpdateFiles(files PackageFiles) { p.files = &files - p.FilesHash = Hash(files.Hash()) + p.FilesHash = files.Hash() } // Stanza creates original stanza from package From 8c15a0ca95cfd2d4e02bd033773e8b7a2d3bd0da Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:24:44 +0300 Subject: [PATCH 19/26] Add AlekSi/pointer to dependencies. #168 --- Gomfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gomfile b/Gomfile index 4a4ce593..d433fd79 100644 --- a/Gomfile +++ b/Gomfile @@ -3,6 +3,7 @@ gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea' gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2' gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556' gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793' +gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943' gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827' gom 'github.com/gin-gonic/gin', :commit => '0808f8a824cfb9aef6ea4fd664af238544b66fc1' gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76' From 25d048fe49b1ef01ccec42ab99cd64103fd951f0 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:26:55 +0300 Subject: [PATCH 20/26] Add Package serialization to JSON via stanza. #168 --- deb/package.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deb/package.go b/deb/package.go index 43e0a39b..cf119de8 100644 --- a/deb/package.go +++ b/deb/package.go @@ -1,6 +1,7 @@ package deb import ( + "encoding/json" "fmt" "github.com/smira/aptly/aptly" "github.com/smira/aptly/utils" @@ -38,6 +39,11 @@ type Package struct { collection *PackageCollection } +// Check interface +var ( + _ json.Marshaler = &Package{} +) + // NewPackageFromControlFile creates Package from parsed Debian control file func NewPackageFromControlFile(input Stanza) *Package { result := &Package{ @@ -205,6 +211,11 @@ func (p *Package) String() string { return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture) } +// MarshalJSON implements json.Marshaller interface +func (p *Package) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Stanza()) +} + // GetField returns fields from package func (p *Package) GetField(name string) string { switch name { From 398303235a49e7ff5316dce461cc3b9bfa9c77d7 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:34:02 +0300 Subject: [PATCH 21/26] Custom JSON marshalling for PackageDiff, updated test for snapshot diff API. #168 --- deb/reflist.go | 23 +++++++++++++++++++++++ system/t12_api/snapshots.py | 20 ++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index db5721db..30845e47 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -2,6 +2,8 @@ package deb import ( "bytes" + "encoding/json" + "github.com/AlekSi/pointer" "github.com/ugorji/go/codec" "sort" ) @@ -154,6 +156,27 @@ type PackageDiff struct { Left, Right *Package } +// Check interface +var ( + _ json.Marshaler = PackageDiff{} +) + +// MarshalJSON implements json.Marshaler interface +func (d PackageDiff) MarshalJSON() ([]byte, error) { + serialized := struct { + Left, Right *string + }{} + + if d.Left != nil { + serialized.Left = pointer.ToString(string(d.Left.Key(""))) + } + if d.Right != nil { + serialized.Right = pointer.ToString(string(d.Right.Key(""))) + } + + return json.Marshal(serialized) +} + // PackageDiffs is a list of PackageDiff records type PackageDiffs []PackageDiff diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index f1952371..79762924 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -158,7 +158,7 @@ class SnapshotsAPITestSearch(APITest): class SnapshotsAPITestDiff(APITest): """ - POST /api/snapshot/:name/diff/:name2 + GET /api/snapshot/:name/diff/:name2 """ def check(self): repos = [self.random_name() for x in xrange(2)] @@ -180,7 +180,19 @@ class SnapshotsAPITestDiff(APITest): self.check_equal(resp.status_code, 201) resp = self.get("/api/snapshots/" + snapshots[0] + "/diff/" + snapshots[1]) - self.check_equal(resp.status_code, 200) - self.check_subset({"Right": None}, resp.json()[0]) - self.check_subset({"Name": "libboost-program-options-dev"}, resp.json()[0]["Left"]) + self.check_equal(resp.json(), [{'Left': 'Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378', + 'Right': None}]) + + resp = self.get("/api/snapshots/" + snapshots[1] + "/diff/" + snapshots[0]) + self.check_equal(resp.status_code, 200) + self.check_equal(resp.json(), [{'Right': 'Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378', + 'Left': None}]) + + resp = self.get("/api/snapshots/" + snapshots[0] + "/diff/" + snapshots[0]) + self.check_equal(resp.status_code, 200) + self.check_equal(resp.json(), []) + + resp = self.get("/api/snapshots/" + snapshots[1] + "/diff/" + snapshots[1]) + self.check_equal(resp.status_code, 200) + self.check_equal(resp.json(), []) From d20300b1523353ae9be26581432dfc8d46b78899 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:46:57 +0300 Subject: [PATCH 22/26] Whitespace fix. #168 --- system/lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/system/lib.py b/system/lib.py index dc6ebd94..19b56428 100644 --- a/system/lib.py +++ b/system/lib.py @@ -276,7 +276,6 @@ class BaseTest(object): diff += "wrong value '%s' for key '%s', expected '%s'\n" % (v, k, b[k]) if diff: raise Exception("content doesn't match:\n" + diff) - def verify_match(self, a, b, match_prepare=None): if match_prepare is not None: From fa2eef564ca191837adb190282a14034949660f0 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:47:10 +0300 Subject: [PATCH 23/26] Enhance Package JSON representation with Key, ShortKey and FilesHash. #168 --- deb/package.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deb/package.go b/deb/package.go index cf119de8..b83628d2 100644 --- a/deb/package.go +++ b/deb/package.go @@ -213,7 +213,12 @@ func (p *Package) String() string { // MarshalJSON implements json.Marshaller interface func (p *Package) MarshalJSON() ([]byte, error) { - return json.Marshal(p.Stanza()) + stanza := p.Stanza() + stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash) + stanza["Key"] = string(p.Key("")) + stanza["ShortKey"] = string(p.ShortKey("")) + + return json.Marshal(stanza) } // GetField returns fields from package From b0489117c892657fad6618688249afe5fb942f1e Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:48:14 +0300 Subject: [PATCH 24/26] Update system tests for new Package serialization. #168 --- system/t12_api/snapshots.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 79762924..dde41d75 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -71,13 +71,13 @@ class SnapshotsAPITestCreateFromRepo(APITest): self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) self.check_subset({u'Architecture': 'i386', - u'Name': 'libboost-program-options-dev', + u'Package': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, self.get("/api/snapshots/" + snapshot_name + "/packages?format=details").json()[0]) self.check_subset({u'Architecture': 'i386', - u'Name': 'libboost-program-options-dev', + u'Package': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, self.get("/api/snapshots/" + snapshot_name + "/packages?format=details", @@ -147,7 +147,7 @@ class SnapshotsAPITestSearch(APITest): self.check_equal(resp.status_code, 200) self.check_equal(len(resp.json()), 1) - self.check_equal(resp.json()[0]["Name"], "libboost-program-options-dev") + self.check_equal(resp.json()[0]["Package"], "libboost-program-options-dev") resp = self.get("/api/snapshots/" + snapshot_name + "/packages") self.check_equal(resp.status_code, 200) From cb99cbec58ed616c90a07656fd75774c69cb832b Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:53:07 +0300 Subject: [PATCH 25/26] Fix final system test. #168 --- system/t12_api/repos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system/t12_api/repos.py b/system/t12_api/repos.py index dbd94429..ba390069 100644 --- a/system/t12_api/repos.py +++ b/system/t12_api/repos.py @@ -163,7 +163,8 @@ class ReposAPITestShowQuery(APITest): self.check_equal(sorted(self.get("/api/repos/" + repo_name + "/packages", params={"q": "Version (> 0.6.1-1.4)"}).json()), ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378', 'Psource pyspi 0.6.1-1.4 f8f1daa806004e89']) - self.check_equal(sorted(self.get("/api/repos/" + repo_name + "/packages", params={"q": "pyspi", "format": "details"}).json()), + self.check_equal(sorted(p['Key'] for p in self.get("/api/repos/" + repo_name + "/packages", + params={"q": "pyspi", "format": "details"}).json()), ['Psource pyspi 0.6.1-1.3 3a8b37cbd9a3559e', 'Psource pyspi 0.6.1-1.4 f8f1daa806004e89']) resp = self.get("/api/repos/" + repo_name + "/packages", params={"q": "pyspi)"}) From 13fc1122f0d7cbacd7e52d1a31fe94d94d82dc19 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 5 Feb 2015 01:56:55 +0300 Subject: [PATCH 26/26] Use Python requests URL params instead of manual GET params. #168 --- system/t12_api/snapshots.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index dde41d75..46f0b513 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -74,14 +74,14 @@ class SnapshotsAPITestCreateFromRepo(APITest): u'Package': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, - self.get("/api/snapshots/" + snapshot_name + "/packages?format=details").json()[0]) + self.get("/api/snapshots/" + snapshot_name + "/packages", params={"format": "details"}).json()[0]) self.check_subset({u'Architecture': 'i386', u'Package': 'libboost-program-options-dev', u'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, - self.get("/api/snapshots/" + snapshot_name + "/packages?format=details", - params={"q": "Version (> 0.6.1-1.4)"}).json()[0]) + self.get("/api/snapshots/" + snapshot_name + "/packages", + params={"format": "details", "q": "Version (> 0.6.1-1.4)"}).json()[0]) class SnapshotsAPITestCreateUpdate(APITest): @@ -143,7 +143,8 @@ class SnapshotsAPITestSearch(APITest): resp = self.post("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot_name}) self.check_equal(resp.status_code, 201) - resp = self.get("/api/snapshots/" + snapshot_name + "/packages?q=libboost-program-options-dev&format=details") + resp = self.get("/api/snapshots/" + snapshot_name + "/packages", + params={"q": "libboost-program-options-dev", "format": "details"}) self.check_equal(resp.status_code, 200) self.check_equal(len(resp.json()), 1)