vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-16 07:10:54 +00:00
Code Issues Projects Releases Wiki Activity

New upstream version 1.3.0

Browse Source
This commit is contained in:
Sébastien Delafond
2018-06-21 15:14:48 +02:00
parent bed9fffa94
commit 09ad0121c6
2288 changed files with 452573 additions and 68516 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deb/changes.go
+3 -3
View File
@@ -9,9 +9,9 @@ import (
"sort"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
)
// Changes is a result of .changes file parsing
deb/checksum_collection.go
+3 -3
View File
@@ -3,9 +3,9 @@ package deb
import (
"bytes"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/utils"
"github.com/ugorji/go/codec"
)
deb/checksum_collection_test.go
+2 -2
View File
@@ -1,8 +1,8 @@
package deb
import (
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
deb/collections.go
+1 -1
View File
@@ -3,7 +3,7 @@ package deb
import (
"sync"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
)
// CollectionFactory is a single place to generate all desired collections
deb/contents.go
+2 -6
View File
@@ -6,8 +6,7 @@ import (
"fmt"
"io"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
"github.com/smira/go-uuid/uuid"
)
@@ -26,10 +25,7 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
}
// Push adds package to contents index, calculating package contents as required
func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
contents := p.Contents(packagePool, progress)
qualifiedName := []byte(p.QualifiedName())
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
for _, path := range contents {
// for performance reasons we only write to leveldb during push.
// merging of qualified names per path will be done in WriteTo
deb/deb.go
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/mkrautz/goar"
"github.com/pkg/errors"
"github.com/smira/aptly/pgp"
"github.com/aptly-dev/aptly/pgp"
"github.com/smira/go-xz"
"github.com/smira/lzma"
)
deb/deb_test.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"path/filepath"
"runtime"
"github.com/smira/aptly/pgp"
"github.com/aptly-dev/aptly/pgp"
. "gopkg.in/check.v1"
)
deb/format.go
+7 -1
View File
@@ -11,6 +11,9 @@ import (
// Stanza or paragraph of Debian control file
type Stanza map[string]string
// MaxFieldSize is maximum stanza field size in bytes
const MaxFieldSize = 2 * 1024 * 1024
// Canonical order of fields in stanza
// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
var (
@@ -214,7 +217,10 @@ type ControlFileReader struct {
// NewControlFileReader creates ControlFileReader, it wraps with buffering
func NewControlFileReader(r io.Reader) *ControlFileReader {
return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))}
scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
scnr.Buffer(nil, MaxFieldSize)
return &ControlFileReader{scanner: scnr}
}
// ReadStanza reeads one stanza from control file
deb/format_test.go
+12
View File
@@ -3,6 +3,7 @@ package deb
import (
"bufio"
"bytes"
"os"
"strings"
. "gopkg.in/check.v1"
@@ -135,6 +136,17 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
}
func (s *ControlFileSuite) TestLongFields(c *C) {
f, err := os.Open("long.stanza")
c.Assert(err, IsNil)
defer f.Close()
r := NewControlFileReader(f)
stanza, e := r.ReadStanza(false)
c.Assert(e, IsNil)
c.Assert(len(stanza["Provides"]), Equals, 586929)
}
func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
for i := 0; i < c.N; i++ {
reader := bytes.NewBufferString(controlFile)
deb/import.go
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"sort"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
)
// CollectPackageFiles walks filesystem collecting all candidates for package files
deb/index_files.go
+47 -12
View File
@@ -8,9 +8,9 @@ import (
"path/filepath"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
)
type indexFiles struct {
@@ -191,18 +191,22 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
}
// if a previous index file already exists exists, backup symlink
if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists {
indexPath := filepath.Join(dst, indexfile)
oldIndexPath := filepath.Join(dst, indexfile+".old")
if exists, _ = file.parent.publishedStorage.FileExists(indexPath); exists {
// if exists, remove old symlink
if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists {
var link string
link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
if err != nil {
file.parent.publishedStorage.Remove(link)
if exists, _ = file.parent.publishedStorage.FileExists(oldIndexPath); exists {
var linkTarget string
linkTarget, err = file.parent.publishedStorage.ReadLink(oldIndexPath)
if err == nil {
// If we managed to resolve the link target: delete it. This is the
// oldest physical index file we no longer need. Once we drop our
// old symlink we'll essentially forget about it existing at all.
file.parent.publishedStorage.Remove(linkTarget)
}
file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
file.parent.publishedStorage.Remove(oldIndexPath)
}
file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
filepath.Join(dst, indexfile+".old"))
file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
}
// create symlink
@@ -325,6 +329,37 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
return file
}
func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile {
if arch == ArchitectureSource {
udeb = false
}
key := fmt.Sprintf("lci-%s-%v", arch, udeb)
file, ok := files.indexes[key]
if !ok {
var relativePath string
if udeb {
relativePath = fmt.Sprintf("Contents-udeb-%s", arch)
} else {
relativePath = fmt.Sprintf("Contents-%s", arch)
}
file = &indexFile{
parent: files,
discardable: true,
compressable: true,
onlyGzip: true,
signable: false,
acquireByHash: files.acquireByHash,
relativePath: relativePath,
}
files.indexes[key] = file
}
return file
}
func (files *indexFiles) ReleaseFile() *indexFile {
return &indexFile{
parent: files,
deb/list.go
+2 -2
View File
@@ -5,8 +5,8 @@ import (
"sort"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
)
// Dependency options
deb/local.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"log"
"sync"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
"github.com/smira/go-uuid/uuid"
"github.com/ugorji/go/codec"
)
deb/local_test.go
+1 -1
View File
@@ -3,7 +3,7 @@ package deb
import (
"errors"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
. "gopkg.in/check.v1"
)
deb/long.stanza
+12
View File
File diff suppressed because one or more lines are too long
deb/package.go
+2 -2
View File
@@ -7,8 +7,8 @@ import (
"strconv"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
)
// Package is single instance of Debian package
deb/package_collection.go
+2 -2
View File
@@ -5,8 +5,8 @@ import (
"fmt"
"path/filepath"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/ugorji/go/codec"
)
deb/package_collection_test.go
+2 -2
View File
@@ -1,8 +1,8 @@
package deb
import (
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
deb/package_files.go
+2 -2
View File
@@ -9,8 +9,8 @@ import (
"strconv"
"strings"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
)
// PackageFile is a single file entry in package
deb/package_files_test.go
+3 -3
View File
@@ -4,9 +4,9 @@ import (
"io/ioutil"
"path/filepath"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/files"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
deb/package_test.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"path/filepath"
"regexp"
"github.com/smira/aptly/files"
"github.com/aptly-dev/aptly/files"
. "gopkg.in/check.v1"
)
deb/ppa.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"regexp"
"strings"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/utils"
)
var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")
deb/ppa_test.go
+1 -1
View File
@@ -1,7 +1,7 @@
package deb
import (
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
deb/publish.go
+44 -10
View File
@@ -17,10 +17,10 @@ import (
"github.com/smira/go-uuid/uuid"
"github.com/ugorji/go/codec"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
)
type repoSourceItem struct {
@@ -315,6 +315,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
"Sources": sources,
"Storage": p.Storage,
"SkipContents": p.SkipContents,
"AcquireByHash": p.AcquireByHash,
})
}
@@ -561,6 +562,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
legacyContentIndexes := map[string]*ContentsIndex{}
for component, list := range lists {
hadUdebs := false
@@ -598,21 +601,32 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
}
// Start a db batch. If we fill contents data we'll need
// to push each path of the package into the database.
// We'll want this batched so as to avoid an excessive
// amount of write() calls.
tempDB.StartBatch()
defer tempDB.FinishBatch()
for _, arch := range p.Architectures {
if pkg.MatchesArchitecture(arch) {
var bufWriter *bufio.Writer
if !p.SkipContents {
key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
qualifiedName := []byte(pkg.QualifiedName())
contents := pkg.Contents(packagePool, progress)
contentIndex := contentIndexes[key]
for _, contentIndexesMap := range []map[string]*ContentsIndex{contentIndexes, legacyContentIndexes} {
contentIndex := contentIndexesMap[key]
if contentIndex == nil {
contentIndex = NewContentsIndex(tempDB)
contentIndexes[key] = contentIndex
if contentIndex == nil {
contentIndex = NewContentsIndex(tempDB)
contentIndexesMap[key] = contentIndex
}
contentIndex.Push(qualifiedName, contents)
}
contentIndex.Push(pkg, packagePool, progress)
}
bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
@@ -704,6 +718,26 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
}
for _, arch := range p.Architectures {
for _, udeb := range []bool{true, false} {
index := legacyContentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
if index == nil || index.Empty() {
continue
}
var bufWriter *bufio.Writer
bufWriter, err = indexes.LegacyContentsIndex(arch, udeb).BufWriter()
if err != nil {
return fmt.Errorf("unable to generate contents index: %v", err)
}
_, err = index.WriteTo(bufWriter)
if err != nil {
return fmt.Errorf("unable to generate contents index: %v", err)
}
}
}
if progress != nil {
progress.Printf("Finalizing metadata files...\n")
}
deb/publish_test.go
+3 -3
View File
@@ -8,9 +8,9 @@ import (
"os"
"path/filepath"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/files"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/files"
"github.com/ugorji/go/codec"
. "gopkg.in/check.v1"
deb/reflist_test.go
+1 -1
View File
@@ -3,7 +3,7 @@ package deb
import (
"errors"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
. "gopkg.in/check.v1"
)
deb/remote.go
+5 -5
View File
@@ -15,11 +15,11 @@ import (
"syscall"
"time"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/http"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/http"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/smira/go-uuid/uuid"
"github.com/ugorji/go/codec"
)
deb/remote_test.go
+7 -7
View File
@@ -7,13 +7,13 @@ import (
"os"
"sort"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/console"
"github.com/smira/aptly/database"
"github.com/smira/aptly/files"
"github.com/smira/aptly/http"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/http"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
deb/snapshot.go
+2 -2
View File
@@ -10,8 +10,8 @@ import (
"sync"
"time"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/utils"
"github.com/smira/go-uuid/uuid"
"github.com/ugorji/go/codec"
)
deb/snapshot_test.go
+1 -1
View File
@@ -3,7 +3,7 @@ package deb
import (
"errors"
"github.com/smira/aptly/database"
"github.com/aptly-dev/aptly/database"
. "gopkg.in/check.v1"
)
deb/uploaders.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"os"
"github.com/DisposaBoy/JsonConfigReader"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
)
// UploadersRule is single rule of format: what packages can group or key upload
deb/uploaders_test.go
+1 -1
View File
@@ -1,7 +1,7 @@
package deb
import (
"github.com/smira/aptly/pgp"
"github.com/aptly-dev/aptly/pgp"
. "gopkg.in/check.v1"
)