Buffer increase (#738)

Increase Scanner buffer size for Stanza reader
This commit is contained in:
Strajan Sebastian Ioan
2018-05-14 17:41:33 +03:00
committed by Andrey Smirnov
parent c7a3a10846
commit d31144b9ae
4 changed files with 32 additions and 1 deletions

View File

@@ -32,3 +32,4 @@ List of contributors, in chronological order:
* Ludovico Cavedon (https://github.com/cavedon)
* Petr Jediny (https://github.com/pjediny)
* Maximilian Stein (https://github.com/steinymity)
* Strajan Sebastian (https://github.com/strajansebastian)

View File

@@ -11,6 +11,9 @@ import (
// Stanza or paragraph of Debian control file
type Stanza map[string]string
// MaxFieldSize is maximum stanza field size in bytes
const MaxFieldSize = 2 * 1024 * 1024
// Canonical order of fields in stanza
// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
var (
@@ -214,7 +217,10 @@ type ControlFileReader struct {
// NewControlFileReader creates ControlFileReader, it wraps with buffering
func NewControlFileReader(r io.Reader) *ControlFileReader {
return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))}
scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
scnr.Buffer(nil, MaxFieldSize)
return &ControlFileReader{scanner: scnr}
}
// ReadStanza reeads one stanza from control file

View File

@@ -3,6 +3,7 @@ package deb
import (
"bufio"
"bytes"
"os"
"strings"
. "gopkg.in/check.v1"
@@ -135,6 +136,17 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
}
func (s *ControlFileSuite) TestLongFields(c *C) {
f, err := os.Open("long.stanza")
c.Assert(err, IsNil)
defer f.Close()
r := NewControlFileReader(f)
stanza, e := r.ReadStanza(false)
c.Assert(e, IsNil)
c.Assert(len(stanza["Provides"]), Equals, 586929)
}
func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
for i := 0; i < c.N; i++ {
reader := bytes.NewBufferString(controlFile)

12
deb/long.stanza Normal file

File diff suppressed because one or more lines are too long