Buffer increase (#738)

Increase Scanner buffer size for Stanza reader

deb/format.go

@@ -11,6 +11,9 @@ import (
 // Stanza or paragraph of Debian control file
 type Stanza map[string]string
 
+// MaxFieldSize is maximum stanza field size in bytes
+const MaxFieldSize = 2 * 1024 * 1024
+
 // Canonical order of fields in stanza
 // Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
 var (
@@ -214,7 +217,10 @@ type ControlFileReader struct {
 
 // NewControlFileReader creates ControlFileReader, it wraps with buffering
 func NewControlFileReader(r io.Reader) *ControlFileReader {
-	return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))}
+	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
+	scnr.Buffer(nil, MaxFieldSize)
+
+	return &ControlFileReader{scanner: scnr}
 }
 
 // ReadStanza reeads one stanza from control file

deb/format_test.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bufio"
 	"bytes"
+	"os"
 	"strings"
 
 	. "gopkg.in/check.v1"
@@ -135,6 +136,17 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
 }
 
+func (s *ControlFileSuite) TestLongFields(c *C) {
+	f, err := os.Open("long.stanza")
+	c.Assert(err, IsNil)
+	defer f.Close()
+
+	r := NewControlFileReader(f)
+	stanza, e := r.ReadStanza(false)
+	c.Assert(e, IsNil)
+	c.Assert(len(stanza["Provides"]), Equals, 586929)
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)

deb/long.stanza

@@ -0,0 +1,12 @@
+Package: project-pshark-core
+Version: 20180221093552.master.31394
+Architecture: amd64
+Maintainer: PShark Build System <build.system@pshark.com>
+Installed-Size: 344993
+Depends: ps-perl, config-pshark-core, lib-ps-perl-accessors, lib-ps-perl-accessors-classic, lib-ps-perl-algorithm-diff, lib-ps-perl-amazon-s3, lib-ps-perl-anyevent, lib-ps-perl-anyevent-handle, lib-ps-perl-anyevent-socket, lib-ps-perl-apr-request-param, lib-ps-perl-archive-tar, lib-ps-perl-archive-zip, lib-ps-perl-array-utils, lib-ps-perl-beanstalk-client, lib-ps-perl-bit-vector, lib-ps-perl-cache-cache, lib-ps-perl-cache-filecache, lib-ps-perl-cache-memcached, lib-ps-perl-cache-memcached-fast, lib-ps-perl-cache-memorycache, lib-ps-perl-calendar-simple, lib-ps-perl-cam-pdf, lib-ps-perl-carp, lib-ps-perl-cgi, lib-ps-perl-cgi-carp, lib-ps-perl-charnames, lib-ps-perl-chi-memoize, lib-ps-perl-class-accessor, lib-ps-perl-class-data-inheritable, lib-ps-perl-class-dbi-abstractsearch, lib-ps-perl-class-dbi-mysql, lib-ps-perl-class-dbi-oracle, lib-ps-perl-class-dbi-pager, lib-ps-perl-class-dbi-plugin-dateformat-oracle, lib-ps-perl-class-inspector, lib-ps-perl-class-load, lib-ps-perl-clone, lib-ps-perl-compress-zlib, lib-ps-perl-config-apacheformat, lib-ps-perl-config-json, lib-ps-perl-constant, lib-ps-perl-crypt-cbc, lib-ps-perl-crypt-eksblowfish-bcrypt, lib-ps-perl-crypt-rsa, lib-ps-perl-crypt-rsa-dataformat, lib-ps-perl-crypt-rsa-es-pkcs1v15, lib-ps-perl-crypt-scryptkdf, lib-ps-perl-crypt-ssleay, lib-ps-perl-css-inliner, lib-ps-perl-cwd, lib-ps-perl-data-dump, lib-ps-perl-data-dumper, lib-ps-perl-date-calc, lib-ps-perl-date-calc-iterator, lib-ps-perl-date-calc-object, lib-ps-perl-date-calendar, lib-ps-perl-date-calendar-profiles, lib-ps-perl-date-format, lib-ps-perl-date-parse, lib-ps-perl-date-simple, lib-ps-perl-datetime, lib-ps-perl-datetime-duration, lib-ps-perl-datetime-event-cron, lib-ps-perl-datetime-format-dbi, lib-ps-perl-datetime-format-excel, lib-ps-perl-datetime-format-iso8601, lib-ps-perl-datetime-format-mysql, lib-ps-perl-datetime-format-natural, lib-ps-perl-datetime-format-strptime, lib-ps-perl-datetime-locale, lib-ps-perl-datetime-timezone, lib-ps-perl-dbd-oracle, lib-ps-perl-dbd-sybase, lib-ps-perl-dbi, lib-ps-perl-devel-cover-db, lib-ps-perl-devel-size, lib-ps-perl-devel-symdump, lib-ps-perl-diagnostics, lib-ps-perl-digest-crc, lib-ps-perl-digest-hmac--sha1, lib-ps-perl-digest-md5, lib-ps-perl-digest-sha1, lib-ps-perl-email-valid, lib-ps-perl-encode, lib-ps-perl-excel-writer-xlsx, lib-ps-perl-exception-class, lib-ps-perl-expect, lib-ps-perl-exporter, lib-ps-perl-fcntl, lib-ps-perl-feature, lib-ps-perl-file-basename, lib-ps-perl-file-copy, lib-ps-perl-file-copy-recursive, lib-ps-perl-file-fetch, lib-ps-perl-file-find, lib-ps-perl-file-find-rule, lib-ps-perl-file-glob, lib-ps-perl-filehandle, lib-ps-perl-file-homedir, lib-ps-perl-file-mimeinfo, lib-ps-perl-file-mmagic, lib-ps-perl-file-ncopy, lib-ps-perl-file-path, lib-ps-perl-file-pathinfo, lib-ps-perl-file-rsync, lib-ps-perl-file-sharedir, lib-ps-perl-file-slurp, lib-ps-perl-file-spec, lib-ps-perl-file-spec-functions, lib-ps-perl-file-stat, lib-ps-perl-file-temp, lib-ps-perl-file-type, lib-ps-perl-gd, lib-ps-perl-gd-graph, lib-ps-perl-gd-graph-bars, lib-ps-perl-gd-graph-lines, lib-ps-perl-gd-image, lib-ps-perl-gd-securityimage, lib-ps-perl-geo-coder-us, lib-ps-perl-geo-googlemaps-mobiletool, lib-ps-perl-geo-proj4, lib-ps-perl-getopt-attribute, lib-ps-perl-getopt-long, lib-ps-perl-getopt-std, lib-ps-perl-hashids, lib-ps-perl-html-entities, lib-ps-perl-html-fillinform, lib-ps-perl-html-form, lib-ps-perl-html-htmldoc, lib-ps-perl-html-mason, lib-ps-perl-html-mason-apachehandler, lib-ps-perl-html-mason-cache-basecache, lib-ps-perl-html-mason-compiler-toobject, lib-ps-perl-html-mason-interp, lib-ps-perl-html-mason-plugin, lib-ps-perl-html-mason-request, lib-ps-perl-html-mason-utils, lib-ps-perl-html-parser, lib-ps-perl-html-pullparser, lib-ps-perl-html-tablecontentparser, lib-ps-perl-html-tableextract, lib-ps-perl-html-tokeparser, lib-ps-perl-html-treebuilder, lib-ps-perl-http-cookies, lib-ps-perl-http-date, lib-ps-perl-http-headers, lib-ps-perl-http-request, lib-ps-perl-http-request-common, lib-ps-perl-http-status, lib-ps-perl-if, lib-ps-perl-image-exiftool, lib-ps-perl-image-info, lib-ps-perl-image-magick, lib-ps-perl-imager, lib-ps-perl-imager-fill, lib-ps-perl-imager-font, lib-ps-perl-io-captureoutput, lib-ps-perl-io-file, lib-ps-perl-io-handle, lib-ps-perl-io-socket, lib-ps-perl-io-socket-inet, lib-ps-perl-io-socket-ssl, lib-ps-perl-io-tee, lib-ps-perl-io-uncompress-gunzip, lib-ps-perl-ipc-cmd, lib-ps-perl-json, lib-ps-perl-json-rpc-client, lib-ps-perl-json-rpc-procedure, lib-ps-perl-json-xs, lib-ps-perl-lingua-en-inflect, lib-ps-perl-list-moreutils, lib-ps-perl-list-util, lib-ps-perl-log-log4perl, lib-ps-perl-lucy-search-termquery, lib-ps-perl-lwp, lib-ps-perl-lwp-parallel-useragent, lib-ps-perl-lwp-protocol-https, lib-ps-perl-lwp-simple, lib-ps-perl-lwp-useragent, lib-ps-perl-mail-mailer, lib-ps-perl-mail-sender, lib-ps-perl-mapscript, lib-ps-perl-match-pidqueries, lib-ps-perl-math-bigfloat, lib-ps-perl-math-bigint, lib-ps-perl-math-polygon, lLine truncated
+Suggests: project-pshark-mapgen, project-pshark-img, ps-wkhtmltopdf
+Provides: lib-ps-perl-accessors-list, lib-ps-perl-agedist, lib-ps-perl-apachedir, lib-ps-perl-db, lib-ps-perl-dissecthtml, lib-ps-perl-fastdissect, lib-ps-perl-fastdissect2, lib-ps-perl-g, lib-ps-perl-gis-geometry, lib-ps-perl-gis-geometry-line, lib-ps-perl-gis-geometry-linestring, lib-ps-perl-gis-geometry-point, lib-ps-perl-gis-geometry-polygon, lib-ps-perl-gis-geometry-vector, lib-ps-perl-householddist, lib-ps-perl-housingprofile, lib-ps-perl-listing, lib-ps-perl-logger, lib-ps-perl-magicparser, lib-ps-perl-moose-meta-attribute-custom-trait-formelement, lib-ps-perl-mycgi, lib-ps-perl-ny--swis--codes, lib-ps-perl-nyc--report--data, lib-ps-perl-permit, lib-ps-perl-project, lib-ps-perl-pshak-access-temp-philly--brt--file, lib-ps-perl-pshark (= 1.0), lib-ps-perl-pshark-abtests, lib-ps-perl-pshark-abvariants, lib-ps-perl-pshark-access, lib-ps-perl-pshark-access-ak-anchorage--commercial, lib-ps-perl-pshark-access-ak-anchorage--commercial--intr, lib-ps-perl-pshark-access-ak-anchorage--master, lib-ps-perl-pshark-access-ak-anchorage--residential, lib-ps-perl-pshark-access-ak-anchorage--residential--accessor, lib-ps-perl-pshark-access-ak-anchorage--sales, lib-ps-perl-pshark-access-al-baldwin--building, lib-ps-perl-pshark-access-al-baldwin--deeds, lib-ps-perl-pshark-access-al-baldwin--land, lib-ps-perl-pshark-access-al-baldwin--tar, lib-ps-perl-pshark-access-al-bibb--tar, lib-ps-perl-pshark-access-al-blount--improvements, lib-ps-perl-pshark-access-al-blount--tar, lib-ps-perl-pshark-access-al-chilton--sales, lib-ps-perl-pshark-access-al-chilton--tar, lib-ps-perl-pshark-access-al-mobile--sales--history, lib-ps-perl-pshark-access-al-mobile--tar, lib-ps-perl-pshark-access-al-mobile--tar--old, lib-ps-perl-pshark-access-al-shelby--sales, lib-ps-perl-pshark-access-al-shelby--tar, lib-ps-perl-pshark-access-al-stclair--bldg, lib-ps-perl-pshark-access-al-stclair--sales, lib-ps-perl-pshark-access-al-stclair--tar, lib-ps-perl-pshark-access-al-tuscaloosa--sales, lib-ps-perl-pshark-access-al-tuscaloosa--tar, lib-ps-perl-pshark-access-ar-crittenden--tar, lib-ps-perl-pshark-access-ar-faulkner--tar, lib-ps-perl-pshark-access-ar-lonoke--tar, lib-ps-perl-pshark-access-ar-pulaski--tar, lib-ps-perl-pshark-access-ar-saline--tar, lib-ps-perl-pshark-access-assmthisttemp-ada--parcels, lib-ps-perl-pshark-access-assmthisttemp-ada--parcels2, lib-ps-perl-pshark-access-assmthisttemp-anne--arundel--tar, lib-ps-perl-pshark-access-assmthisttemp-anne--arundel--tar2, lib-ps-perl-pshark-access-assmthisttemp-anoka--tar, lib-ps-perl-pshark-access-assmthisttemp-ashtabula--pardat5, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--13, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--14, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--2010, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--2011, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--2012, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--2015, lib-ps-perl-pshark-access-assmthisttemp-az--maricopa--parcel--2016, lib-ps-perl-pshark-access-assmthisttemp-baltimore--tar, lib-ps-perl-pshark-access-assmthisttemp-bastrop--tar, lib-ps-perl-pshark-access-assmthisttemp-boulder--values2, lib-ps-perl-pshark-access-assmthisttemp-bracken--prop--2013, lib-ps-perl-pshark-access-assmthisttemp-brazoria--prop--13, lib-ps-perl-pshark-access-assmthisttemp-buncombe--tar, lib-ps-perl-pshark-access-assmthisttemp-butler--pardat2, lib-ps-perl-pshark-access-assmthisttemp-ca--alameda--tar--2010, lib-ps-perl-pshark-access-assmthisttemp-ca--alameda--tar--2015, lib-ps-perl-pshark-access-assmthisttemp-ca--alameda--tar--2016, lib-ps-perl-pshark-access-assmthisttemp-ca--contra--costa--tar--2015, lib-ps-perl-pshark-access-assmthisttemp-ca--fresno--sec--asmt--roll--2015, lib-ps-perl-pshark-access-assmthisttemp-ca--fresno--sec--asmt--roll--2016, lib-ps-perl-pshark-access-assmthisttemp-ca--fresno--secureassesroll2014, lib-ps-perl-pshark-access-assmthisttemp-ca--kern--tar--14--15, lib-ps-perl-pshark-access-assmthisttemp-ca--kern--tar--15--16, lib-ps-perl-pshark-access-assmthisttemp-ca--kern--tar--16--17, lib-ps-perl-pshark-access-assmthisttemp-ca--marin--tar--12, lib-ps-perl-pshark-access-assmthisttemp-ca--marin--tar--14, lib-ps-perl-pshark-access-assmthisttemp-ca--marin--tar--15, lib-ps-perl-pshark-access-assmthisttemp-ca--marin--tar--16, lib-ps-perl-pshark-access-assmthisttemp-ca--mendocino--tar2, lib-ps-perl-pshark-access-assmthisttemp-ca--monterey--tar5, lib-ps-perl-pshark-access-assmthisttemp-ca--orange--tar--14, lib-ps-perl-pshark-access-assmthisttemp-ca--orange--tar--2015, lib-ps-perl-pshark-access-assmthisttemp-ca--orange--tar--2016, lib-ps-perl-pshark-access-assmthisttemp-ca--placer--tar--14--15, lib-ps-perl-pshark-access-assmthisttemp-ca--placer--tar--2015, lib-ps-perl-pshark-access-assmthisttemp-ca--placer--tar--2016, lib-ps-perl-pshark-access-assmthisttemp-ca--placer--tar5, lib-ps-perl-pshark-access-assmthisttemp-ca--riverside--tar--14, lib-ps-perlLine truncated
+Section: web
+Priority: important
+Multi-Arch: no
+Description: ps core package