diff --git a/Makefile b/Makefile index 4be0b733..9e7e1d24 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GOVERSION=$(shell go version | awk '{print $$3;}') -PACKAGES=database deb files http utils -ALL_PACKAGES=aptly cmd console database deb files http utils +PACKAGES=database deb files http query utils +ALL_PACKAGES=aptly cmd console database deb files http query utils BINPATH=$(abspath ./_vendor/bin) GOM_ENVIRONMENT=-test PYTHON?=python diff --git a/query/lex.go b/query/lex.go new file mode 100644 index 00000000..dc95d8d9 --- /dev/null +++ b/query/lex.go @@ -0,0 +1,211 @@ +package query + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +// itemType identifies the type of lex items. +type itemType int + +const eof = -1 + +const ( + itemError itemType = iota // error occurred; + // value is text of error + itemEOF + itemLeftParen // ( + itemRightParen // ) + itemOr // | + itemAnd // , + itemNot // ! + itemLt // << + itemLtEq // <=, < + itemGt // >> + itemGtEq // >=, > + itemEq // = + itemPatMatch // % + itemRegexp // ~ + itemString +) + +// item represents a token returned from the scanner. +type item struct { + typ itemType // Type, such as itemNumber. + val string // Value, such as "23.2". +} + +func (i item) String() string { + if i.typ == itemString { + return fmt.Sprintf("%#v", i.val) + } + if i.typ == itemEOF { + return "" + } + return i.val +} + +// stateFn represents the state of the scanner +// as a function that returns the next state. +type stateFn func(*lexer) stateFn + +// lexer holds the state of the scanner. +type lexer struct { + name string // used only for error reports. + input string // the string being scanned. + start int // start position of this item. + pos int // current position in the input. + width int // width of last rune read from input. + items chan item // channel of scanned items. + last item +} + +func lex(name, input string) (*lexer, chan item) { + l := &lexer{ + name: name, + input: input, + items: make(chan item), + } + go l.run() // Concurrently run state machine. + return l, l.items +} + +// emit passes an item back to the client. +func (l *lexer) emit(t itemType) { + l.items <- item{t, l.input[l.start:l.pos]} + l.start = l.pos +} + +// run lexes the input by executing state functions until +// the state is nil. +func (l *lexer) run() { + for state := lexMain; state != nil; { + state = state(l) + } + close(l.items) // No more tokens will be delivered. +} + +// next returns the next rune in the input. +func (l *lexer) next() (r rune) { + if l.pos >= len(l.input) { + l.width = 0 + return eof + } + r, l.width = + utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += l.width + return r +} + +// ignore skips over the pending input before this point. +func (l *lexer) ignore() { + l.start = l.pos +} + +// backup steps back one rune. +// Can be called only once per call of next. +func (l *lexer) backup() { + l.pos -= l.width +} + +// peek returns but does not consume +// the next rune in the input. +func (l *lexer) peek() rune { + r := l.next() + l.backup() + return r +} + +func (l *lexer) Current() item { + if l.last.typ == 0 { + l.last = <-l.items + } + + return l.last +} + +func (l *lexer) Consume() { + l.last = <-l.items +} + +// error returns an error token and terminates the scan +// by passing back a nil pointer that will be the next +// state, terminating l.run. +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{ + itemError, + fmt.Sprintf(format, args...), + } + return nil +} + +func lexMain(l *lexer) stateFn { + switch r := l.next(); { + case r == eof: + l.emit(itemEOF) + return nil + case unicode.IsSpace(r): + l.ignore() + case r == '(': + l.emit(itemLeftParen) + case r == ')': + l.emit(itemRightParen) + case r == '|': + l.emit(itemOr) + case r == ',': + l.emit(itemAnd) + case r == '!': + l.emit(itemNot) + case r == '<': + r2 := l.next() + if r2 == '<' { + l.emit(itemLt) + } else if r2 == '=' { + l.emit(itemLtEq) + } else { + l.backup() + l.emit(itemLtEq) + } + case r == '>': + r2 := l.next() + if r2 == '>' { + l.emit(itemGt) + } else if r2 == '=' { + l.emit(itemGtEq) + } else { + l.backup() + l.emit(itemGtEq) + } + case r == '=': + l.emit(itemEq) + case r == '%': + l.emit(itemPatMatch) + case r == '~': + l.emit(itemRegexp) + default: + l.backup() + return lexString + } + + return lexMain +} + +func lexString(l *lexer) stateFn { + for { + r := l.next() + if unicode.IsSpace(r) || strings.IndexRune("()|,!<>=%~", r) > 0 { + l.backup() + l.emit(itemString) + return lexMain(l) + } + + if r == eof { + l.emit(itemString) + l.emit(itemEOF) + return nil + } + } + +} diff --git a/query/lex_test.go b/query/lex_test.go new file mode 100644 index 00000000..6a492553 --- /dev/null +++ b/query/lex_test.go @@ -0,0 +1,46 @@ +package query + +import ( + "fmt" + . "launchpad.net/gocheck" +) + +type LexerSuite struct { +} + +var _ = Suite(&LexerSuite{}) + +func (s *LexerSuite) TestLexing(c *C) { + _, ch := lex("query", "package (<< 1.3), $Source | !app") + + c.Check(<-ch, Equals, item{typ: itemString, val: "package"}) + c.Check(<-ch, Equals, item{typ: itemLeftParen, val: "("}) + c.Check(<-ch, Equals, item{typ: itemLt, val: "<<"}) + c.Check(<-ch, Equals, item{typ: itemString, val: "1.3"}) + c.Check(<-ch, Equals, item{typ: itemRightParen, val: ")"}) + c.Check(<-ch, Equals, item{typ: itemAnd, val: ","}) + c.Check(<-ch, Equals, item{typ: itemString, val: "$Source"}) + c.Check(<-ch, Equals, item{typ: itemOr, val: "|"}) + c.Check(<-ch, Equals, item{typ: itemNot, val: "!"}) + c.Check(<-ch, Equals, item{typ: itemString, val: "app"}) + c.Check(<-ch, Equals, item{typ: itemEOF, val: ""}) +} + +func (s *LexerSuite) TestConsume(c *C) { + l, _ := lex("query", "package (<< 1.3)") + + c.Check(l.Current(), Equals, item{typ: itemString, val: "package"}) + c.Check(l.Current(), Equals, item{typ: itemString, val: "package"}) + l.Consume() + c.Check(l.Current(), Equals, item{typ: itemLeftParen, val: "("}) + l.Consume() + c.Check(l.Current(), Equals, item{typ: itemLt, val: "<<"}) +} + +func (s *LexerSuite) TestString(c *C) { + l, _ := lex("query", "package (<< 1.3)") + + c.Check(fmt.Sprintf("%s", l.Current()), Equals, "\"package\"") + l.Consume() + c.Check(fmt.Sprintf("%s", l.Current()), Equals, "(") +} diff --git a/query/query.go b/query/query.go new file mode 100644 index 00000000..03e9c4a9 --- /dev/null +++ b/query/query.go @@ -0,0 +1,17 @@ +// Package query implements query language for +package query + +/* + + Query language resembling Debian dependencies and reprepro + queries: http://mirrorer.alioth.debian.org/reprepro.1.html + + Query := A | A '|' Query + A := B | B ',' A + B := C | '!' B + C := '(' Query ')' | D + D := + field := | | $special_field + condition := '(' value ')' | + operator := | << | < | <= | > | >> | >= | = | % | ~ +*/ diff --git a/query/query_test.go b/query/query_test.go new file mode 100644 index 00000000..84989efd --- /dev/null +++ b/query/query_test.go @@ -0,0 +1,11 @@ +package query + +import ( + . "launchpad.net/gocheck" + "testing" +) + +// Launch gocheck tests +func Test(t *testing.T) { + TestingT(t) +} diff --git a/query/syntax.go b/query/syntax.go new file mode 100644 index 00000000..bff3017f --- /dev/null +++ b/query/syntax.go @@ -0,0 +1,164 @@ +package query + +import ( + "fmt" + "github.com/smira/aptly/deb" + "strings" + "unicode" + "unicode/utf8" +) + +type parser struct { + name string // used only for error reports. + input *lexer // the input lexer + err error // error stored while parsing +} + +func parse(input *lexer) (PackageQuery, error) { + p := &parser{ + name: input.name, + input: input, + } + query := p.parse() + if p.err != nil { + return nil, p.err + } + return query, nil +} + +// Entry into parser +func (p *parser) parse() PackageQuery { + defer func() { + if r := recover(); r != nil { + p.err = fmt.Errorf("parsing failed: %s", r) + } + }() + + q := p.Query() + if p.input.Current().typ != itemEOF { + panic(fmt.Sprintf("unexpected token %s: expecting end of query", p.input.Current())) + } + return q +} + +// Query := A | A '|' Query +func (p *parser) Query() PackageQuery { + q := p.A() + if p.input.Current().typ == itemOr { + p.input.Consume() + return &OrQuery{L: q, R: p.Query()} + } + return q +} + +// A := B | B ',' A +func (p *parser) A() PackageQuery { + q := p.B() + if p.input.Current().typ == itemAnd { + p.input.Consume() + return &AndQuery{L: q, R: p.A()} + } + return q +} + +// B := C | '!' B +func (p *parser) B() PackageQuery { + if p.input.Current().typ == itemNot { + p.input.Consume() + return &NotQuery{Q: p.B()} + } + return p.C() +} + +// C := '(' Query ')' | D +func (p *parser) C() PackageQuery { + if p.input.Current().typ == itemLeftParen { + p.input.Consume() + q := p.Query() + if p.input.Current().typ != itemRightParen { + panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current())) + } + p.input.Consume() + return q + } + return p.D() +} + +func operatorToRelation(operator itemType) int { + switch operator { + case 0: + return deb.VersionDontCare + case itemLt: + return deb.VersionLess + case itemLtEq: + return deb.VersionLessOrEqual + case itemGt: + return deb.VersionGreater + case itemGtEq: + return deb.VersionGreaterOrEqual + case itemEq: + return deb.VersionEqual + case itemPatMatch: + return deb.VersionPatternMatch + case itemRegexp: + return deb.VersionRegexp + } + panic("unable to map token to relation") +} + +// D := +// field := | | $special_field +func (p *parser) D() PackageQuery { + if p.input.Current().typ != itemString { + panic(fmt.Sprintf("unexpected token %s: expecting field or package name", p.input.Current())) + } + + field := p.input.Current().val + p.input.Consume() + + operator, value := p.Condition() + + r, _ := utf8.DecodeRuneInString(field) + if strings.HasPrefix(field, "$") || unicode.IsUpper(r) { + // special field or regular field + return &FieldQuery{Field: field, Relation: operatorToRelation(operator), Value: value} + } + + // regular dependency-like query + return &DependencyQuery{Dep: deb.Dependency{Pkg: field, Relation: operatorToRelation(operator), Version: value}} +} + +// condition := '(' value ')' | +// operator := | << | < | <= | > | >> | >= | = | % | ~ +func (p *parser) Condition() (operator itemType, value string) { + if p.input.Current().typ != itemLeftParen { + return + } + p.input.Consume() + + if p.input.Current().typ == itemLt || + p.input.Current().typ == itemGt || + p.input.Current().typ == itemLtEq || + p.input.Current().typ == itemGtEq || + p.input.Current().typ == itemEq || + p.input.Current().typ == itemPatMatch || + p.input.Current().typ == itemRegexp { + operator = p.input.Current().typ + p.input.Consume() + } else { + operator = itemEq + } + + if p.input.Current().typ != itemString { + panic(fmt.Sprintf("unexpected token %s: expecting value", p.input.Current())) + } + value = p.input.Current().val + p.input.Consume() + + if p.input.Current().typ != itemRightParen { + panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current())) + } + p.input.Consume() + + return +} diff --git a/query/syntax_test.go b/query/syntax_test.go new file mode 100644 index 00000000..1711526d --- /dev/null +++ b/query/syntax_test.go @@ -0,0 +1,64 @@ +package query + +import ( + "github.com/smira/aptly/deb" + . "launchpad.net/gocheck" +) + +type SyntaxSuite struct { +} + +var _ = Suite(&SyntaxSuite{}) + +func (s *SyntaxSuite) TestParsing(c *C) { + l, _ := lex("query", "package (<< 1.3), $Source") + q, err := parse(l) + + c.Assert(err, IsNil) + c.Check(q.(*AndQuery).L, DeepEquals, &DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionLess, Version: "1.3"}}) + c.Check(q.(*AndQuery).R, DeepEquals, &FieldQuery{Field: "$Source"}) + + l, _ = lex("query", "package (1.3), Name (lala) | !$Source") + q, err = parse(l) + + c.Assert(err, IsNil) + c.Check(q.(*OrQuery).L.(*AndQuery).L, DeepEquals, &DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionEqual, Version: "1.3"}}) + c.Check(q.(*OrQuery).L.(*AndQuery).R, DeepEquals, &FieldQuery{Field: "Name", Relation: deb.VersionEqual, Value: "lala"}) + c.Check(q.(*OrQuery).R.(*NotQuery).Q, DeepEquals, &FieldQuery{Field: "$Source"}) + + l, _ = lex("query", "package, ((!(Name | $Source (~ a.*))))") + q, err = parse(l) + + c.Assert(err, IsNil) + c.Check(q.(*AndQuery).L, DeepEquals, &DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionDontCare}}) + c.Check(q.(*AndQuery).R.(*NotQuery).Q.(*OrQuery).L, DeepEquals, &FieldQuery{Field: "Name", Relation: deb.VersionDontCare}) + c.Check(q.(*AndQuery).R.(*NotQuery).Q.(*OrQuery).R, DeepEquals, &FieldQuery{Field: "$Source", Relation: deb.VersionRegexp, Value: "a.*"}) + + l, _ = lex("query", "package (> 5.3.7)") + q, err = parse(l) + + c.Assert(err, IsNil) + c.Check(q, DeepEquals, &DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionGreaterOrEqual, Version: "5.3.7"}}) +} + +func (s *SyntaxSuite) TestParsingErrors(c *C) { + l, _ := lex("query", "package (> 5.3.7), ") + _, err := parse(l) + c.Check(err, ErrorMatches, "parsing failed: unexpected token : expecting field or package name") + + l, _ = lex("query", "package>5.3.7)") + _, err = parse(l) + c.Check(err, ErrorMatches, "parsing failed: unexpected token >: expecting end of query") + + l, _ = lex("query", "package | !|") + _, err = parse(l) + c.Check(err, ErrorMatches, "parsing failed: unexpected token |: expecting field or package name") + + l, _ = lex("query", "((package )") + _, err = parse(l) + c.Check(err, ErrorMatches, "parsing failed: unexpected token : expecting '\\)'") + + l, _ = lex("query", "!package )") + _, err = parse(l) + c.Check(err, ErrorMatches, "parsing failed: unexpected token \\): expecting end of query") +} diff --git a/query/tree.go b/query/tree.go new file mode 100644 index 00000000..6a507ac6 --- /dev/null +++ b/query/tree.go @@ -0,0 +1,62 @@ +package query + +import ( + "github.com/smira/aptly/deb" +) + +// PackageQuery is interface of predicate on Package +type PackageQuery interface { + Matches(pkg *deb.Package) bool +} + +// OrQuery is L | R +type OrQuery struct { + L, R PackageQuery +} + +// AndQuery is L , R +type AndQuery struct { + L, R PackageQuery +} + +// NotQuery is ! Q +type NotQuery struct { + Q PackageQuery +} + +// FieldQuery is generic request against field +type FieldQuery struct { + Field string + Relation int + Value string +} + +// DependencyQuery is generic Debian-dependency like query +type DependencyQuery struct { + Dep deb.Dependency +} + +// Matches if any of L, R matches +func (q *OrQuery) Matches(pkg *deb.Package) bool { + return q.L.Matches(pkg) || q.R.Matches(pkg) +} + +// Matches if both of L, R matches +func (q *AndQuery) Matches(pkg *deb.Package) bool { + return q.L.Matches(pkg) && q.R.Matches(pkg) +} + +// Matches if not matches +func (q *NotQuery) Matches(pkg *deb.Package) bool { + return !q.Q.Matches(pkg) +} + +// Matches on generic field +func (q *FieldQuery) Matches(pkg *deb.Package) bool { + panic("not implemented yet") +} + +// Matches on dependency condition +func (q *DependencyQuery) Matches(pkg *deb.Package) bool { + return pkg.MatchesDependency(q.Dep) +}