forked from flashcat/categraf
673 lines
11 KiB
Go
673 lines
11 KiB
Go
// Copyright 2011 Google Inc. All Rights Reserved.
|
|
// This file is available under the Apache license.
|
|
|
|
package parser
|
|
|
|
import (
|
|
"flag"
|
|
"strings"
|
|
"testing"
|
|
|
|
"flashcat.cloud/categraf/inputs/mtail/internal/runtime/compiler/ast"
|
|
"flashcat.cloud/categraf/inputs/mtail/internal/runtime/compiler/position"
|
|
"flashcat.cloud/categraf/inputs/mtail/internal/testutil"
|
|
)
|
|
|
|
var parserTestDebug = flag.Bool("parser_test_debug", false, "Turn on parser debug output if set.")
|
|
|
|
var parserTests = []struct {
|
|
name string
|
|
program string
|
|
}{
|
|
{
|
|
"empty",
|
|
"",
|
|
},
|
|
|
|
{
|
|
"newline",
|
|
"\n",
|
|
},
|
|
|
|
{
|
|
"declare counter",
|
|
"counter lines_total\n",
|
|
},
|
|
|
|
{
|
|
"declare counter string name",
|
|
"counter lines_total as \"line-count\"\n",
|
|
},
|
|
|
|
{
|
|
"declare dimensioned counter",
|
|
"counter foo by bar\n",
|
|
},
|
|
|
|
{
|
|
"declare dimensioned metric with limit",
|
|
"counter foo by a, b limit 100",
|
|
},
|
|
|
|
{
|
|
"declare multi-dimensioned counter",
|
|
"counter foo by bar, baz, quux\n",
|
|
},
|
|
|
|
{
|
|
"declare hidden counter",
|
|
"hidden counter foo\n",
|
|
},
|
|
|
|
{
|
|
"declare gauge",
|
|
"gauge foo\n",
|
|
},
|
|
|
|
{
|
|
"declare timer",
|
|
"timer foo\n",
|
|
},
|
|
|
|
{
|
|
"declare text",
|
|
"text stringy\n",
|
|
},
|
|
|
|
{
|
|
"declare histogram",
|
|
"histogram foo buckets 0, 1, 2\n",
|
|
},
|
|
{
|
|
"declare histogram float",
|
|
"histogram foo buckets 0, 0.01, 0.1, 1, 10\n",
|
|
},
|
|
{
|
|
"declare histogram by ",
|
|
"histogram foo by code buckets 0, 1, 2\n",
|
|
},
|
|
{
|
|
"declare histogram reversed syntax ",
|
|
"histogram foo buckets 0, 1, 2 by code\n",
|
|
},
|
|
|
|
{
|
|
"simple pattern action",
|
|
"/foo/ {}\n",
|
|
},
|
|
|
|
{
|
|
"increment counter",
|
|
"counter lines_total\n" +
|
|
"/foo/ {\n" +
|
|
" lines_total++\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"decrement counter",
|
|
`counter i
|
|
/foo/ {
|
|
i--
|
|
}
|
|
`,
|
|
},
|
|
|
|
{
|
|
"regex match includes escaped slashes",
|
|
"counter foo\n" +
|
|
"/foo\\// { foo++\n}\n",
|
|
},
|
|
|
|
{
|
|
"numeric capture group reference",
|
|
"/(foo)/ {\n" +
|
|
" $1++\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"strptime and capref",
|
|
"/(.*)/ {\n" +
|
|
"strptime($1, \"2006-01-02T15:04:05Z07:00\")\n" +
|
|
" }\n",
|
|
},
|
|
|
|
{
|
|
"named capture group reference",
|
|
"/(?P<date>[[:digit:]-\\/ ])/ {\n" +
|
|
" strptime($date, \"%Y/%m/%d %H:%M:%S\")\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"nested match conditions",
|
|
"counter foo\n" +
|
|
"counter bar\n" +
|
|
"/match(\\d+)/ {\n" +
|
|
" foo += $1\n" +
|
|
" /^bleh (\\S+)/ {\n" +
|
|
" bar++\n" +
|
|
" $1++\n" +
|
|
" }\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"nested scope",
|
|
"counter foo\n" +
|
|
"/fo(o)/ {\n" +
|
|
" $1++\n" +
|
|
" /bar(xxx)/ {\n" +
|
|
" $1 += $1\n" +
|
|
" foo = $1\n" +
|
|
" }\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"comment then code",
|
|
"# %d [%p]\n" +
|
|
"/^(?P<date>\\d+\\/\\d+\\/\\d+ \\d+:\\d+:\\d+) \\[(?P<pid>\\d+)\\] / {\n" +
|
|
" strptime($1, \"2006/01/02 15:04:05\")\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"assignment",
|
|
"counter variable\n" +
|
|
"/(?P<foo>.*)/ {\n" +
|
|
"variable = $foo\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"increment operator",
|
|
"counter var\n" +
|
|
"/foo/ {\n" +
|
|
" var++\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"incby operator",
|
|
"counter var\n" +
|
|
"/foo/ {\n var += 2\n}\n",
|
|
},
|
|
|
|
{
|
|
"additive",
|
|
"counter time_total\n" +
|
|
"/(?P<foo>.*)/ {\n" +
|
|
" time_total = timestamp() - time_total\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"multiplicative",
|
|
"counter a\n" +
|
|
"counter b\n" +
|
|
" /foo/ {\n a = a * b\n" +
|
|
" a = a ** b\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"additive and mem storage",
|
|
"counter time_total\n" +
|
|
"counter variable by foo\n" +
|
|
"/(?P<foo>.*)/ {\n" +
|
|
" time_total += timestamp() - variable[$foo]\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"conditional expressions",
|
|
"counter foo\n" +
|
|
"/(?P<foo>.*)/ {\n" +
|
|
" $foo > 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
" $foo >= 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
" $foo < 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
" $foo <= 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
" $foo == 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
" $foo != 0 {\n" +
|
|
" foo += $foo\n" +
|
|
" }\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"decorator definition and invocation",
|
|
"def foo { next\n }\n" +
|
|
"@foo { }\n",
|
|
},
|
|
|
|
{
|
|
"const regex",
|
|
"const X /foo/\n" +
|
|
"/foo / + X + / bar/ {\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"multiline regex starting with newline",
|
|
"const FOO\n" +
|
|
"/some regex here/ +\n" +
|
|
"/plus some other things/",
|
|
},
|
|
|
|
{
|
|
"multiline regex",
|
|
"/foo / +\n" +
|
|
"/barrr/ {\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"len",
|
|
"/(?P<foo>foo)/ {\n" +
|
|
"len($foo) > 0 {\n" +
|
|
"}\n" +
|
|
"}\n",
|
|
},
|
|
|
|
{
|
|
"def and next",
|
|
"def foobar {/(?P<date>.*)/ {" +
|
|
" next" +
|
|
"}" +
|
|
"}",
|
|
},
|
|
|
|
{
|
|
"const",
|
|
`const IP /\d+(\.\d+){3}/`,
|
|
},
|
|
|
|
{
|
|
"bitwise",
|
|
`gauge a
|
|
/foo(\d)/ {
|
|
a = $1 & 7
|
|
a = $1 | 8
|
|
a = $1 << 4
|
|
a = $1 >> 20
|
|
a = $1 ^ 15
|
|
a = ~ 1
|
|
}`,
|
|
},
|
|
|
|
{
|
|
"logical",
|
|
`0 || 1 && 0 {
|
|
}
|
|
`,
|
|
},
|
|
|
|
{
|
|
"floats",
|
|
`gauge foo
|
|
/foo/ {
|
|
foo = 3.14
|
|
}`,
|
|
},
|
|
|
|
{
|
|
"simple otherwise action",
|
|
"otherwise {}\n",
|
|
},
|
|
|
|
{
|
|
"pattern action then otherwise action",
|
|
`counter lines_total by type
|
|
/foo/ {
|
|
lines_total["foo"]++
|
|
}
|
|
otherwise {
|
|
lines_total["misc"] += 10
|
|
}`,
|
|
},
|
|
|
|
{
|
|
"simple else clause",
|
|
"/foo/ {} else {}",
|
|
},
|
|
|
|
{
|
|
"nested else clause",
|
|
"/foo/ { / bar/ {} } else { /quux/ {} else {} }",
|
|
},
|
|
|
|
{
|
|
"mod operator",
|
|
`gauge a
|
|
/foo/ {
|
|
a = 3 % 1
|
|
}`,
|
|
},
|
|
|
|
{
|
|
"delete",
|
|
`counter foo by bar
|
|
/foo/ {
|
|
del foo[$1]
|
|
}`,
|
|
},
|
|
|
|
{
|
|
"delete after",
|
|
`counter foo by bar
|
|
/foo/ {
|
|
del foo[$1] after 168h
|
|
}`,
|
|
},
|
|
|
|
{"getfilename", `
|
|
getfilename()
|
|
`},
|
|
|
|
{"indexed expression arg list", `
|
|
counter foo by a,b
|
|
/(\d) (\d+)/ {
|
|
foo[$1,$2]++
|
|
}`},
|
|
|
|
{"paren expr", `
|
|
(0) || (1 && 3) {
|
|
}`},
|
|
|
|
{"regex cond expr", `
|
|
/(\d)/ && 1 {
|
|
}
|
|
`},
|
|
|
|
{"concat expr 1", `
|
|
const X /foo/
|
|
/bar/ + X {
|
|
}`},
|
|
{"concat expr 2", `
|
|
const X /foo/
|
|
X {
|
|
}`},
|
|
|
|
{"match expression 1", `
|
|
$foo =~ /bar/ {
|
|
}
|
|
$foo !~ /bar/ {
|
|
}
|
|
`},
|
|
{"match expression 2", `
|
|
$foo =~ /bar/ + X {
|
|
}`},
|
|
{"match expression 3", `
|
|
const X /foo/
|
|
$foo =~ X {
|
|
}`},
|
|
|
|
{"capref used in def", `
|
|
/(?P<x>.*)/ && $x > 0 {
|
|
}`},
|
|
|
|
{"match expr 4", `
|
|
/(?P<foo>.{6}) (?P<bar>.*)/ {
|
|
$foo =~ $bar {
|
|
}
|
|
}`},
|
|
|
|
{"stop", `
|
|
// {
|
|
stop
|
|
}`},
|
|
|
|
{"substitution", `
|
|
/(\d,\d)/ {
|
|
subst(",", ",", $1)
|
|
}`},
|
|
|
|
{"pattern in arg expr list", `
|
|
/(\d,\d)/ {
|
|
subst(/,/, "", $1)
|
|
}`},
|
|
}
|
|
|
|
func TestParserRoundTrip(t *testing.T) {
|
|
if *parserTestDebug {
|
|
mtailDebug = 3
|
|
}
|
|
for _, tc := range parserTests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
p := newParser(tc.name, strings.NewReader(tc.program))
|
|
r := mtailParse(p)
|
|
|
|
if r != 0 || p.root == nil || len(p.errors) > 0 {
|
|
t.Error("1st pass parse errors:\n")
|
|
for _, e := range p.errors {
|
|
t.Errorf("\t%s\n", e)
|
|
}
|
|
t.Fatal()
|
|
}
|
|
|
|
if *parserTestDebug {
|
|
s := Sexp{}
|
|
t.Log("AST:\n" + s.Dump(p.root))
|
|
}
|
|
|
|
u := Unparser{}
|
|
output := u.Unparse(p.root)
|
|
|
|
p2 := newParser(tc.name+" 2", strings.NewReader(output))
|
|
r = mtailParse(p2)
|
|
if r != 0 || p2.root == nil || len(p2.errors) > 0 {
|
|
t.Errorf("2nd pass parse errors:\n")
|
|
for _, e := range p2.errors {
|
|
t.Errorf("\t%s\n", e)
|
|
}
|
|
t.Logf("2nd pass input was:\n%s", output)
|
|
t.Logf("2nd pass diff:\n%s", testutil.Diff(tc.program, output))
|
|
t.Fatal()
|
|
}
|
|
|
|
u = Unparser{}
|
|
output2 := u.Unparse(p2.root)
|
|
|
|
testutil.ExpectNoDiff(t, output2, output)
|
|
})
|
|
}
|
|
}
|
|
|
|
type parserInvalidProgram struct {
|
|
name string
|
|
program string
|
|
errors []string
|
|
}
|
|
|
|
var parserInvalidPrograms = []parserInvalidProgram{
|
|
{
|
|
"unknown character",
|
|
"?\n",
|
|
[]string{"unknown character:1:1: Unexpected input: '?'"},
|
|
},
|
|
|
|
{
|
|
"unterminated regex",
|
|
"/foo\n",
|
|
[]string{
|
|
"unterminated regex:1:2-4: Unterminated regular expression: \"/foo\"",
|
|
"unterminated regex:1:2-4: syntax error: unexpected end of file, expecting '/' to end regex",
|
|
},
|
|
},
|
|
|
|
{
|
|
"unterminated string",
|
|
" \"foo }\n",
|
|
[]string{"unterminated string:1:2-7: Unterminated quoted string: \"\\\"foo }\""},
|
|
},
|
|
|
|
{
|
|
"unterminated const regex",
|
|
"const X /(?P<foo>",
|
|
[]string{
|
|
"unterminated const regex:1:10-17: Unterminated regular expression: \"/(?P<foo>\"",
|
|
"unterminated const regex:1:10-17: syntax error: unexpected end of file, expecting '/' to end regex",
|
|
},
|
|
},
|
|
|
|
{
|
|
"unbalanced {",
|
|
"/foo/ {\n",
|
|
[]string{"unbalanced {:2:1: syntax error: unexpected end of file, expecting '}' to end block"},
|
|
},
|
|
{
|
|
"unbalanced else {",
|
|
"/foo/ { } else {\n",
|
|
[]string{"unbalanced else {:2:1: syntax error: unexpected end of file, expecting '}' to end block"},
|
|
},
|
|
{
|
|
"unbalanced otherwise {",
|
|
"otherwise {\n",
|
|
[]string{"unbalanced otherwise {:2:1: syntax error: unexpected end of file, expecting '}' to end block"},
|
|
},
|
|
|
|
{
|
|
"index of non-terminal 1",
|
|
`// {
|
|
foo++[$1]++
|
|
}`,
|
|
[]string{"index of non-terminal 1:2:7: syntax error: unexpected indexing of an expression"},
|
|
},
|
|
{
|
|
"index of non-terminal 2",
|
|
`// {
|
|
0[$1]++
|
|
}`,
|
|
[]string{"index of non-terminal 2:2:3: syntax error: unexpected indexing of an expression"},
|
|
},
|
|
{
|
|
"index of pattern",
|
|
`/foo/[0]
|
|
`,
|
|
[]string{"index of pattern:1:6: syntax error: unexpected indexing of an expression"},
|
|
},
|
|
|
|
{
|
|
"statement with no effect",
|
|
`/(\d)foo/ {
|
|
timestamp() - $1
|
|
}`,
|
|
[]string{"statement with no effect:3:18: syntax error: statement with no effect, missing an assignment, `+' concatenation, or `{}' block?"},
|
|
},
|
|
|
|
{
|
|
"pattern without block",
|
|
`/(?P<a>.)/
|
|
`,
|
|
[]string{"pattern without block:2:11: syntax error: statement with no effect, missing an assignment, `+' concatenation, or `{}' block?"},
|
|
},
|
|
|
|
{
|
|
"paired pattern without block",
|
|
`/(?P<a>.)/
|
|
/(?P<b>.)/ {}
|
|
`,
|
|
[]string{"paired pattern without block:2:11: syntax error: statement with no effect, missing an assignment, `+' concatenation, or `{}' block?"},
|
|
},
|
|
|
|
{
|
|
"dimensioned limit per dimension",
|
|
"counter foo by a limit 10, b",
|
|
[]string{"dimensioned limit per dimension:1:26: syntax error: unexpected COMMA"},
|
|
},
|
|
}
|
|
|
|
func TestParseInvalidPrograms(t *testing.T) {
|
|
if *parserTestDebug {
|
|
mtailDebug = 3
|
|
}
|
|
for _, tc := range parserInvalidPrograms {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
p := newParser(tc.name, strings.NewReader(tc.program))
|
|
mtailParse(p)
|
|
|
|
testutil.ExpectNoDiff(t,
|
|
strings.Join(tc.errors, "\n"), // want
|
|
strings.TrimRight(p.errors.Error(), "\n")) // got
|
|
if p.errors.Error() == "no errors" && *parserTestDebug {
|
|
s := Sexp{}
|
|
t.Log("AST:\n" + s.Dump(p.root))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
var parsePositionTests = []struct {
|
|
name string
|
|
program string
|
|
positions []*position.Position
|
|
}{
|
|
{
|
|
"empty",
|
|
"",
|
|
nil,
|
|
},
|
|
{
|
|
"variable",
|
|
`counter foo`,
|
|
[]*position.Position{{"variable", 0, 8, 10}},
|
|
},
|
|
{
|
|
"pattern",
|
|
`const ID /foo/`,
|
|
[]*position.Position{{"pattern", 0, 9, 13}},
|
|
},
|
|
{
|
|
"multiline regex",
|
|
"const ID\n" +
|
|
"/foo/ +\n" +
|
|
"/bar/",
|
|
// TODO: Update position for the first token to `1, 0, 4` when position tracking is fixed
|
|
[]*position.Position{{"multiline regex", 1, 4, 4}, {"multiline regex", 2, 0, 4}},
|
|
},
|
|
}
|
|
|
|
func TestParsePositionTests(t *testing.T) {
|
|
for _, tc := range parsePositionTests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
// Not t.Parallel() because the parser is not reentrant, and mtailDebug is a global.
|
|
root, err := Parse(tc.name, strings.NewReader(tc.program))
|
|
testutil.FatalIfErr(t, err)
|
|
p := &positionCollector{}
|
|
ast.Walk(p, root)
|
|
testutil.ExpectNoDiff(t, tc.positions, p.positions, testutil.AllowUnexported(position.Position{}))
|
|
})
|
|
}
|
|
}
|
|
|
|
type positionCollector struct {
|
|
positions []*position.Position
|
|
}
|
|
|
|
func (p *positionCollector) VisitBefore(node ast.Node) (ast.Visitor, ast.Node) {
|
|
switch n := node.(type) {
|
|
case *ast.VarDecl, *ast.PatternLit:
|
|
p.positions = append(p.positions, n.Pos())
|
|
}
|
|
return p, node
|
|
}
|
|
|
|
func (p *positionCollector) VisitAfter(node ast.Node) ast.Node {
|
|
return node
|
|
}
|