diff --git a/mongo/arith_fold_test.go b/mongo/arith_fold_test.go deleted file mode 100644 index 1488ee9d..00000000 --- a/mongo/arith_fold_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package mongo_test - -import ( - "strings" - "testing" - - mongo "github.com/bytebase/omni/mongo" - "github.com/bytebase/omni/mongo/ast" -) - -// numberIn digs the value out of the first statement's first document field. -func foldedField(t *testing.T, js, field string) ast.Node { - t.Helper() - stmts, err := mongo.Parse(js) - if err != nil { - t.Fatalf("parse failed: %v", err) - } - var found ast.Node - var scan func(n ast.Node) - scan = func(n ast.Node) { - if found != nil || n == nil { - return - } - switch v := n.(type) { - case *ast.Document: - for _, kv := range v.Pairs { - if kv.Key == field { - found = kv.Value - return - } - scan(kv.Value) - } - case *ast.Array: - for _, el := range v.Elements { - scan(el) - } - case *ast.CollectionStatement: - for _, a := range v.Args { - scan(a) - } - } - } - scan(stmts[0].AST) - if found == nil { - t.Fatalf("field %s not found", field) - } - return found -} - -// TestArithmeticConstantFolding pins constant-expression folding with -// JavaScript number semantics. Value AND text-form (BSON type carrier) are -// asserted: mongosh-verified on MongoDB 7 — integral results within int32 -// arrive as BSON int, everything else as double; % follows JS remainder -// signs; division is float division. -func TestArithmeticConstantFolding(t *testing.T) { - num := func(t *testing.T, js string) *ast.NumberLiteral { - t.Helper() - n, ok := foldedField(t, js, "x").(*ast.NumberLiteral) - if !ok { - t.Fatalf("expected folded NumberLiteral") - } - return n - } - - for _, tc := range []struct { - name string - js string - value string - isFloat bool - }{ - {"customer ttl", `db.c.find({x: 90 * 24 * 60 * 60});`, "7776000", false}, - {"int division stays int form", `db.c.find({x: 6 / 2});`, "3", false}, - {"float division", `db.c.find({x: 6 / 4});`, "1.5", true}, - {"js remainder sign", `db.c.find({x: -7 % 3});`, "-1", false}, - {"float remainder", `db.c.find({x: 5.5 % 2});`, "1.5", true}, - {"parens precedence", `db.c.find({x: (1 + 2) * 3});`, "9", false}, - {"unary in expression", `db.c.find({x: 3 * -2});`, "-6", false}, - {"adjacent signed number is subtraction", `db.c.find({x: 5 -2});`, "3", false}, - {"int32 boundary stays int", `db.c.find({x: 2147483647 + 0});`, "2147483647", false}, - {"beyond int32 becomes double form", `db.c.find({x: 2147483647 + 1});`, "2.147483648e+09", true}, - } { - t.Run(tc.name, func(t *testing.T) { - n := num(t, tc.js) - if n.Value != tc.value || n.IsFloat != tc.isFloat { - t.Fatalf("folded to Value=%q IsFloat=%v, want Value=%q IsFloat=%v", n.Value, n.IsFloat, tc.value, tc.isFloat) - } - }) - } - - t.Run("string concatenation", func(t *testing.T) { - s, ok := foldedField(t, `db.c.find({x: "a" + "b"});`, "x").(*ast.StringLiteral) - if !ok || s.Value != "ab" { - t.Fatalf("expected folded StringLiteral \"ab\", got %#v", s) - } - }) - - t.Run("customer statement end to end", func(t *testing.T) { - js := `db.cs_customer_frequency.createIndex({ trans_date: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60, name: "idx2" });` - n, ok := foldedField(t, js, "expireAfterSeconds").(*ast.NumberLiteral) - if !ok || n.Value != "7776000" || n.IsFloat { - t.Fatalf("expireAfterSeconds folded wrong: %#v", n) - } - }) -} - -// TestArithmeticFoldingRejections pins the closed-whitelist boundary: only -// pure-literal subtrees fold; everything else fails, in the same shapes as -// before the change where applicable. -func TestArithmeticFoldingRejections(t *testing.T) { - for _, tc := range []struct { - name string - js string - wantErr string - }{ - {"division by zero", `db.c.find({x: 1 / 0});`, "not a finite number"}, - {"zero by zero", `db.c.find({x: 0 / 0});`, "not a finite number"}, - {"number plus string", `db.c.find({x: 1 + "a"});`, "syntax error"}, - {"string times number", `db.c.find({x: "a" * 2});`, "syntax error"}, - {"identifier in arithmetic", `db.c.find({x: y * 2});`, "syntax error"}, - {"call in arithmetic", `db.c.find({x: ISODate("2024-01-01") + 1});`, "syntax error"}, - {"new still unsupported", `db.c.insertOne({d: new Date()});`, `"new" keyword is not supported`}, - {"method call form still rejected", `db.c.find({x: Date.now()});`, "expected ("}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := mongo.Parse(tc.js) - if err == nil { - t.Fatal("expected parse error") - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("error %q does not contain %q", err.Error(), tc.wantErr) - } - }) - } -} diff --git a/mongo/parse_test.go b/mongo/parse_test.go index 7f91adde..bc57eb4c 100644 --- a/mongo/parse_test.go +++ b/mongo/parse_test.go @@ -50,18 +50,20 @@ func TestParseStrict(t *testing.T) { }) } -// TestParseCreateIndexArithmetic pins the BYT-9950 statement: createIndex with -// a constant arithmetic TTL expression must parse through the public API. +// TestParseCreateIndexArithmetic pins the BYT-9950 statement: arithmetic +// expressions are not supported, and the strict Parse must surface the error +// instead of silently dropping the statement. func TestParseCreateIndexArithmetic(t *testing.T) { input := `db.cs_customer_frequency.createIndex( { trans_date: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60, name: "cs_customer_frequency_idx2" } );` - stmts, err := mongo.Parse(input) - if err != nil { - t.Fatalf("unexpected error: %v", err) + _, err := mongo.Parse(input) + if err == nil { + t.Fatal("expected parse error for arithmetic expression") } - if len(stmts) != 1 { - t.Fatalf("expected 1 statement, got %d", len(stmts)) + var pe *parser.ParseError + if !errors.As(err, &pe) { + t.Fatalf("expected *parser.ParseError, got %T: %v", err, err) } } diff --git a/mongo/parser/expression.go b/mongo/parser/expression.go index 77b987a5..2a4cae59 100644 --- a/mongo/parser/expression.go +++ b/mongo/parser/expression.go @@ -2,228 +2,14 @@ package parser import ( "fmt" - "math" - "strconv" "strings" "github.com/bytebase/omni/mongo/ast" ) -// parseExpression parses a value expression (document, array, literal, -// helper, identifier), including postfix .method(args) chains and constant -// arithmetic over literals. mongosh evaluates JavaScript, so expressions -// like 90 * 24 * 60 * 60 are legal wherever a value fits; this parser folds -// the pure-literal subset at parse time with JavaScript number semantics -// (all arithmetic in float64) and rejects everything else, keeping the -// accepted surface a closed whitelist: -// -// - binary * / % + - over number literals, ( ) grouping, unary + - -// - string + string concatenation -// - operands that are not literals (identifiers, calls) do not fold and -// fail exactly like before -// - results that are Infinity or NaN are rejected: BSON can carry them -// but a folded Infinity in DDL options is always a script bug, and the -// literal text form cannot round-trip them -// -// The folded literal's text form encodes the BSON type downstream -// (mongosh-verified): integral results within int32 keep integer form -// (-> int32), everything else takes a float form with '.' or 'e' -// (-> double), matching what mongosh itself sends. +// parseExpression parses a value expression (document, array, literal, helper, identifier), +// including optional postfix .method(args) chains (e.g., Binary.createFromBase64("...")). func (p *Parser) parseExpression() (ast.Node, error) { - return p.parseAdditive() -} - -// parseAdditive parses + and - over multiplicative expressions, folding -// literal operands. The lexer folds a sign into a following number, so -// "5 -2" arrives as two adjacent number tokens; an adjacent sign-prefixed -// number in operand position is JavaScript subtraction/addition and is -// treated as the corresponding binary operation. -func (p *Parser) parseAdditive() (ast.Node, error) { - left, err := p.parseMultiplicative() - if err != nil { - return nil, err - } - for { - var op byte - switch { - case p.cur.Type == '+': - op = '+' - p.advance() - case p.cur.Type == '-': - op = '-' - p.advance() - case p.cur.Type == tokNumber && len(p.cur.Str) > 0 && (p.cur.Str[0] == '-' || p.cur.Str[0] == '+'): - // Adjacent signed number: implicit addition of a signed term. - op = '+' - default: - return left, nil - } - right, err := p.parseMultiplicative() - if err != nil { - return nil, err - } - left, err = p.foldBinary(op, left, right) - if err != nil { - return nil, err - } - } -} - -// parseMultiplicative parses *, / and % over unary expressions, folding -// literal operands. -func (p *Parser) parseMultiplicative() (ast.Node, error) { - left, err := p.parseUnary() - if err != nil { - return nil, err - } - for p.cur.Type == '*' || p.cur.Type == '/' || p.cur.Type == '%' { - op := byte(p.cur.Type) - p.advance() - right, err := p.parseUnary() - if err != nil { - return nil, err - } - left, err = p.foldBinary(op, left, right) - if err != nil { - return nil, err - } - } - return left, nil -} - -// parseUnary parses an optional +/- sign applied to a unary expression, -// then a postfix expression. The sign only folds over number literals. -func (p *Parser) parseUnary() (ast.Node, error) { - if p.cur.Type == '-' || p.cur.Type == '+' { - opTok := p.cur - neg := p.cur.Type == '-' - p.advance() - operand, err := p.parseUnary() - if err != nil { - return nil, err - } - num, ok := operand.(*ast.NumberLiteral) - if !ok { - return nil, p.arithErrorAt(opTok.Loc, string(rune(opTok.Type))) - } - v, err := numberValue(num) - if err != nil { - return nil, p.arithErrorAt(opTok.Loc, string(rune(opTok.Type))) - } - if neg { - v = -v - } - return p.foldedNumber(v, ast.Loc{Start: opTok.Loc, End: num.Loc.End}) - } - return p.parsePostfix() -} - -// foldBinary folds one binary arithmetic operation over literal operands. -// Number op number uses JavaScript semantics (float64 throughout, -// math.Mod remainder). "+" additionally concatenates string literals. -// Any other operand combination is a syntax error at the current position: -// the folding surface stays a closed literal-only whitelist, and -// JavaScript's mixed-type coercion table is deliberately not modeled. -func (p *Parser) foldBinary(op byte, left, right ast.Node) (ast.Node, error) { - loc := ast.Loc{Start: left.GetLoc().Start, End: right.GetLoc().End} - - if op == '+' { - if ls, ok := left.(*ast.StringLiteral); ok { - rs, ok := right.(*ast.StringLiteral) - if !ok { - return nil, p.syntaxErrorAtCur() - } - return &ast.StringLiteral{Value: ls.Value + rs.Value, Loc: loc}, nil - } - } - - ln, ok := left.(*ast.NumberLiteral) - if !ok { - return nil, p.syntaxErrorAtCur() - } - rn, ok := right.(*ast.NumberLiteral) - if !ok { - return nil, p.syntaxErrorAtCur() - } - lv, err := numberValue(ln) - if err != nil { - return nil, p.syntaxErrorAtCur() - } - rv, err := numberValue(rn) - if err != nil { - return nil, p.syntaxErrorAtCur() - } - - var v float64 - switch op { - case '+': - v = lv + rv - case '-': - v = lv - rv - case '*': - v = lv * rv - case '/': - v = lv / rv - case '%': - v = math.Mod(lv, rv) - default: - return nil, p.syntaxErrorAtCur() - } - return p.foldedNumber(v, loc) -} - -// arithErrorAt returns a ParseError for an arithmetic operator applied to -// a non-literal operand. -func (p *Parser) arithErrorAt(pos int, opText string) *ParseError { - line, col := p.lineCol(pos) - return &ParseError{ - Message: fmt.Sprintf("syntax error at or near %q", opText), - Position: pos, - Line: line, - Column: col, - } -} - -// numberValue evaluates a number literal as a JavaScript number (float64). -func numberValue(n *ast.NumberLiteral) (float64, error) { - return strconv.ParseFloat(n.Value, 64) -} - -// foldedNumber renders a folded value back to a literal whose text form -// selects the same BSON type mongosh would send: integral within int32 -// keeps integer form (-> int32); everything else takes a float form with -// '.' or 'e' (-> double). Infinity and NaN are rejected. -func (p *Parser) foldedNumber(v float64, loc ast.Loc) (*ast.NumberLiteral, error) { - if math.IsInf(v, 0) || math.IsNaN(v) { - line, col := p.lineCol(loc.Start) - return nil, &ParseError{ - Message: "arithmetic result is not a finite number", - Position: loc.Start, - Line: line, - Column: col, - } - } - if v == math.Trunc(v) && v >= math.MinInt32 && v <= math.MaxInt32 { - return &ast.NumberLiteral{ - Value: strconv.FormatInt(int64(v), 10), - IsFloat: false, - Loc: loc, - }, nil - } - text := strconv.FormatFloat(v, 'g', -1, 64) - if !strings.ContainsAny(text, ".eE") { - text += ".0" - } - return &ast.NumberLiteral{ - Value: text, - IsFloat: true, - Loc: loc, - }, nil -} - -// parsePostfix parses a primary value with optional postfix .method(args) -// chains (e.g., Binary.createFromBase64("...")). -func (p *Parser) parsePostfix() (ast.Node, error) { node, err := p.parseValue() if err != nil { return nil, err @@ -274,19 +60,6 @@ func (p *Parser) parseValue() (ast.Node, error) { case '[': return p.parseArray() - case '(': - // Parenthesized constant expression: (1 + 2) * 3. - p.advance() - inner, err := p.parseAdditive() - if err != nil { - return nil, err - } - if p.cur.Type != ')' { - return nil, p.syntaxErrorAtCur() - } - p.advance() - return inner, nil - case tokRegex: tok := p.advance() pattern, flags := splitRegex(tok.Str)