perf: rewrite field:* to _exists_:field

parser/seqql_filter.go

@@ -125,13 +125,17 @@ func parseFulltextSearchFilter(lex *lexer, fieldName string, t seq.TokenizerType
 		if err != nil {
 			return nil, fmt.Errorf("parsing keyword for field %q: %s", fieldName, err)
 		}
-		return &ASTNode{Value: &Literal{Field: fieldName, Terms: terms}}, nil
+
+		lit := Literal{Field: fieldName, Terms: terms}
+		return &ASTNode{Value: lit.Optimize()}, nil
+
 	case seq.TokenizerTypeText:
 		tokens, err := parseSeqQLText(fieldName, value, caseSensitive)
 		if err != nil {
 			return nil, fmt.Errorf("parsing text for field %q: %s", fieldName, err)
 		}
 		return buildAndTree(tokens), nil
+
 	default:
 		panic(fmt.Errorf("BUG: unexpected index type: %d", t))
 	}
@@ -288,8 +292,12 @@ func parseSeqQLKeyword(token string, caseSensitive bool) ([]Term, error) {
 
 func parseSeqQLText(field, token string, sensitive bool) ([]Token, error) {
 	if token == "" {
-		return []Token{&Literal{Field: field, Terms: []Term{newTextTerm("")}}}, nil
+		return []Token{&Literal{
+			Field: field,
+			Terms: []Term{newTextTerm("")}},
+		}, nil
 	}
+
 	var tokens []Token
 	current := &Literal{Field: field}
 
@@ -318,17 +326,19 @@ func parseSeqQLText(field, token string, sensitive bool) ([]Token, error) {
 		// So create new literal.
 
 		if len(current.Terms) != 0 {
-			tokens = append(tokens, current)
+			tokens = append(tokens, current.Optimize())
 			current = &Literal{Field: field}
 		}
+
 		token = token[size:]
 	}
+
 	if term.Data != "" {
 		current.appendTerm(term, sensitive)
 	}
 
 	if current != nil && len(current.Terms) > 0 {
-		tokens = append(tokens, current)
+		tokens = append(tokens, current.Optimize())
 	}
 
 	if len(tokens) == 0 {
@@ -338,5 +348,6 @@ func parseSeqQLText(field, token string, sensitive bool) ([]Token, error) {
 			Terms: []Term{{Kind: TermText, Data: ""}},
 		})
 	}
+
 	return tokens, nil
 }

parser/seqql_filter_test.go

@@ -76,6 +76,7 @@ func TestSeqQLAll(t *testing.T) {
 	test("*", "*")
 	test("((*))", "*")
 	test("((*) OR service:foo) AND service:bar", "((* or service:foo) and service:bar)")
+	test("_exists_:*", "_exists_:*")
 
 	// Propagate "not".
 	test(`NOT NOT text:a`, `text:a`)
@@ -106,7 +107,7 @@ func TestSeqQLAll(t *testing.T) {
 	test(`service:"some*thing*"`, `service:some*thing*`)
 	test(`service:some*thing*`, `service:some*thing*`)
 	test(`service:*thing*`, `service:*thing*`)
-	test(`service:"*"`, `service:*`)
+	test(`service:"*"`, `_exists_:service`)
 	test(`service:"cms"*"inter"*"api"`, `service:cms*inter*api`)
 
 	// Test keyword wildcards.
@@ -129,7 +130,7 @@ func TestSeqQLAll(t *testing.T) {
 	test(`keyword:'#''$'"^"`, `keyword:"#$^"`)
 	test(`message:'#''$'"^"`, `message:""`)
 	test(`'#':'#'`, `"#":"#"`)
-	test(`"*":"*"`, `"\*":*`)
+	test(`"*":"*"`, `_exists_:"\*"`)
 	test("`*`:`*`", `"\*":"\*"`)
 	test(`m:a AND OR : r`, `(m:a and "OR":r)`)
 
@@ -143,8 +144,8 @@ func TestSeqQLAll(t *testing.T) {
 	test(`service:a or not service:b or service:c`, `(not (not service:c and (not service:a and service:b)))`)
 	test(`not (service:a or service:c)`, `(not (service:a or service:c))`)
 	test(`NOT Not service:a`, `service:a`)
-	test(`service:*`, `service:*`)
-	test(` service : * `, `service:*`)
+	test(`service:*`, `_exists_:service`)
+	test(` service : * `, `_exists_:service`)
 	test(`service:a or service:b AND NOT service:c`, `(service:a or (not service:c and service:b))`)
 
 	// Test comments.
@@ -162,10 +163,10 @@ service:"wms-svc-logistics-megasort" and level:"#"
 
 	// Test complex search with wildcards.
 	test(`text:"\*\**"`, `text:"\*\*"*`)
-	test(`text:'value=*' AND text:'value="\*"*'`, `((text:value and text:*) and ((text:value and text:"\*") and text:*))`)
-	test(`text:value'="\*\*"*' AND text:"\*\*"`, `(((text:value and text:"\*\*") and text:*) and text:"\*\*")`)
+	test(`text:'value=*' AND text:'value="\*"*'`, `((text:value and _exists_:text) and ((text:value and text:"\*") and _exists_:text))`)
+	test(`text:value'="\*\*"*' AND text:"\*\*"`, `(((text:value and text:"\*\*") and _exists_:text) and text:"\*\*")`)
 	test(`text:'value=*' AND text:'value="\*"*' AND text:'value="\*\*"*' AND text:"\*\*" AND text:"\*\**"`,
-		`(((((text:value and text:*) and ((text:value and text:"\*") and text:*)) and ((text:value and text:"\*\*") and text:*)) and text:"\*\*") and text:"\*\*"*)`)
+		`(((((text:value and _exists_:text) and ((text:value and text:"\*") and _exists_:text)) and ((text:value and text:"\*\*") and _exists_:text)) and text:"\*\*") and text:"\*\*"*)`)
 
 	// Test escape.
 	test("keyword:`+7 995 28 07`", "keyword:\"+7 995 28 07\"")
@@ -215,8 +216,8 @@ service:"wms-svc-logistics-megasort" and level:"#"
 
 	// Test filter 'in'.
 	test(`service:in(auth-api, api-gateway, clickhouse-shard-*)`, `((service:auth-api or service:api-gateway) or service:clickhouse-shard-*)`)
-	test(`service:in(*, *, *)`, `((service:* or service:*) or service:*)`)
-	test(`service:in(*)`, `service:*`)
+	test(`service:in(*, *, *)`, `((_exists_:service or _exists_:service) or _exists_:service)`)
+	test(`service:in(*)`, `_exists_:service`)
 	test(`level:in(1)`, `level:1`)
 	test(`level:in(1, '2', 'three')`, `((level:1 or level:2) or level:three)`)
 	test(`level:in(1, '2', ''*3*"")`, `((level:1 or level:2) or level:*3*)`)

parser/token_literal.go

@@ -24,6 +24,24 @@ func (n *Literal) DumpSeqQL(o *strings.Builder) {
 	}
 }
 
+func (n *Literal) Optimize() *Literal {
+	// We cannot rewrite _all_:* to _exists_:_all_
+	ok := n.Field != seq.TokenAll &&
+		// We cannot rewrite _exists_:* to _exists_:_exists_
+		n.Field != seq.TokenExists &&
+		// We cannot rewrite anything except foo:*
+		(len(n.Terms) == 1 && n.Terms[0].IsWildcard())
+
+	if !ok {
+		return n
+	}
+
+	return &Literal{
+		Field: seq.TokenExists,
+		Terms: []Term{newTextTerm(n.Field)},
+	}
+}
+
 func (n *Literal) appendTerm(term Term, sensitive bool) {
 	if !sensitive {
 		term.Data = strings.ToLower(term.Data)

pattern/pattern_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/ozontech/seq-db/parser"
 )
 
+const fieldName = "m"
+
 type testTokenProvider struct {
 	ordered  *simpleTokenProvider
 	shuffled *simpleTokenProvider
@@ -155,15 +157,28 @@ func parseSingleTokenForTests(query string) (parser.Token, error) {
 
 func search(t *testing.T, tp *simpleTokenProvider, req string, expect []string) {
 	searchType := "full"
+
 	if tp.Ordered() {
 		searchType = "narrow"
 	}
 
-	token, err := parseSingleTokenForTests("m:" + req)
+	token, err := parseSingleTokenForTests(fieldName + ":" + req)
 	require.NoError(t, err)
-	s := newSearcher(token, tp)
+
+	// When parsing query we perform following rewrite:
+	//   foo:* --> _exists_:foo
+	// This is workaround for that so we could really
+	// test the pattern checking.
+	if req == "*" {
+		token = &parser.Literal{
+			Field: fieldName,
+			Terms: []parser.Term{{Kind: parser.TermSymbol}},
+		}
+	}
 
 	res := []string{}
+	s := newSearcher(token, tp)
+
 	for i := s.FirstTID(); i <= s.LastTID(); i++ {
 		val := tp.GetToken(i)
 
@@ -176,8 +191,8 @@ func search(t *testing.T, tp *simpleTokenProvider, req string, expect []string)
 			res = append(res, string(val))
 		}
 	}
-	sort.Strings(res)
 
+	sort.Strings(res)
 	assert.Equal(t, expect, res, "%s search request %q failed", searchType, req)
 }