diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 55d2aa247..0d2af99d2 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -64,6 +64,56 @@ cc_library( ], ) +cc_library( + name = "pratt_parser_worker", + srcs = ["pratt_parser_worker.cc"], + hdrs = ["pratt_parser_worker.h"], + deps = [ + ":ast_factory_interface", + ":lexer", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:strings", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "pratt_parser", + srcs = ["pratt_parser.cc"], + hdrs = ["pratt_parser.h"], + deps = [ + ":ast_factory", + ":pratt_parser_worker", + "//common:ast", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + ], +) + cc_test( name = "ast_factory_test", srcs = ["ast_factory_test.cc"], @@ -84,3 +134,29 @@ cc_test( "//internal:testing", ], ) + +cc_test( + name = "pratt_parser_test", + srcs = ["pratt_parser_test.cc"], + deps = [ + ":lexer", + ":pratt_parser", + ":pratt_parser_worker", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:options", + "//parser:parser_interface", + "//testutil:expr_printer", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc new file mode 100644 index 000000000..f86974a36 --- /dev/null +++ b/parser/internal/pratt_parser.cc @@ -0,0 +1,217 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "parser/internal/ast_factory.h" // IWYU pragma: keep +#include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +namespace { + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + std::string_view message) { + return absl::StrCat( + absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + location.line, location.column + 1, message), + source.DisplayErrorLocation(location)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +class PrattParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit PrattParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override { + for (const cel::Macro& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); + } + + absl::Status AddLibrary(cel::ParserLibrary library) override { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); + } + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); + } + + absl::StatusOr> Build() override { + using std::swap; + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const cel::ParserLibrary& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const cel::Macro& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique( + options_, std::move(macro_registry), std::move(library_ids)); + } + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +} // namespace + +template class PrattParserWorker; + +absl::StatusOr> PrattParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + if (source.content().size() > options_.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrFormat( + "expression size exceeds codepoint limit: %zu > %d", + source.content().size(), options_.expression_size_codepoint_limit)); + } + std::vector issues; + PrattParserWorker worker(source, options_, &issues); + Expr expr = worker.Parse(); + if (worker.is_recursion_limit_exceeded()) { + return absl::CancelledError( + absl::StrFormat("Expression recursion limit exceeded. limit: %d", + options_.max_recursion_depth)); + } + if (worker.has_errors()) { + std::string err_msg = FormatIssues(source, issues); + if (parse_issues != nullptr) { + parse_issues->swap(issues); + } + return absl::InvalidArgumentError(err_msg); + } + + cel::SourceInfo source_info; + source_info.set_location(std::string(source.description())); + for (const auto& [id, pos] : worker.GetNodePositions()) { + source_info.mutable_positions().insert({id, pos}); + } + source_info.mutable_line_offsets().reserve(worker.GetLineOffsets().size()); + for (int32_t offset : worker.GetLineOffsets()) { + source_info.mutable_line_offsets().push_back(offset); + } + return std::make_unique(std::move(expr), std::move(source_info)); +} + +std::unique_ptr PrattParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h new file mode 100644 index 000000000..c39c68521 --- /dev/null +++ b/parser/internal/pratt_parser.h @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +// PrattParserImpl implements the Pratt parsing algorithm for CEL expressions. +// +// WARNING: Since this implementation uses recursive descent to parse +// expressions, its stack consumption depends on expression nesting and +// recursion limits (`ParserOptions::max_recursion_depth`). Note that: +// In production builds (e.g., inside fibers with small default stacks such as +// 64KB), the available stack space may be much tighter than in default thread +// stacks. +// Consequently, `ParserOptions::max_recursion_depth` may need to be tuned +// depending on the caller environment to prevent stack overflow. +class PrattParserImpl final : public cel::Parser { + public: + explicit PrattParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~PrattParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()); + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc new file mode 100644 index 000000000..b1dd81fbb --- /dev/null +++ b/parser/internal/pratt_parser_test.cc @@ -0,0 +1,1434 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/internal/lexer.h" +#include "parser/internal/pratt_parser_worker.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "testutil/expr_printer.h" + +// Change to 0 to test with the ANTLR parser to check for differences. +#define USE_PRATT_PARSER 1 + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::testing::Eq; +using ::testing::NotNull; + +absl::StatusOr> Parse( + std::string_view expression, + const cel::ParserOptions& options = cel::ParserOptions(), + std::vector* issues = nullptr) { +#if USE_PRATT_PARSER + std::unique_ptr builder = NewPrattParserBuilder(options); +#else + std::unique_ptr builder = cel::NewParserBuilder(options); +#endif + CEL_ASSIGN_OR_RETURN(std::unique_ptr parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr source, + cel::NewSource(expression)); + return parser->Parse(*source, issues); +} + +struct TestCase { + std::string_view source; + std::string_view expected_ast; + bool enable_optional_syntax = false; + bool enable_variadic_logical_operators = false; +}; + +class PrattParserTest : public testing::TestWithParam {}; + +std::string Unindent(std::string_view multiline) { + std::vector unindented_lines; + int indent = -1; + for (std::string_view line : absl::StrSplit(multiline, '\n')) { + std::size_t pos = line.find_first_not_of(" \t"); + if (pos == std::string_view::npos) continue; + if (indent == -1) indent = pos; + unindented_lines.push_back(std::string(line.substr(indent))); + } + return absl::StrJoin(unindented_lines, "\n"); +} + +std::string_view ConstantKind(const cel::Constant& c) { + switch (c.kind_case()) { + case ConstantKindCase::kBool: + return "bool"; + case ConstantKindCase::kInt: + return "int64"; + case ConstantKindCase::kUint: + return "uint64"; + case ConstantKindCase::kDouble: + return "double"; + case ConstantKindCase::kString: + return "string"; + case ConstantKindCase::kBytes: + return "bytes"; + case ConstantKindCase::kNull: + return "NullValue"; + default: + return "unspecified_constant"; + } +} + +std::string_view ExprKind(const cel::Expr& e) { + switch (e.kind_case()) { + case ExprKindCase::kConstant: + // special cased, this doesn't appear. + return "Expr.Constant"; + case ExprKindCase::kIdentExpr: + return "Expr.Ident"; + case ExprKindCase::kSelectExpr: + return "Expr.Select"; + case ExprKindCase::kCallExpr: + return "Expr.Call"; + case ExprKindCase::kListExpr: + return "Expr.CreateList"; + case ExprKindCase::kMapExpr: + return "Expr.CreateMap"; + case ExprKindCase::kStructExpr: + return "Expr.CreateStruct"; + case ExprKindCase::kComprehensionExpr: + return "Expr.Comprehension"; + default: + return "unspecified_expr"; + } +} + +class KindAndIdAdorner : public cel::test::ExpressionAdorner { + public: + std::string Adorn(const cel::Expr& e) const override { + if (e.has_const_expr()) { + const cel::Constant& const_expr = e.const_expr(); + return absl::StrCat("^#", e.id(), ":", ConstantKind(const_expr), "#"); + } else { + return absl::StrCat("^#", e.id(), ":", ExprKind(e), "#"); + } + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } +}; + +TEST_P(PrattParserTest, Parse) { + const TestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_variadic_logical_operators = + test_case.enable_variadic_logical_operators; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + Parse(test_case.source, options)); + const Expr& root = ast->root_expr(); + KindAndIdAdorner kind_and_id_adorner; + test::ExprPrinter printer(kind_and_id_adorner); + EXPECT_EQ(Unindent(printer.Print(root)), Unindent(test_case.expected_ast)); +} + +std::vector GetParserTestCases() { + return { + TestCase{ + .source = "null", + .expected_ast = R"( + null^#1:NullValue# + )", + }, + TestCase{ + .source = "true", + .expected_ast = R"( + true^#1:bool# + )", + }, + TestCase{ + .source = "false", + .expected_ast = R"( + false^#1:bool# + )", + }, + TestCase{ + .source = "123", + .expected_ast = R"( + 123^#1:int64# + )", + }, + TestCase{ + .source = "-123", + .expected_ast = R"( + -123^#1:int64# + )", + }, + TestCase{ + .source = "-9223372036854775808", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "0xA", + .expected_ast = R"( + 10^#1:int64# + )", + }, + TestCase{ + .source = "-0x1A", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0X1a", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0x8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "-0X8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "42u", + .expected_ast = R"( + 42u^#1:uint64# + )", + }, + TestCase{ + .source = "0u", + .expected_ast = R"( + 0u^#1:uint64# + )", + }, + TestCase{ + .source = "0xAu", + .expected_ast = R"( + 10u^#1:uint64# + )", + }, + TestCase{ + .source = "3.14159", + .expected_ast = R"( + 3.14159^#1:double# + )", + }, + TestCase{ + .source = "-3.14159", + .expected_ast = R"( + -3.14159^#1:double# + )", + }, + TestCase{ + .source = "-5.5e-3", + .expected_ast = R"( + -0.0055^#1:double# + )", + }, + TestCase{ + .source = "b'hello'", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "b\"hello\"", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "'hello world'", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"hello world\"", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"\u2764\"", + .expected_ast = "\"\u2764\"^#1:string#", + }, + TestCase{ + .source = "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", + .expected_ast = R"( + "\x07\x08\x0c\n\r\t\x0b'\"\\? Legal escapes"^#1:string# + )", + }, + TestCase{ + .source = "a", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "(a)", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "((a))", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "a.b", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?b", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a.b.c", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select#.c^#3:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b`", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b-c`", + .expected_ast = R"( + a^#1:Expr.Ident#.b-c^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?`b`", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "-x", + .expected_ast = R"( + -_( + x^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "- -1", + .expected_ast = R"( + -_( + -1^#2:int64# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-(1 + 2)", + .expected_ast = R"( + -_( + _+_( + 1^#2:int64#, + 2^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "---a", + .expected_ast = R"( + -_( + -_( + -_( + a^#4:Expr.Ident# + )^#3:Expr.Call# + )^#2:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!false", + .expected_ast = R"( + !_( + false^#2:bool# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!a", + .expected_ast = R"( + !_( + a^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-!true", + .expected_ast = R"( + -_( + !_( + true^#3:bool# + )^#2:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a + b", + .expected_ast = R"( + _+_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a - b", + .expected_ast = R"( + _-_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "4--4", + .expected_ast = R"( + _-_( + 4^#1:int64#, + -4^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a * b", + .expected_ast = R"( + _*_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a / b", + .expected_ast = R"( + _/_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a % b", + .expected_ast = R"( + _%_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a in b", + .expected_ast = R"( + @in( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a == b", + .expected_ast = R"( + _==_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a != b", + .expected_ast = R"( + _!=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a > b", + .expected_ast = R"( + _>_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a >= b", + .expected_ast = R"( + _>=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a < b", + .expected_ast = R"( + _<_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a <= b", + .expected_ast = R"( + _<=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a && b", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _&&_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d && e", + .expected_ast = R"( + _&&_( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _&&_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "a || b", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _||_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d || e", + .expected_ast = R"( + _||_( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _||_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "10 - 3 - 2", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 - 1 / 2 == 6 % 1", + .expected_ast = R"( + _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# + )^#10:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 == 7 && true || false", + .expected_ast = R"( + _||_( + _&&_( + _==_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + 7^#7:int64# + )^#6:Expr.Call#, + true^#8:bool# + )^#9:Expr.Call#, + false^#10:bool# + )^#11:Expr.Call# + )", + }, + TestCase{ + .source = "x > 0 ? 'pos' : 'neg'", + .expected_ast = R"( + _?_:_( + _>_( + x^#1:Expr.Ident#, + 0^#3:int64# + )^#2:Expr.Call#, + "pos"^#5:string#, + "neg"^#6:string# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "a?b:c", + .expected_ast = R"( + _?_:_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident#, + c^#4:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[b]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[?b]", + .expected_ast = R"( + _[?_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a[3]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + 3^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[1,3,4][0]", + .expected_ast = R"( + _[_]( + [ + 1^#2:int64#, + 3^#3:int64#, + 4^#4:int64# + ]^#1:Expr.CreateList#, + 0^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a()", + .expected_ast = R"( + a()^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b)", + .expected_ast = R"( + a( + b^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b, c)", + .expected_ast = R"( + a( + b^#2:Expr.Ident#, + c^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a.b()", + .expected_ast = R"( + a^#1:Expr.Ident#.b()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(1)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + 1^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(c)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + c^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "size(x) == x.size()", + .expected_ast = R"( + _==_( + size( + x^#2:Expr.Ident# + )^#1:Expr.Call#, + x^#4:Expr.Ident#.size()^#5:Expr.Call# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "\"foo\".size()", + .expected_ast = R"( + "foo"^#1:string#.size()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[true, true].size() == 2", + .expected_ast = R"( + _==_( + [ + true^#2:bool#, + true^#3:bool# + ]^#1:Expr.CreateList#.size()^#4:Expr.Call#, + 2^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "[]", + .expected_ast = R"( + []^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[a]", + .expected_ast = R"( + [ + a^#2:Expr.Ident# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3,]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[?1, 2]", + .expected_ast = R"( + [ + ?1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{}", + .expected_ast = R"( + {}^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{'key': 'value', 'num': 42}", + .expected_ast = R"( + { + "key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{?'key': 'value', 'num': 42}", + .expected_ast = R"( + { + ?"key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{foo: 5, bar: \"xyz\"}", + .expected_ast = R"( + { + foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#, + bar^#5:Expr.Ident#:"xyz"^#7:string#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "google.protobuf.Empty{}", + .expected_ast = R"( + google.protobuf.Empty{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ }", + .expected_ast = R"( + foo{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ a:b }", + .expected_ast = R"( + foo{ + a:b^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b`: 1}", + .expected_ast = R"( + A{ + b:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b-c`: 1}", + .expected_ast = R"( + A{ + b-c:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{?field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + ?field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + .enable_optional_syntax = true, + }, + }; +} + +std::string TestName(const testing::TestParamInfo& test_info) { + std::string name = absl::StrCat(test_info.index, "-", test_info.param.source); + absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); + return name; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, + testing::ValuesIn(GetParserTestCases()), TestName); + +struct ErrorTestCase { + std::string_view source; + std::string_view expected_error; + bool enable_optional_syntax = false; + bool enable_quoted_identifiers = false; +}; + +class PrattParserErrorTest : public testing::TestWithParam {}; + +std::string FormatIssues(const cel::Source& source, + const std::vector& issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, + absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + issue.location().line, issue.location().column + 1, + issue.message()), + source.DisplayErrorLocation(issue.location())); + }); +} + +TEST_P(PrattParserErrorTest, ParseSyntaxError) { + const ErrorTestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_quoted_identifiers = test_case.enable_quoted_identifiers; + std::vector issues; + absl::StatusOr> result = + Parse(test_case.source, options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument, + Eq(test_case.expected_error))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource(test_case.source)); + EXPECT_EQ(FormatIssues(*source, issues), test_case.expected_error); +} + +std::vector GetErrorTestCases() { + return { + ErrorTestCase{ + .source = "1 + 2 * 3 4", + .expected_error = + "ERROR: :1:11: unexpected token after expression\n" + " | 1 + 2 * 3 4\n" + " | ..........^", + }, + ErrorTestCase{ + .source = "1{}", + .expected_error = + "ERROR: :1:2: unexpected token after expression\n" + " | 1{}\n" + " | .^", + }, + ErrorTestCase{ + .source = "true ? 1", + .expected_error = + "ERROR: :1:9: expected ':' in conditional expression\n" + " | true ? 1\n" + " | ........^", + }, + ErrorTestCase{ + .source = "a.?b", + .expected_error = "ERROR: :1:2: unsupported syntax '.?'\n" + " | a.?b\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.", + .expected_error = + "ERROR: :1:3: expected identifier after '.'\n" + " | a.\n" + " | ..^", + }, + ErrorTestCase{ + .source = "a[?0]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | a[?0]\n" + " | .^", + }, + ErrorTestCase{ + .source = ". *", + .expected_error = "ERROR: :1:3: expected identifier\n" + " | . *\n" + " | ..^", + }, + ErrorTestCase{ + .source = ".as", + .expected_error = "ERROR: :1:2: reserved identifier: as\n" + " | .as\n" + " | .^", + }, + ErrorTestCase{ + .source = "* 2", + .expected_error = + "ERROR: :1:1: unexpected token\n" + " | * 2\n" + " | ^\n" + "ERROR: :1:3: unexpected token after expression\n" + " | * 2\n" + " | ..^", + }, + ErrorTestCase{ + .source = "(1 + 2", + .expected_error = + "ERROR: :1:7: mismatched input expecting ')'\n" + " | (1 + 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "[?1]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | [?1]\n" + " | .^", + }, + ErrorTestCase{ + .source = "[1, 2", + .expected_error = "ERROR: :1:6: expected ']'\n" + " | [1, 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{?'k': 'v'}", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | {?'k': 'v'}\n" + " | .^", + }, + ErrorTestCase{ + .source = "{'k' 'v'}", + .expected_error = "ERROR: :1:6: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{'k': 'v'", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | {'k': 'v'\n" + " | .........^", + }, + ErrorTestCase{ + .source = "Msg{?f: 1}", + .expected_error = "ERROR: :1:5: unsupported syntax '?'\n" + " | Msg{?f: 1}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{1: 2}", + .expected_error = "ERROR: :1:5: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{f 10}", + .expected_error = "ERROR: :1:7: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", + }, + ErrorTestCase{ + .source = "Msg{f: 10", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | Msg{f: 10\n" + " | .........^", + }, + ErrorTestCase{ + .source = "f(1, 2", + .expected_error = + "ERROR: :1:7: mismatched input expecting ')'\n" + " | f(1, 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999", + .expected_error = "ERROR: :1:1: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999u", + .expected_error = "ERROR: :1:1: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", + }, + ErrorTestCase{ + .source = "1e", + .expected_error = + "ERROR: :1:1: floating point literal missing digits after " + "exponent separator\n" + " | 1e\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"unterminated", + .expected_error = "ERROR: :1:1: unterminated string literal\n" + " | \"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "b\"unterminated", + .expected_error = "ERROR: :1:1: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "a.?`foo`", + .expected_error = "ERROR: :1:4: unsupported syntax '`'\n" + " | a.?`foo`\n" + " | ...^", + .enable_optional_syntax = true, + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "a.`foo`()", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`foo`()\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`()", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`()\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`b@c`", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`b@c`\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.``", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.``\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`foo`", + .expected_error = "ERROR: :1:3: unsupported syntax '`'\n" + " | a.`foo`\n" + " | ..^", + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "`foo", + .expected_error = + "ERROR: :1:1: unterminated quoted identifier\n" + " | `foo\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "f(*, 1e, {2 3})", + .expected_error = + "ERROR: :1:3: unexpected token\n" + " | f(*, 1e, {2 3})\n" + " | ..^\n" + "ERROR: :1:6: floating point literal missing digits after " + "exponent separator\n" + " | f(*, 1e, {2 3})\n" + " | .....^\n" + "ERROR: :1:13: expected ':' in map entry\n" + " | f(*, 1e, {2 3})\n" + " | ............^", + }, + ErrorTestCase{ + .source = "(1 + *) + 2", + .expected_error = "ERROR: :1:6: unexpected token\n" + " | (1 + *) + 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "f(1 + *, 2)", + .expected_error = "ERROR: :1:7: unexpected token\n" + " | f(1 + *, 2)\n" + " | ......^", + }, + ErrorTestCase{ + .source = "(a. + 1)", + .expected_error = + "ERROR: :1:5: expected identifier after '.'\n" + " | (a. + 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "f(a., 1)", + .expected_error = + "ERROR: :1:5: expected identifier after '.'\n" + " | f(a., 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "[a., 1]", + .expected_error = + "ERROR: :1:4: expected identifier after '.'\n" + " | [a., 1]\n" + " | ...^", + }, + ErrorTestCase{ + .source = "-0x8000000000000001", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -0x8000000000000001\n" + " | .^", + }, + ErrorTestCase{ + .source = "-0x10000000000000000", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -0x10000000000000000\n" + " | .^", + }, + ErrorTestCase{ + .source = "-9223372036854775809", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -9223372036854775809\n" + " | .^", + }, + ErrorTestCase{ + .source = "-999999999999999999999999999999999999999", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -999999999999999999999999999999999999999\n" + " | .^", + }, + ErrorTestCase{ + .source = "-", + .expected_error = + "ERROR: :1:2: Syntax error: mismatched input '' " + "expecting expression\n" + " | -\n" + " | .^", + }, + ErrorTestCase{ + .source = "- *", + .expected_error = "ERROR: :1:3: unexpected token\n" + " | - *\n" + " | ..^", + }, + ErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error = "ERROR: :1:9: unexpected character\n" + " | \"😀😀😀😀😀\" ~error\n" + " | .．．．．．..^", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserErrorTest, PrattParserErrorTest, + testing::ValuesIn(GetErrorTestCases())); + +TEST(PrattParserTest, SourceInfoPositionsPopulated) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, Parse("a + b")); + const cel::SourceInfo& source_info = ast->source_info(); + EXPECT_FALSE(source_info.positions().empty()); + + const cel::Expr& root = ast->root_expr(); + EXPECT_EQ(source_info.positions().at(root.id()), 2); + ASSERT_TRUE(root.has_call_expr()); + ASSERT_EQ(root.call_expr().args().size(), 2); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[0].id()), 0); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[1].id()), 4); +} + +TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 5; + EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("[[[[[[1]]]]]]", options), + StatusIs(absl::StatusCode::kCancelled)); +} + +TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 2; + EXPECT_THAT(Parse("[1] + [2] + [3]", options), IsOkAndHolds(NotNull())); +} + +class TestParserWorker : public ParserWorker { + // Expose the protected constructor and methods for testing. + public: + using ParserWorker::GetTokenText; + using ParserWorker::ParserWorker; +}; + +TEST(ParserWorkerTest, GetTokenTextBoundsChecking) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource("hello world")); + cel::ParserOptions options; + TestParserWorker worker(*source, options, nullptr); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 5}), + "hello"); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = -1, .end = 5}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 5, .end = 2}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 100}), + ""); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc new file mode 100644 index 000000000..ff60e7962 --- /dev/null +++ b/parser/internal/pratt_parser_worker.cc @@ -0,0 +1,158 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser_worker.h" + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/strings/str_cat.h" +#include "common/source.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +ParserWorker::ParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : source_(source), + options_(options), + lexer_(source_), + parse_issues_(parse_issues) {} + +void ParserWorker::InitTokenStream() { + current_token_ = Token{.type = TokenType::kError, .start = 0, .end = 0}; + peek_token_ = NextSignificantToken(); +} + +std::string ParserWorker::GetTokenText(const Token& tok) const { + if (tok.start >= 0 && tok.end >= tok.start && + tok.end <= static_cast(source_.content().size())) { + return source_.content().ToString(tok.start, tok.end); + } + return ""; +} + +Token ParserWorker::NextSignificantToken() { + while (true) { + Token tok = lexer_.Lex(); + if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { + continue; + } + if (tok.type == TokenType::kError) { + ReportError(tok, lexer_.GetError().message); + } + return tok; + } +} + +Token ParserWorker::NextToken() { + current_token_ = peek_token_; + if (peek_token_.type != TokenType::kEnd) { + peek_token_ = NextSignificantToken(); + } + return current_token_; +} + +bool ParserWorker::Expect(TokenType type, std::string_view msg) { + if (peek_token_.type == type) { + NextToken(); + return true; + } + if (peek_token_.type != TokenType::kError) { + std::string err_msg; + if (msg.empty()) { + std::string tok_text = GetTokenText(peek_token_); + std::string formatted_tok; + if (peek_token_.type == TokenType::kEnd) { + formatted_tok = ""; + } else { + formatted_tok = absl::StrCat("'", tok_text, "'"); + } + err_msg = absl::StrCat("mismatched input ", formatted_tok, " expecting '", + TokenTypeToString(type), "'"); + } else { + err_msg = std::string(msg); + } + ReportError(peek_token_, err_msg); + } + SynchronizeOnDelimiter(); + return false; +} + +void ParserWorker::SynchronizeOnDelimiter() { + if (is_recovery_limit_exceeded()) { + while (peek_token_.type != TokenType::kEnd) { + NextToken(); + } + return; + } + while (peek_token_.type != TokenType::kEnd) { + if (peek_token_.type == TokenType::kComma || + peek_token_.type == TokenType::kRightParen || + peek_token_.type == TokenType::kRightBracket || + peek_token_.type == TokenType::kRightBrace) { + break; + } + NextToken(); + } +} +int64_t ParserWorker::NextId(int32_t position) { + int64_t id = next_id_++; + if (position >= 0) { + positions_.insert({id, position}); + } + return id; +} + +int64_t ParserWorker::CopyId(int64_t id) { + if (id == 0) { + return 0; + } + int32_t pos = 0; + if (auto it = positions_.find(id); it != positions_.end()) { + pos = it->second; + } + return NextId(pos); +} + +void ParserWorker::EraseId(int64_t id) { + positions_.erase(id); + if (next_id_ == id + 1) { + --next_id_; + } +} + +void ParserWorker::ReportError(int32_t position, std::string_view msg) { + cel::SourceLocation loc; + if (auto found = source_.GetLocation(position); found.has_value()) { + loc = *found; + } + ReportError(loc, msg); +} + +void ParserWorker::ReportError(const SourceLocation& loc, + std::string_view msg) { + error_count_++; + if (parse_issues_ != nullptr) { + parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); + } +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h new file mode 100644 index 000000000..a510f32a3 --- /dev/null +++ b/parser/internal/pratt_parser_worker.h @@ -0,0 +1,860 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/strings.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +class ParserWorker { + public: + ParserWorker(const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues); + + const absl::flat_hash_map& GetNodePositions() const { + return positions_; + } + absl::Span GetLineOffsets() const { + return source_.line_offsets(); + } + bool has_errors() const { return error_count_ > 0; } + bool is_recursion_limit_exceeded() const { return recursion_limit_exceeded_; } + + protected: + const cel::Source& source() const { return source_; } + const cel::ParserOptions& options() const { return options_; } + // Token stream management + void InitTokenStream(); + Token NextSignificantToken(); + Token NextToken(); + bool Expect(TokenType type, absl::string_view msg = ""); + std::string GetTokenText(const Token& tok) const; + void SynchronizeOnDelimiter(); + + // ID and Position tracking + int64_t NextId(int32_t position); + int64_t NextId(const Token& token) { return NextId(token.start); } + int64_t NextId() { return next_id_++; } + int64_t CopyId(int64_t id); + void EraseId(int64_t id); + + // Error reporting and recovery + bool is_recovery_limit_exceeded() const { + return error_count_ >= options_.error_recovery_limit; + } + void ReportError(int32_t position, absl::string_view msg); + void ReportError(const SourceLocation& loc, absl::string_view msg); + void ReportError(const Token& token, absl::string_view msg) { + ReportError(token.start, msg); + } + + const cel::Source& source_; + cel::ParserOptions options_; + Lexer lexer_; + Token current_token_; + Token peek_token_; + int recursion_depth_ = 0; + int64_t next_id_ = 1; + absl::flat_hash_map positions_; + std::vector* absl_nullable parse_issues_; + int error_count_ = 0; + bool lexer_error_reported_ = false; + bool recursion_limit_exceeded_ = false; +}; + +// Generic Pratt parser implementation parameterized by the AST node type +// (`ExprNode`). +// +// This class implements the core recursive-descent and operator-precedence +// parsing logic for CEL without depending on a concrete expression node data +// structure. All inspection and construction of AST nodes is performed through +// `AstFactoryInterface`. +// +// See `AstFactoryInterface` documentation in `ast_factory_interface.h` for +// details on how to use this generic parser with alternative AST structures. +template +class PrattParserWorker : public ParserWorker { + public: + explicit PrattParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : ParserWorker(source, options, parse_issues) { + this->InitTokenStream(); + } + + ExprNode Parse(); + + private: + using CelOperator = ::google::api::expr::common::CelOperator; + + ExprNode ParseExpr(); + ExprNode ParseConditionalOr(); + ExprNode ParseConditionalAnd(); + ExprNode ParseRelation(); + ExprNode ParseCalc(int min_prec); + ExprNode ParseUnary(); + ExprNode ParseMember(); + ExprNode ParsePrimary(); + ExprNode ParseList(); + ExprNode ParseMap(); + ExprNode ParseStruct(int64_t obj_id, absl::string_view struct_name); + std::vector ParseArguments(TokenType close_token); + ExprNode ParseIntLiteral(); + ExprNode ParseUintLiteral(); + ExprNode ParseDoubleLiteral(); + ExprNode ParseStringLiteral(); + ExprNode ParseBytesLiteral(); + std::string NormalizeIdent(const Token& tok, bool allow_quoted); + std::optional ExtractStructName(const ExprNode& expr); + int32_t GetLeftmostPosition(const ExprNode& expr); + ExprNode BalancedTree(absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi); + ExprNode BalanceLogical(absl::string_view op, std::vector terms, + std::vector ops, bool enable_variadic); + + AstFactoryInterface ast_factory_; +}; + +template +ExprNode PrattParserWorker::Parse() { + ExprNode expr = ParseExpr(); + if (is_recursion_limit_exceeded()) { + return expr; + } + if (peek_token_.type != TokenType::kEnd && + peek_token_.type != TokenType::kError) { + ReportError(peek_token_, "unexpected token after expression"); + } + return expr; +} + +template +ExprNode PrattParserWorker::ParseExpr() { + if (recursion_limit_exceeded_) { + return ExprNode(); + } + if (recursion_depth_ >= options_.max_recursion_depth) { + recursion_limit_exceeded_ = true; + return ExprNode(); + } + recursion_depth_++; + ExprNode lhs = ParseConditionalOr(); + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + int64_t op_id = NextId(); + ExprNode true_expr = ParseConditionalOr(); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + recursion_depth_--; + return lhs; + } + ExprNode false_expr = ParseExpr(); + recursion_depth_--; + return ast_factory_.NewCall( + op_id, CelOperator::CONDITIONAL, + std::vector{std::move(lhs), std::move(true_expr), + std::move(false_expr)}); + } + recursion_depth_--; + return lhs; +} + +// Parses logical OR expressions (e.g., `a || b || c`). +template +ExprNode PrattParserWorker::ParseConditionalOr() { + ExprNode lhs = ParseConditionalAnd(); + if (peek_token_.type != TokenType::kLogicalOr) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == TokenType::kLogicalOr) { + Token op_tok = NextToken(); + ExprNode rhs = ParseConditionalAnd(); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); + } + return BalanceLogical(CelOperator::LOGICAL_OR, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses logical AND expressions (e.g., `a && b && c`). +template +ExprNode PrattParserWorker::ParseConditionalAnd() { + ExprNode lhs = ParseRelation(); + if (peek_token_.type != TokenType::kLogicalAnd) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == TokenType::kLogicalAnd) { + Token op_tok = NextToken(); + ExprNode rhs = ParseRelation(); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); + } + return BalanceLogical(CelOperator::LOGICAL_AND, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses relational & equality ops (e.g., `a < b`, `x == y`, `a in b`). +template +ExprNode PrattParserWorker::ParseRelation() { + ExprNode lhs = ParseCalc(0); + while (true) { + TokenType tok = peek_token_.type; + absl::string_view op_name; + switch (tok) { + case TokenType::kLess: + op_name = CelOperator::LESS; + break; + case TokenType::kLessEqual: + op_name = CelOperator::LESS_EQUALS; + break; + case TokenType::kGreater: + op_name = CelOperator::GREATER; + break; + case TokenType::kGreaterEqual: + op_name = CelOperator::GREATER_EQUALS; + break; + case TokenType::kEqualEqual: + op_name = CelOperator::EQUALS; + break; + case TokenType::kExclamationEqual: + op_name = CelOperator::NOT_EQUALS; + break; + case TokenType::kIn: + op_name = CelOperator::IN; + break; + default: + return lhs; + } + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(0); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses arithmetic calculation expressions (e.g., `a + b * c`, `x - y % z`). +template +ExprNode PrattParserWorker::ParseCalc(int min_prec) { + ExprNode lhs = ParseUnary(); + while (true) { + TokenType tok = peek_token_.type; + int prec = 0; + absl::string_view op_name; + if (tok == TokenType::kAsterisk) { + prec = 2; + op_name = CelOperator::MULTIPLY; + } else if (tok == TokenType::kSlash) { + prec = 2; + op_name = CelOperator::DIVIDE; + } else if (tok == TokenType::kPercent) { + prec = 2; + op_name = CelOperator::MODULO; + } else if (tok == TokenType::kPlus) { + prec = 1; + op_name = CelOperator::ADD; + } else if (tok == TokenType::kMinus) { + prec = 1; + op_name = CelOperator::SUBTRACT; + } else { + break; + } + + if (prec < min_prec) break; + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(prec + 1); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses unary logical NOT and negation expressions (e.g., `!a`, `-b`). +template +ExprNode PrattParserWorker::ParseUnary() { + TokenType tok = peek_token_.type; + if (tok == TokenType::kExclamation) { + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode operand = ParseUnary(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), + std::vector{std::move(operand)}); + } + if (tok == TokenType::kMinus) { + Token op = NextToken(); + if (peek_token_.type == TokenType::kInt) { + Token lit_tok = NextToken(); + std::string text = GetTokenText(lit_tok); + int64_t int_val = 0; + bool success = false; + if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { + uint64_t uint_val = 0; + if (absl::SimpleHexAtoi(text, &uint_val)) { + if (uint_val <= uint64_t{0x8000000000000000}) { + if (uint_val == uint64_t{0x8000000000000000}) { + int_val = std::numeric_limits::min(); + } else { + int_val = -static_cast(uint_val); + } + success = true; + } + } + } else { + std::string val = absl::StrCat("-", text); + success = absl::SimpleAtoi(val, &int_val); + } + if (success) { + return ast_factory_.NewIntConst(NextId(op.start), int_val); + } + ReportError(lit_tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + if (peek_token_.type == TokenType::kFloat) { + Token lit_tok = NextToken(); + std::string val = absl::StrCat("-", GetTokenText(lit_tok)); + double double_val = 0.0; + if (absl::SimpleAtod(val, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(op.start), double_val); + } + ReportError(lit_tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + // Regular negate call + int64_t op_id = NextId(op); + ExprNode operand = ParseUnary(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), + std::vector{std::move(operand)}); + } + return ParseMember(); +} + +// Parses member calls & indexing (e.g., `a.b`, `a.b(c)`, `a[b]`, `a.?b`). +template +ExprNode PrattParserWorker::ParseMember() { + ExprNode lhs = ParsePrimary(); + while (true) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot) { + Token dot_tok = NextToken(); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(dot_tok, "unsupported syntax '.?'"); + } + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportError(id_tok, "expected identifier after '.'"); + } + SynchronizeOnDelimiter(); + return lhs; + } + bool is_member_call = peek_token_.type == TokenType::kLeftParen; + std::string id_text = + NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call); + if (optional) { + int64_t op_id = NextId(dot_tok); + lhs = ast_factory_.NewCall( + op_id, "_?._", + std::vector{ + std::move(lhs), + ast_factory_.NewStringConst(NextId(id_tok), id_text)}); + } else if (peek_token_.type == TokenType::kLeftParen) { + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); + std::vector args = ParseArguments(TokenType::kRightParen); + lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), + std::move(args)); + } else { + lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); + } + } else if (tok == TokenType::kLeftBracket) { + Token bracket_tok = NextToken(); + int64_t op_id = NextId(bracket_tok); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(bracket_tok, "unsupported syntax '?'"); + } + } + ExprNode index = ParseExpr(); + Expect(TokenType::kRightBracket, "expected ']'"); + lhs = ast_factory_.NewCall( + op_id, optional ? "_[?_]" : CelOperator::INDEX, + std::vector{std::move(lhs), std::move(index)}); + } else if (tok == TokenType::kLeftBrace) { + int32_t struct_pos = GetLeftmostPosition(lhs); + if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { + lhs = ParseStruct(NextId(struct_pos), *struct_name); + } else { + break; + } + } else { + break; + } + } + return lhs; +} + +// Parses primary expressions & literals (e.g., `null`, `true`, `x`, +// `has(x.y)`). +template +ExprNode PrattParserWorker::ParsePrimary() { + ExprNode expr; + TokenType tok_type = peek_token_.type; + if (tok_type == TokenType::kLeftParen) { + NextToken(); + expr = ParseExpr(); + Expect(TokenType::kRightParen); + } else if (tok_type == TokenType::kNull) { + Token tok = NextToken(); + expr = ast_factory_.NewNullConst(NextId(tok)); + } else if (tok_type == TokenType::kTrue || tok_type == TokenType::kFalse) { + Token tok = NextToken(); + expr = ast_factory_.NewBoolConst(NextId(tok), tok_type == TokenType::kTrue); + } else if (tok_type == TokenType::kInt) { + expr = ParseIntLiteral(); + } else if (tok_type == TokenType::kUint) { + expr = ParseUintLiteral(); + } else if (tok_type == TokenType::kFloat) { + expr = ParseDoubleLiteral(); + } else if (tok_type == TokenType::kString) { + expr = ParseStringLiteral(); + } else if (tok_type == TokenType::kBytes) { + expr = ParseBytesLiteral(); + } else if (tok_type == TokenType::kLeftBracket) { + expr = ParseList(); + } else if (tok_type == TokenType::kLeftBrace) { + expr = ParseMap(); + } else if (tok_type == TokenType::kDot || tok_type == TokenType::kIdent || + tok_type == TokenType::kReservedWord) { + bool leading_dot = false; + Token first_tok = peek_token_; + if (tok_type == TokenType::kDot) { + NextToken(); + leading_dot = true; + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportError(id_tok, "expected identifier"); + } + expr = ast_factory_.NewUnspecified(NextId(id_tok)); + } else { + std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/false); + if (cel::internal::LexisIsReserved(id_text)) { + ReportError(id_tok, + absl::StrFormat("reserved identifier: %s", id_text)); + } + std::string name = + leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); + int64_t id = NextId(leading_dot ? first_tok : id_tok); + if (peek_token_.type == TokenType::kLeftParen) { + NextToken(); + std::vector args = ParseArguments(TokenType::kRightParen); + expr = ast_factory_.NewCall(id, name, std::move(args)); + } else { + expr = ast_factory_.NewIdent(id, std::move(name)); + } + } + } else { + Token bad_tok = NextToken(); + if (bad_tok.type != TokenType::kError) { + if (bad_tok.type == TokenType::kEnd) { + ReportError( + bad_tok, + "Syntax error: mismatched input '' expecting expression"); + } else { + ReportError(bad_tok, "unexpected token"); + } + } + expr = ast_factory_.NewUnspecified(NextId(bad_tok)); + } + + return expr; +} + +// Parses list creation literals (e.g., `[1, 2, ?3]`). +template +ExprNode PrattParserWorker::ParseList() { + Token open_tok = NextToken(); + int64_t list_id = NextId(open_tok); + ListNodeBuilder builder = ast_factory_.NewListBuilder(list_id); + while (peek_token_.type != TokenType::kRightBracket && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + ExprNode elem = ParseExpr(); + builder.Add(std::move(elem), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBracket, "expected ']'"); + return builder.Build(); +} + +// Parses map creation literals (e.g., `{"key": "value", ?"opt_key": 42}`). +template +ExprNode PrattParserWorker::ParseMap() { + Token open_tok = NextToken(); + int64_t map_id = NextId(open_tok); + MapNodeBuilder builder = ast_factory_.NewMapBuilder(map_id); + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + Token key_start = peek_token_; + if (key_start.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + key_start = peek_token_; + } + ExprNode key = ParseExpr(); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in map entry")) { + break; + } + int64_t entry_id = NextId(colon); + ExprNode val = ParseExpr(); + builder.Add(entry_id, std::move(key), std::move(val), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return builder.Build(); +} + +// Parses struct literas for a type name (e.g., `Msg{field: 10, ?opt: "val"}`). +template +ExprNode PrattParserWorker::ParseStruct( + int64_t obj_id, absl::string_view struct_name) { + Token open_tok = NextToken(); + StructNodeBuilder builder = + ast_factory_.NewStructBuilder(obj_id, std::string(struct_name)); + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + Token field_tok = NextToken(); + if (field_tok.type != TokenType::kIdent && + field_tok.type != TokenType::kReservedWord) { + ReportError(field_tok, "expected struct field name"); + SynchronizeOnDelimiter(); + break; + } + std::string field_name = NormalizeIdent(field_tok, /*allow_quoted=*/true); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in struct field")) { + break; + } + int64_t field_id = NextId(colon); + ExprNode val = ParseExpr(); + builder.Add(field_id, std::move(field_name), std::move(val), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return builder.Build(); +} + +// Parses comma-separated arguments (e.g., `(arg1, arg2)` in call). +template +std::vector PrattParserWorker::ParseArguments( + TokenType close_token) { + std::vector args; + if (peek_token_.type != close_token && peek_token_.type != TokenType::kEnd) { + while (true) { + args.push_back(ParseExpr()); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + if (peek_token_.type == close_token) { + break; + } + continue; + } + break; + } + } + Expect(close_token); + return args; +} + +// Parses decimal & hexadecimal ints (e.g., `42`, `0x1A`). +template +ExprNode PrattParserWorker::ParseIntLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + int64_t int_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + } else if (absl::SimpleAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + ReportError(tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses unsigned ints (e.g., `42u`, `0x1Au`). +template +ExprNode PrattParserWorker::ParseUintLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + if (!value.empty() && (value.back() == 'u' || value.back() == 'U')) { + value.pop_back(); + } + uint64_t uint_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + } else if (absl::SimpleAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + ReportError(tok, "invalid uint literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses floating-point numbers (e.g., `3.14159`, `1e-10`). +template +ExprNode PrattParserWorker::ParseDoubleLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + double double_val = 0.0; + if (absl::SimpleAtod(value, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(tok), double_val); + } + ReportError(tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses string literals (e.g., `"hello"`, `'world'`, `"""multi"""`). +template +ExprNode PrattParserWorker::ParseStringLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseStringLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewStringConst(NextId(tok), std::move(*status_or_val)); +} + +// Parses byte sequence literals (e.g., `b"hello"`, `b'world'`). +template +ExprNode PrattParserWorker::ParseBytesLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseBytesLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewBytesConst(NextId(tok), std::move(*status_or_val)); +} + +// Normalizes regular & quoted identifiers (e.g., `foo`, `` `quoted.ident` ``). +template +std::string PrattParserWorker::NormalizeIdent(const Token& tok, + bool allow_quoted) { + std::string text = GetTokenText(tok); + if (text.empty()) return ""; + if (text.front() == '`') { + if (!allow_quoted) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + if (!options_.enable_quoted_identifiers) { + ReportError(tok, "unsupported syntax '`'"); + } + if (text.size() < 2 || text.back() != '`') { + ReportError(tok, "unterminated quoted identifier"); + return ""; + } + // Validate the quoted identifier syntax: + // ESC_IDENTIFIER : '`' (LETTER | DIGIT | '_' | '.' | '-' | '/' | ' ')+ '`'; + std::string_view inner(text.data() + 1, text.size() - 2); + if (inner.empty()) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + for (char c : inner) { + if (!absl::ascii_isalnum(static_cast(c)) && c != '_' && + c != '.' && c != '-' && c != '/' && c != ' ') { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + } + return std::string(inner); + } + return std::string(text); +} + +// Extracts qualified type names (e.g., `a.b.Msg` from `a.b.Msg{}`). +template +std::optional PrattParserWorker::ExtractStructName( + const ExprNode& expr) { + if (ast_factory_.IsConst(expr)) { + return std::nullopt; + } + if (ast_factory_.IsIdent(expr)) { + std::string name(ast_factory_.GetIdentName(expr)); + EraseId(ast_factory_.GetId(expr)); + return name; + } + if (ast_factory_.IsSelect(expr)) { + if (ast_factory_.IsPresenceTest(expr)) return std::nullopt; + const ExprNode* operand = ast_factory_.GetSelectOperand(expr); + if (operand == nullptr) return std::nullopt; + EraseId(ast_factory_.GetId(expr)); + absl::optional prefix = ExtractStructName(*operand); + if (!prefix) return std::nullopt; + std::string name = + absl::StrCat(*prefix, ".", ast_factory_.GetSelectField(expr)); + return name; + } + return std::nullopt; +} + +template +int32_t PrattParserWorker::GetLeftmostPosition(const ExprNode& expr) { + if (ast_factory_.IsIdent(expr)) { + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; + } + if (ast_factory_.IsSelect(expr)) { + return GetLeftmostPosition(*ast_factory_.GetSelectOperand(expr)); + } + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; +} + +// Recursively constructs a balanced binary AST tree for chained associative +// operators (e.g., chains of `+` or `*`). To prevent deep recursion and +// stack overflow during evaluation of expressions like `a + b + c + d`, this +// method splits the operand terms in half at midpoint `(lo + hi + 1) / 2` and +// combines the left and right subtrees with binary call nodes. +template +ExprNode PrattParserWorker::BalancedTree( + absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi) { + int mid = (lo + hi + 1) / 2; + std::vector arguments; + arguments.reserve(2); + if (mid == lo) { + arguments.push_back(std::move(terms[mid])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, lo, mid - 1)); + } + if (mid == hi) { + arguments.push_back(std::move(terms[mid + 1])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, mid + 1, hi)); + } + return ast_factory_.NewCall(ops[mid], std::string(op), std::move(arguments)); +} + +// Constructs the AST representation for chained logical operators (e.g., +// `a && b && c` or `a || b || c`). When variadic logical ops are enabled +// (`enable_variadic = true`), it creates a single N-ary function call node +// containing all terms as arguments (e.g., `_&&_(a, b, c)`). Otherwise, it +// delegates to `BalancedTree` to produce a balanced binary tree of logical +// operations `(a && b) && c`. +template +ExprNode PrattParserWorker::BalanceLogical( + absl::string_view op, std::vector terms, std::vector ops, + bool enable_variadic) { + if (terms.size() == 1) { + return std::move(terms[0]); + } + if (enable_variadic) { + return ast_factory_.NewCall(ops[0], std::string(op), std::move(terms)); + } + return BalancedTree(op, terms, ops, 0, ops.size() - 1); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_