diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 569d88555..09c538bef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,20 @@ jobs: - name: Check for grammar ambiguities run: ./gradlew checkGrammarAmbiguity --no-daemon + - name: Report changed lookahead contexts + env: + LOOKAHEAD_BASE: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + run: ./gradlew lookaheadAudit --no-daemon -PlookaheadBase="$LOOKAHEAD_BASE" + + - name: Upload lookahead review report + if: always() + uses: actions/upload-artifact@v5 + with: + name: lookahead-review + path: build/reports/lookahead/ + retention-days: 7 + if-no-files-found: ignore + - name: Upload grammar warning report if: failure() uses: actions/upload-artifact@v5 @@ -216,4 +230,4 @@ jobs: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v5 \ No newline at end of file + uses: actions/deploy-pages@v5 diff --git a/build.gradle b/build.gradle index b7a311274..1f4cae69e 100644 --- a/build.gradle +++ b/build.gradle @@ -846,3 +846,23 @@ tasks.register('checkGrammarAmbiguity') { tasks.named('check') { dependsOn('checkGrammarAmbiguity') } + +// Report base/working-tree context changes without making historical lookahead debt a new gate. +// The analyzer lives in test sources: no audit classes or dependencies enter the library JAR. +tasks.register('lookaheadAudit', JavaExec) { + group = 'verification' + description = 'Reports changed lookahead decisions and stale evidence against a Git base.' + dependsOn(testClasses) + classpath = sourceSets.test.runtimeClasspath + mainClass = 'net.sf.jsqlparser.parser.audit.LookaheadAudit' + javaLauncher = javaToolchains.launcherFor(java.toolchain) + jvmArgs '-Xss4m', '-Xmx1g', '-Dfile.encoding=UTF-8' + workingDir = projectDir + doFirst { + args = [projectDir.absolutePath, + providers.gradleProperty('lookaheadBase').getOrElse('HEAD'), + layout.buildDirectory.dir('reports/lookahead').get().asFile.absolutePath, + file('config/lookahead-evidence.tsv').absolutePath, + providers.gradleProperty('lookaheadFailOnFindings').getOrElse('false')] + } +} diff --git a/config/lookahead-evidence.tsv b/config/lookahead-evidence.tsv new file mode 100644 index 000000000..438564f68 --- /dev/null +++ b/config/lookahead-evidence.tsv @@ -0,0 +1,2 @@ +# keycontext fingerprintreasonregression test +ConnectionOrCloudConnectionDefinition f0d3e3f23a41c3cf239c643bae45e52bfadc80e117be4d15cb8af2b7f54931bb AT conn and AT CLOUD share the first token; retain the second-token decision. LookaheadCleanupTest.preservesCloudAndOrdinaryConnections diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 2210dbe9d..29c3115a6 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -1265,6 +1265,17 @@ public class CCJSqlParser extends AbstractJSqlParser { } } + /** + * Both named-parameter forms match only a single-token name (or K_OUTER) directly followed + * by their assignment token; the 2-token lookahead fails for anything else, so this first + * check is equivalent to running that scan. + */ + private boolean isNamedParameterAssignmentAhead() { + int kind = getToken(2).kind; + return kind == K_ORACLE_NAMED_PARAMETER_ASSIGNMENT + || kind == K_POSTGRES_NAMED_PARAMETER_ASSIGNMENT; + } + /** Distinguish EXISTS identifiers from predicates before suffixes and clause boundaries. */ private boolean isExistsIdentifierAhead() { if (getToken(1).kind != K_EXISTS) { @@ -4548,7 +4559,6 @@ DBMSDestination DBMSDestination() #DBMSDestination: { connectionDefinition = ConnectionDefinition() { dbmsDestination.setConnectionDefinition(connectionDefinition); } ( - LOOKAHEAD(3) table = Table() { dbmsDestination.setTable(table); } [ LOOKAHEAD(2) columns = ParenthesedColumnList() { dbmsDestination.setColumns(columns); } ] [ LOOKAHEAD(2) dbmsTableDestinationOptions = DBMSTableDestinationOptionList() { dbmsDestination.setDBMSTableDestinationOptions(dbmsTableDestinationOptions); } ] @@ -11048,7 +11058,11 @@ Expression Condition(): LOOKAHEAD(2, { !isExistsIdentifierAhead() }) result=ExistsExpression() | - [ LOOKAHEAD(2, { getToken(1).kind == K_PRIOR }) { oraclePrior = EqualsTo.ORACLE_PRIOR_START; } ] + // last alternative of the choice: entering it and letting the productions + // themselves reject is equivalent to scanning first, so skip the deep + // lookahead simulation (this path runs for every plain condition) + LOOKAHEAD({ !interrupted }) + [ LOOKAHEAD({ getToken(1).kind == K_PRIOR }) { oraclePrior = EqualsTo.ORACLE_PRIOR_START; } ] left=SimpleExpression() { result = left; } // Consume Oracle (+) once, before dispatching @@ -11198,10 +11212,13 @@ Expression SQLCondition(): | LOOKAHEAD(2, { !isExistsIdentifierAhead() }) result=ExistsExpression() - | left = SimpleExpression() { result = left; } + | + // last alternative of the choice: skip the redundant lookahead simulation + LOOKAHEAD({ !interrupted }) + left = SimpleExpression() { result = left; } [ LOOKAHEAD(2) ( - LOOKAHEAD(2, ) result=OverlapsCondition(left) + LOOKAHEAD({ getToken(1).kind == K_OVERLAPS }) result=OverlapsCondition(left) | LOOKAHEAD(3, {!interrupted}) result=InExpression(left) | @@ -11663,9 +11680,9 @@ ExpressionList ComplexExpressionList(): } { ( - LOOKAHEAD(2) expr=OracleNamedFunctionParameter() + LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=OracleNamedFunctionParameter() | - LOOKAHEAD(2) expr=PostgresNamedFunctionParameter() + LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=PostgresNamedFunctionParameter() | expr=Expression() ) @@ -11676,9 +11693,9 @@ ExpressionList ComplexExpressionList(): ( LOOKAHEAD(2, {!interrupted}) "," ( - LOOKAHEAD(2) expr=OracleNamedFunctionParameter() + LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=OracleNamedFunctionParameter() | - LOOKAHEAD(2) expr=PostgresNamedFunctionParameter() + LOOKAHEAD(2, { isNamedParameterAssignmentAhead() }) expr=PostgresNamedFunctionParameter() | LOOKAHEAD( RelObjectName() "->", { !isJsonArrowDialect() } ) expr=LambdaExpression() | @@ -12094,7 +12111,8 @@ Expression PrimaryExpression() #PrimaryExpression: //| LOOKAHEAD(FunctionAllColumns()) retval=FunctionAllColumns() // support timestamp expressions - | LOOKAHEAD(2, {!interrupted}) (token= | token=) { retval = new TimeKeyExpression(token.image); } + | LOOKAHEAD({ !interrupted && (getToken(1).kind == K_TIME_KEY_EXPR || getToken(1).kind == K_CURRENT) }) + (token= | token=) { retval = new TimeKeyExpression(token.image); } | LOOKAHEAD(2, {!interrupted}) retval=DateTimeLiteralExpression() @@ -12114,7 +12132,7 @@ Expression PrimaryExpression() #PrimaryExpression: | LOOKAHEAD(2, { !interrupted && getToken(1).kind == K_KEY && getToken(2).kind != OPENING_BRACKET }) retval=KeyExpression() - | LOOKAHEAD(2, {!interrupted}) { retval = new AllValue(); } + | LOOKAHEAD({ !interrupted && getToken(1).kind == K_ALL }) { retval = new AllValue(); } | LOOKAHEAD({ !interrupted && isCharacterSetIntroducedStringAhead() }) retval=CharacterSetIntroducedString() @@ -12128,7 +12146,8 @@ Expression PrimaryExpression() #PrimaryExpression: "(" "+" ")" { ((Column) retval).setOldOracleJoinSyntax(EqualsTo.ORACLE_JOIN_RIGHT); } ] - | LOOKAHEAD(2, {!interrupted}) (token= | token=) { retval = new BooleanValue(token.image); } + | LOOKAHEAD({ !interrupted && (getToken(1).kind == K_TRUE || getToken(1).kind == K_FALSE) }) + (token= | token=) { retval = new BooleanValue(token.image); } | retval=CharacterLiteral() { linkAST(retval,jjtThis); } diff --git a/src/site/sphinx/contribution.rst b/src/site/sphinx/contribution.rst index 41c327fbb..f7e3a3758 100644 --- a/src/site/sphinx/contribution.rst +++ b/src/site/sphinx/contribution.rst @@ -120,6 +120,31 @@ When adding a new keyword token to the Grammar: 2) If the keyword must be reserved (e. |_| g. core SQL syntax like ``SELECT``, ``FROM``, ``WHERE``), add it to the ``Reserved SQL Keywords`` TOKEN block **after** the ``MAX_NON_RESERVED_WORD`` sentinel. 3) Verify that existing tests pass and that the keyword can be used as a ``Schema``, ``Table``, ``Column``, ``Function`` or ``Alias`` name where expected. +Review Lookahead Changes +------------------------------ + +Run the reusable lookahead report against the commit used as the review baseline: + +.. code-block:: shell + + ./gradlew lookaheadAudit -PlookaheadBase=HEAD + +The default baseline is ``HEAD``, so this command also examines uncommitted grammar changes. CI uses the pull request base commit. Results are written to ``build/reports/lookahead/report.json``. The analyzer runs with the project's existing JDK and JavaCC dependencies; it does not require Python or modify sources. + +The machine calculation is fresh for both versions. Review the changed decisions, dependency roots, and stale evidence in the summary instead of manually rechecking every unchanged lookahead. Structural anchors ignore source line numbers and unrelated lookahead insertions. Identical anchors remain ambiguous and cannot reuse an individual evidence record. + +The report computes lexical FIRST/nullable/FOLLOW facts and follows direct and indirect production calls. Nullable competing alternatives contribute FOLLOW to the decision; unknown actions on either side prevent a disjoint-prefix suggestion. A changed callee can invalidate evidence even when the local FIRST set stays the same; a caller's nullable suffix can change a callee's FOLLOW context. Semantic predicates, Java actions and production declarations are treated conservatively, rather than assumed equivalent. + +All checked-in production Java files, runtime resources, ``build.gradle``, ``settings.gradle``, ``gradle.properties``, ``pom.xml`` and Gradle wrapper properties are monitored as opaque inputs. The dedicated cleanup regression test, analyzer sources and resolved JavaCC core/Java generator JARs are also bound to the evidence. Unrecognized watched symlinks fail the check instead of being skipped. Generated parser behavior is bound to the grammar and generator JARs; commit-derived build/version metadata is excluded. Evidence covers tracked static inputs and tested behavior, not every possible runtime configuration. + +``config/lookahead-evidence.tsv`` stores a small number of investigated decisions, their exact context fingerprint, reason and regression reference. It is not an approval of all historical sites or permission to publish changes. The report writes current unambiguous bindings to ``binding-candidates.tsv``; review and rerun the referenced tests before updating a stale record. A production name is accepted only when it identifies one decision; otherwise use a unique structural anchor or disambiguate the case. + +Pure one-terminal probes can suggest an equivalent token-kind condition. Non-nullable branches with disjoint FIRST sets can suggest default dispatch instead of a deeper scan. These are reusable syntactic rules, not an equivalence theorem for imperative JavaCC code. Retain state/dialect/interruption checks, verify complete consumption and AST behavior, and test malformed neighbors before adopting a candidate. + +Error locations can change when a rejected branch is entered directly. In the ``DBMSDestination`` cleanup, ``EXA AT conn TABLE`` remains invalid, but now reports the missing table name after consuming ``TABLE`` instead of falling through to the ``STATEMENT`` alternative. The dedicated regression checks this consumption boundary; exact diagnostic text is not promised unchanged. + +The dedicated report step does not turn historical debt into a new mandatory gate. ``-PlookaheadFailOnFindings=true`` opts into failure when contexts need review; generation or analysis errors always fail. ``LookaheadAuditTest`` exercises changed dependencies, nullable/FOLLOW effects, stable anchors, opaque inputs and conservative handling; ordinary ``check`` runs these tests alongside the parser regressions. Full applicable checks and paired project JMH measurements are still required before publishing a cleanup. + Commit a Pull Request --------------------------------- diff --git a/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java b/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java new file mode 100644 index 000000000..6b0057467 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java @@ -0,0 +1,76 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser; + +import static org.junit.jupiter.api.Assertions.*; +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.statement.ConnectionDefinition; +import net.sf.jsqlparser.statement.CloudConnectionDefinition; +import net.sf.jsqlparser.statement.export.DBMSDestination; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class LookaheadCleanupTest { + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn TABLE t", "ORA AT conn TABLE schemaName.tableName", + "JDBC DRIVER = 'driverName' AT conn TABLE \"t\"", + "EXA AT conn TABLE t (c1, c2) REPLACE TRUNCATE"}) + void dispatchesTableDestinationsWithCompleteConsumption(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + DBMSDestination destination = parser.DBMSDestination(); + assertNotNull(destination.getTable()); + assertNull(destination.getStatement()); + assertNotNull(destination.getConnectionDefinition()); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true, + p -> p.withDialect(Dialect.EXASOL)); + } + + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn STATEMENT 'insert into t values (?)'", + "ORA AT conn STATEMENT 'insert into t values (?)'"}) + void preservesStatementDestinations(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + DBMSDestination destination = parser.DBMSDestination(); + assertNull(destination.getTable()); + assertEquals("insert into t values (?)", destination.getStatement().getValue()); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true, + p -> p.withDialect(Dialect.EXASOL)); + } + + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn TABLE", "ORA AT conn STATEMENT", "EXA AT conn SELECT t"}) + void rejectsIncompleteAndWrongDestinationPrefixes(String sql) { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + assertThrows(ParseException.class, parser::DBMSDestination); + if (sql.endsWith("TABLE")) { + assertEquals(CCJSqlParserConstants.K_TABLE, parser.getToken(0).kind); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + } + } + + @ParameterizedTest + @ValueSource(strings = {"AT conn", "AT '127.0.0.1'", "AT CLOUD NONE conn", + "AT CLOUD AZURE BLOBSTORAGE conn"}) + void preservesCloudAndOrdinaryConnections(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + ConnectionDefinition connection = parser.ConnectionOrCloudConnectionDefinition(); + if (sql.startsWith("AT CLOUD")) { + assertInstanceOf(CloudConnectionDefinition.class, connection); + } else { + assertEquals(ConnectionDefinition.class, connection.getClass()); + } + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO CSV " + sql + " FILE 'file.csv'", + true, p -> p.withDialect(Dialect.EXASOL)); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/LookaheadDispatchTest.java b/src/test/java/net/sf/jsqlparser/parser/LookaheadDispatchTest.java new file mode 100644 index 000000000..0aa5ee61d --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/LookaheadDispatchTest.java @@ -0,0 +1,223 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser; + +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.stream.Stream; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.AllValue; +import net.sf.jsqlparser.expression.BooleanValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.OracleNamedFunctionParameter; +import net.sf.jsqlparser.expression.OverlapsCondition; +import net.sf.jsqlparser.expression.PostgresNamedFunctionParameter; +import net.sf.jsqlparser.expression.TimeKeyExpression; +import net.sf.jsqlparser.expression.operators.relational.Between; +import net.sf.jsqlparser.expression.operators.relational.EqualsTo; +import net.sf.jsqlparser.expression.operators.relational.ExistsExpression; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; +import net.sf.jsqlparser.expression.operators.relational.SupportsOldOracleJoinSyntax; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.PlainSelect; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +class LookaheadDispatchTest { + @ParameterizedTest + @MethodSource("singleTokenExpressions") + void preservesSingleTokenExpressionKinds(String sql, Class type, + String deparsed) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + Expression expression = parser.PrimaryExpression(); + assertInstanceOf(type, expression); + assertEquals(deparsed, expression.toString()); + assertEquals(CCJSqlParserConstants.EOF, parser.getNextToken().kind); + Expression parsed = CCJSqlParserUtil.parseExpression(sql, false); + assertInstanceOf(type, parsed); + assertEquals(deparsed, parsed.toString()); + } + + static Stream singleTokenExpressions() { + return Stream.of( + Arguments.of("TRUE", BooleanValue.class, "true"), + Arguments.of("FALSE", BooleanValue.class, "false"), + Arguments.of("ALL", AllValue.class, "ALL"), + Arguments.of("CURRENT", TimeKeyExpression.class, "CURRENT"), + Arguments.of("CURRENT_DATE", TimeKeyExpression.class, "CURRENT_DATE"), + Arguments.of("CURRENT_TIME", TimeKeyExpression.class, "CURRENT_TIME"), + Arguments.of("CURRENT_TIMESTAMP", TimeKeyExpression.class, "CURRENT_TIMESTAMP"), + Arguments.of("CURRENT_TIMEZONE", TimeKeyExpression.class, "CURRENT_TIMEZONE"), + Arguments.of("CURRENT_DATE()", TimeKeyExpression.class, "CURRENT_DATE()")); + } + + @ParameterizedTest + @ValueSource(strings = {"\"TRUE\"", "\"CURRENT_DATE\"", "\"ALL\""}) + void preservesQuotedNames(String name) throws JSQLParserException { + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed("SELECT " + name); + assertEquals(name, assertInstanceOf(Column.class, select.getSelectItem(0).getExpression()) + .getColumnName()); + } + + @ParameterizedTest + @ValueSource(strings = {"TRUE", "ALL", "CURRENT_DATE"}) + void honorsInterruptionBeforeSingleTokenExpressions(String sql) { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + parser.interrupted = true; + assertThrows(ParseException.class, parser::PrimaryExpression); + assertEquals(sql, parser.getNextToken().image); + } + + @ParameterizedTest + @MethodSource("conditions") + void preservesConditionAlternatives(String sql, Class type) + throws Exception { + CCJSqlParser conditionParser = CCJSqlParserUtil.newParser(sql); + Expression condition = conditionParser.Condition(); + assertInstanceOf(type, condition); + assertEquals(sql, condition.toString()); + assertEquals(CCJSqlParserConstants.EOF, conditionParser.getNextToken().kind); + + CCJSqlParser sqlConditionParser = CCJSqlParserUtil.newParser(sql); + Expression sqlCondition = sqlConditionParser.SQLCondition(); + assertInstanceOf(type, sqlCondition); + assertEquals(sql, sqlCondition.toString()); + assertEquals(CCJSqlParserConstants.EOF, sqlConditionParser.getNextToken().kind); + + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed( + "SELECT * FROM t WHERE " + sql); + assertInstanceOf(type, select.getWhere()); + } + + static Stream conditions() { + return Stream.of( + Arguments.of("a", Column.class), + Arguments.of("exists", Column.class), + Arguments.of("exists.col", Column.class), + Arguments.of("EXISTS (SELECT 1)", ExistsExpression.class), + Arguments.of("EXISTS (1)", ExistsExpression.class), + Arguments.of("a IN (1, 2)", InExpression.class), + Arguments.of("a BETWEEN 1 AND 2", Between.class), + Arguments.of("a IS NULL", IsNullExpression.class), + Arguments.of("(a, b) OVERLAPS (c, d)", OverlapsCondition.class)); + } + + @ParameterizedTest + @ValueSource(strings = {"a", "exists", "exists.col"}) + void preservesComparisons(String column) throws Exception { + String sql = column + " = 1"; + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + EqualsTo condition = assertInstanceOf(EqualsTo.class, parser.Condition()); + assertEquals(column, assertInstanceOf(Column.class, condition.getLeftExpression()) + .getFullyQualifiedName()); + assertEquals(sql, condition.toString()); + assertEquals(CCJSqlParserConstants.EOF, parser.getNextToken().kind); + assertSqlCanBeParsedAndDeparsed("SELECT * FROM t WHERE " + sql); + } + + @Test + void preservesOraclePriorPosition() throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser("PRIOR a = b"); + EqualsTo condition = assertInstanceOf(EqualsTo.class, parser.Condition()); + assertEquals(SupportsOldOracleJoinSyntax.ORACLE_PRIOR_START, + condition.getOraclePriorPosition()); + assertEquals("a", assertInstanceOf(Column.class, condition.getLeftExpression()) + .getColumnName()); + assertEquals("b", assertInstanceOf(Column.class, condition.getRightExpression()) + .getColumnName()); + assertEquals(CCJSqlParserConstants.EOF, parser.getNextToken().kind); + assertSqlCanBeParsedAndDeparsed("SELECT * FROM t CONNECT BY PRIOR a = b"); + } + + @ParameterizedTest + @ValueSource(strings = {"a IN (", "a BETWEEN 1 AND", "EXISTS (SELECT", + "(a, b) OVERLAPS ("}) + void rejectsMalformedConditions(String sql) { + assertThrows(ParseException.class, () -> CCJSqlParserUtil.newParser(sql).Condition()); + assertThrows(ParseException.class, () -> CCJSqlParserUtil.newParser(sql).SQLCondition()); + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse("SELECT * FROM t WHERE " + sql)); + } + + @ParameterizedTest + @ValueSource(strings = {"a =", "exists ="}) + void rejectsMalformedComparisons(String sql) { + assertThrows(ParseException.class, () -> CCJSqlParserUtil.newParser(sql).Condition()); + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse("SELECT * FROM t WHERE " + sql)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void honorsInterruptionBeforeConditionDispatch(boolean sqlCondition) { + CCJSqlParser parser = CCJSqlParserUtil.newParser("1"); + parser.interrupted = true; + assertThrows(ParseException.class, () -> { + if (sqlCondition) { + parser.SQLCondition(); + } else { + parser.Condition(); + } + }); + assertEquals("1", parser.getNextToken().image); + } + + @ParameterizedTest + @CsvSource(value = {"arg|=>", "arg|:=", "OUTER|=>", "OUTER|:=", "\"arg\"|=>", + "\"arg\"|:=", "overlaps|=>", "overlaps|:="}, delimiter = '|') + void preservesNamedParameterKindsInEveryArgumentPosition(String name, String assignment) + throws JSQLParserException { + for (String prefix : new String[] {"", "a.b, "}) { + String sql = "SELECT f(" + prefix + name + " " + assignment + " value_col)"; + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sql); + Function function = assertInstanceOf(Function.class, + select.getSelectItem(0).getExpression()); + int index = prefix.isEmpty() ? 0 : 1; + assertEquals(index + 1, function.getParameters().size()); + if (index == 1) { + assertEquals("a.b", assertInstanceOf(Column.class, + function.getParameters().get(0)).getFullyQualifiedName()); + } + Expression argument = function.getParameters().get(index); + Expression value; + if ("=>".equals(assignment)) { + OracleNamedFunctionParameter parameter = + assertInstanceOf(OracleNamedFunctionParameter.class, argument); + assertEquals(name, parameter.getName()); + value = parameter.getExpression(); + } else { + PostgresNamedFunctionParameter parameter = + assertInstanceOf(PostgresNamedFunctionParameter.class, argument); + assertEquals(name, parameter.getName()); + value = parameter.getExpression(); + } + assertEquals("value_col", assertInstanceOf(Column.class, value).getColumnName()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"a.b => 1", "a.b := 1", "1 => 1", "1 := 1", "'a' => 1", + "'a' := 1", "arg =>", "arg :="}) + void rejectsInvalidNamedParametersInEveryArgumentPosition(String argument) { + for (String prefix : new String[] {"", "a.b, "}) { + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse("SELECT f(" + prefix + argument + ")")); + } + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java new file mode 100644 index 000000000..0fef27023 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java @@ -0,0 +1,410 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import org.javacc.parser.*; + +/** Conservative lexical facts for review; never a parser equivalence proof. */ +final class LookaheadAnalysis { + final Context context; + final Map first = new LinkedHashMap<>(); + final Map> follows = new LinkedHashMap<>(); + final IdentityHashMap> localFollows = new IdentityHashMap<>(); + final IdentityHashMap parents = new IdentityHashMap<>(); + final List> rows = new ArrayList<>(); + + static final class First { + final Set tokens = new TreeSet<>(); + boolean nullable; + final Set caveats = new TreeSet<>(); + + void add(First other) { + tokens.addAll(other.tokens); + caveats.addAll(other.caveats); + } + + String signature() { + return tokens + ":" + nullable + ":" + caveats; + } + } + + LookaheadAnalysis(Context context) { + this.context = context; + } + + static List children(Expansion e) { + if (e instanceof Sequence) { + return ((Sequence) e).units; + } + if (e instanceof Choice) { + return ((Choice) e).getChoices(); + } + if (e instanceof ZeroOrOne) { + return List.of(((ZeroOrOne) e).getExpansion()); + } + if (e instanceof ZeroOrMore) { + return List.of(((ZeroOrMore) e).getExpansion()); + } + if (e instanceof OneOrMore) { + return List.of(((OneOrMore) e).getExpansion()); + } + if (e instanceof TryBlock) { + return List.of(((TryBlock) e).exp); + } + return List.of(); // A NonTerminal is a call; do not traverse its target here. + } + + First inspect(Expansion e) { + First f = new First(); + if (e == null) { + f.caveats.add("opaque-production"); + return f; + } + if (e instanceof RegularExpression) { + f.tokens.add(((RegularExpression) e).ordinal); + return f; + } + if (e instanceof Lookahead) { + return inspectLookahead((Lookahead) e, f); + } + if (e instanceof Action) { + f.nullable = true; + f.caveats.add("java-action"); + return f; + } + if (e instanceof NonTerminal) { + return inspectCall((NonTerminal) e, f); + } + if (e instanceof Sequence) { + return inspectSequence((Sequence) e, f); + } + if (e instanceof Choice) { + return inspectChoice((Choice) e, f); + } + return inspectGroup(e, f); + } + + private First inspectLookahead(Lookahead lookahead, First result) { + result.nullable = true; + if (!lookahead.getActionTokens().isEmpty()) { + result.caveats.add("semantic-predicate"); + } + return result; + } + + private First inspectCall(NonTerminal call, First result) { + First target = first.get(call.getName()); + if (target == null) { + result.caveats.add("unresolved-call"); + } else { + result.add(target); + result.nullable = target.nullable; + } + return result; + } + + private First inspectSequence(Sequence sequence, First result) { + result.nullable = true; + for (Expansion child : children(sequence)) { + First current = inspect(child); + result.add(current); + if (!current.nullable) { + result.nullable = false; + break; + } + } + return result; + } + + private First inspectChoice(Choice choice, First result) { + for (Expansion child : children(choice)) { + First current = inspect(child); + result.add(current); + result.nullable |= current.nullable; + } + return result; + } + + private First inspectGroup(Expansion e, First f) { + List cs = children(e); + if (!cs.isEmpty()) { + First c = inspect(cs.get(0)); + f.add(c); + f.nullable = c.nullable; + if (e instanceof ZeroOrOne || e instanceof ZeroOrMore) { + f.nullable = true; + } + if (e instanceof TryBlock) { + f.caveats.add("exception-recovery"); + } + } else { + f.caveats.add("opaque-" + e.getClass().getSimpleName()); + } + return f; + } + + void computeFirst() { + for (NormalProduction p : context.globals().bnfproductions) { + first.put(p.getLhs(), new First()); + } + boolean changed; + do { + changed = false; + for (NormalProduction p : context.globals().bnfproductions) { + First f = inspect(p.getExpansion()); + if (!f.signature().equals(first.get(p.getLhs()).signature())) { + first.put(p.getLhs(), f); + changed = true; + } + } + } while (changed); + } + + void recordParents(Expansion e, Expansion parent) { + if (e == null) { + return; + } + parents.put(e, parent); + for (Expansion child : children(e)) { + recordParents(child, e); + } + } + + boolean propagate(Expansion e, Set after) { + if (e == null) { + return false; + } + boolean changed = localFollows.computeIfAbsent(e, k -> new TreeSet<>()).addAll(after); + if (e instanceof NonTerminal) { + changed |= follows.get(((NonTerminal) e).getName()).addAll(after); + } else { + if (e instanceof Sequence) { + Set next = new TreeSet<>(after); + List cs = children(e); + for (int i = cs.size() - 1; i >= 0; i--) { + Expansion c = cs.get(i); + changed |= propagate(c, next); + First cf = inspect(c); + Set n = new TreeSet<>(cf.tokens); + if (cf.nullable) { + n.addAll(next); + } + next = n; + } + } else { + Set next = new TreeSet<>(after); + if (e instanceof ZeroOrMore || e instanceof OneOrMore) { + next.addAll(inspect(e).tokens); + } + for (Expansion c : children(e)) { + changed |= propagate(c, next); + } + } + } + return changed; + } + + void computeFollows() { + // All generated BNF methods are callable fragment entry points. -1 means unknown caller + // continuation, rather than pretending that Statement is the only grammar entry point. + for (String name : first.keySet()) { + follows.put(name, new TreeSet<>(Set.of(-1))); + } + boolean changed; + do { + changed = false; + for (NormalProduction p : context.globals().bnfproductions) { + changed |= propagate(p.getExpansion(), follows.get(p.getLhs())); + } + } while (changed); + } + + boolean oneTokenOnly(Expansion e) { + if (e instanceof RegularExpression) { + return true; + } + if (e instanceof Sequence) { + List real = new ArrayList<>(); + for (Expansion c : children(e)) { + if (c instanceof Lookahead && !((Lookahead) c).isExplicit() + && ((Lookahead) c).getActionTokens().isEmpty()) { + continue; + } + real.add(c); + } + return real.size() == 1 && oneTokenOnly(real.get(0)); + } + if (e instanceof Choice) { + return !children(e).isEmpty() && children(e).stream().allMatch(this::oneTokenOnly); + } + return false; + } + + List tokenNames(Set tokens) { + List names = new ArrayList<>(); + for (int n : tokens) { + if (n == -1) { + names.add(""); + continue; + } + if (n == 0) { + names.add(""); + continue; + } + RegularExpression re = context.globals().rexps_of_tokens.get(n); + if (re != null && !re.label.isEmpty()) { + names.add("<" + re.label + ">"); + } else { + if (re instanceof RStringLiteral) { + names.add("\"" + ((RStringLiteral) re).image + "\""); + } else { + names.add(""); + } + } + } + return names; + } + + void visit(Expansion e, String production, Set seen) { + if (e == null || !seen.add(e)) { + return; + } + if (e instanceof Lookahead && ((Lookahead) e).isExplicit()) { + recordLookahead((Lookahead) e, production); + } + for (Expansion child : children(e)) { + visit(child, production, seen); + } + } + + private First competitors(Expansion decision, Expansion branch) { + First result = new First(); + if (decision instanceof Choice) { + Set selected = Collections.newSetFromMap(new IdentityHashMap<>()); + selected.add(branch); + for (Expansion other : children(decision)) { + if (!selected.contains(other)) { + addCompetitor(result, other, decision); + } + } + return result; + } + if (decision instanceof ZeroOrOne || decision instanceof ZeroOrMore + || decision instanceof OneOrMore) { + result.tokens.addAll(localFollows.getOrDefault(decision, Set.of(-1))); + } else { + result.tokens.add(-1); + } + return result; + } + + private void addCompetitor(First result, Expansion other, Expansion decision) { + First competing = inspect(other); + result.add(competing); + if (competing.nullable) { + result.tokens.addAll(localFollows.getOrDefault(decision, Set.of(-1))); + } + } + + private String triage(Lookahead la, Expansion decision, First branch, First competitors, + Set overlap, String semantic) { + if (!semantic.isEmpty()) { + return la.getAmount() == 0 ? "retain-state-or-policy" : "retain-combined-guard"; + } + if (oneTokenOnly(la.getLaExpansion())) { + return "review-exact-token-probe"; + } + if (la.getAmount() == 1) { + return "review-default-depth"; + } + if (decision instanceof Choice && !branch.nullable && overlap.isEmpty() + && branch.caveats.isEmpty() && competitors.caveats.isEmpty() + && !competitors.tokens.contains(-1)) { + return "review-disjoint-FIRST1"; + } + return "review-shared-prefix-or-unknown"; + } + + private void recordLookahead(Lookahead la, String production) { + Expansion sequence = parents.get(la); + Expansion decision = parents.get(sequence); + First branch = inspect(sequence); + First competing = competitors(decision, sequence); + Set overlap = new TreeSet<>(branch.tokens); + overlap.retainAll(competing.tokens); + String semantic = la.getActionTokens().stream().map(t -> t.image) + .collect(java.util.stream.Collectors.joining(" ")); + Map row = new TreeMap<>(); + row.put("production", production); + row.put("line", la.getLine()); + row.put("column", la.getColumn()); + row.put("amount", la.getAmount()); + row.put("semantic", semantic); + row.put("triage", triage(la, decision, branch, competing, overlap, semantic)); + row.put("probeFirst", tokenNames(inspect(la.getLaExpansion()).tokens)); + row.put("decision", decision == null ? "unknown" : decision.getClass().getSimpleName()); + row.put("branchFirst", tokenNames(branch.tokens)); + row.put("nullable", branch.nullable); + row.put("caveats", branch.caveats); + row.put("competitorFirst", tokenNames(competing.tokens)); + row.put("competitorCaveats", competing.caveats); + row.put("overlap", tokenNames(overlap)); + rows.add(row); + } + + static String sha(String value) { + return sha(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + static String sha(byte[] value) { + try { + byte[] bytes = java.security.MessageDigest.getInstance("SHA-256").digest(value); + StringBuilder result = new StringBuilder(); + for (byte b : bytes) { + result.append(String.format(Locale.ROOT, "%02x", b)); + } + return result.toString(); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + static String json(Object value) { + if (value == null || value instanceof Boolean || value instanceof Number) { + return String.valueOf(value); + } + if (value instanceof Map) { + List parts = new ArrayList<>(); + ((Map) value).forEach((k, v) -> parts.add(json(k.toString()) + ":" + json(v))); + return "{" + String.join(",", parts) + "}"; + } + if (value instanceof Collection) { + List parts = new ArrayList<>(); + for (Object item : (Collection) value) { + parts.add(json(item)); + } + return "[" + String.join(",", parts) + "]"; + } + StringBuilder result = new StringBuilder("\""); + for (char ch : value.toString().toCharArray()) { + if (ch == '"' || ch == '\\') { + result.append('\\').append(ch); + } else { + if (ch < 32) { + result.append(String.format(Locale.ROOT, "\\u%04x", (int) ch)); + } else { + result.append(ch); + } + } + } + return result.append('"').toString(); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java new file mode 100644 index 000000000..5e3e0b535 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java @@ -0,0 +1,234 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import java.util.stream.Collectors; + +/** Repository entry point: read git and files, calculate both snapshots, write a review report. */ +public final class LookaheadAudit { + private static final String GRAMMAR = + "src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt"; + private static final String TOOL = "src/test/java/net/sf/jsqlparser/parser/audit/"; + private static final Set BUILD_INPUTS = Set.of("build.gradle", "settings.gradle", + "gradle.properties", "pom.xml", "gradle/wrapper/gradle-wrapper.properties", + "src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java"); + + private LookaheadAudit() {} + + public static void main(String[] args) throws Exception { + if (args.length < 4) { + throw new IllegalArgumentException( + "root base-ref report-directory evidence.tsv [fail-on-findings]"); + } + Path root = Path.of(args[0]).toAbsolutePath(); + Path output = Path.of(args[2]).toAbsolutePath(); + + Files.createDirectories(output); + String base = baseCommit(root, args[1]); + String engine = engine(root); + Map beforeInputs = baseInputs(root, base); + Map afterInputs = workingInputs(root); + + System.out.println("Calculating base lookahead facts at " + base); + LookaheadSnapshot before = + generate(git(root, "show", base + ":" + GRAMMAR), output.resolve("base"), + LookaheadAnalysis.sha(LookaheadAnalysis.json(beforeInputs)), engine); + System.out.println("Calculating working-tree lookahead facts"); + LookaheadSnapshot after = + generate(Files.readString(root.resolve(GRAMMAR)), output.resolve("current"), + LookaheadAnalysis.sha(LookaheadAnalysis.json(afterInputs)), engine); + Evidence evidence = readEvidence(Path.of(args[3]), after.contexts()); + Map report = LookaheadSnapshot.compare(before, after, evidence.bindings); + report.put("baseCommit", base); + report.put("watchedBefore", beforeInputs); + report.put("watchedCurrent", afterInputs); + report.put("guarantee", + "Tracked-context change detection only; no automatic rewrite or semantic equivalence claim."); + Files.writeString(output.resolve("report.json"), LookaheadAnalysis.json(report) + "\n"); + Files.write(output.resolve("binding-candidates.tsv"), evidence.proposed); + System.out.println("Lookahead report: " + output.resolve("report.json")); + if (args.length == 5 && Boolean.parseBoolean(args[4]) + && Boolean.TRUE.equals(report.get("needsReview"))) { + throw new IllegalStateException("Lookahead contexts require review; see report.json"); + } + } + + private static String baseCommit(Path root, String argument) throws Exception { + if (!git(root, "rev-parse", "--show-object-format").trim().equals("sha1")) { + throw new IllegalArgumentException( + "This repository entry point requires Git SHA-1 objects"); + } + String ref = argument.matches("0+") ? "HEAD" : argument; + return git(root, "rev-parse", "--verify", "--end-of-options", ref + "^{commit}").trim(); + } + + private static Evidence readEvidence(Path path, + Map>> contexts) throws IOException { + Evidence evidence = new Evidence(); + for (String line : Files.readAllLines(path)) { + if (!line.isBlank() && !line.startsWith("#")) { + evidence.bind(line, contexts); + } + } + return evidence; + } + + private static final class Evidence { + private final Map bindings = new TreeMap<>(); + private final List proposed = new ArrayList<>(); + + private void bind(String line, Map>> contexts) { + String[] fields = line.split("\t", 4); + if (fields.length != 4 || fields[2].isBlank() || fields[3].isBlank()) { + throw new IllegalArgumentException( + "Evidence requires key, fingerprint, reason and test reference"); + } + List matching = contexts.keySet().stream() + .filter(k -> k.equals(fields[0]) || k.startsWith(fields[0] + ":")) + .collect(Collectors.toList()); + boolean unique = matching.size() == 1; + String key = unique ? matching.get(0) : "unbound:" + fields[0]; + bindings.put(key, fields[1]); + if (unique && contexts.get(key).size() == 1) { + proposed.add(fields[0] + "\t" + contexts.get(key).get(0).get("fingerprint") + "\t" + + fields[2] + "\t" + fields[3]); + } + } + } + + static boolean watched(String path) { + return path.startsWith("src/main/java/") || path.startsWith("src/main/resources/") + || BUILD_INPUTS.contains(path); + } + + static LookaheadSnapshot generate(String source, Path directory, String external, String engine) + throws Exception { + Files.createDirectories(directory); + Path input = directory.resolve("JSqlParserCC.jjt"); + Path generated = directory.resolve("generated"); + + Files.writeString(input, source); + Path java = Path.of(System.getProperty("java.home"), "bin", + System.getProperty("os.name").startsWith("Windows") ? "java.exe" : "java"); + Process process = new ProcessBuilder(java.toString(), "-Dfile.encoding=UTF-8", "-cp", + System.getProperty("java.class.path"), + "org.javacc.jjtree.Main", "-GRAMMAR_ENCODING=UTF-8", + "-OUTPUT_DIRECTORY=" + generated, input.toString()) + .redirectErrorStream(true).redirectOutput(directory.resolve("jjtree.log").toFile()) + .start(); + if (process.waitFor() != 0) { + throw new IllegalStateException( + "JJTree failed; see " + directory.resolve("jjtree.log")); + } + return LookaheadSnapshot.parse(Files.readString(generated.resolve("JSqlParserCC.jj")), + external, engine); + } + + private static Map baseInputs(Path root, String base) throws Exception { + Map result = new TreeMap<>(); + for (String entry : git(root, "ls-tree", "-r", "-z", "--full-tree", base).split("\u0000")) { + if (entry.isEmpty()) { + continue; + } + String[] fields = entry.split("\t", 2); + String[] header = fields[0].split(" "); + + if (watched(fields[1])) { + if (!header[1].equals("blob") || !header[0].startsWith("100")) { + throw new IllegalStateException("Unsupported watched input " + fields[1]); + } + result.put(fields[1], header[2]); + } + } + return result; + } + + private static Map workingInputs(Path root) throws Exception { + Map result = new TreeMap<>(); + for (String folder : List.of("src/main/java", "src/main/resources")) { + Path start = root.resolve(folder); + if (!Files.exists(start)) { + continue; + } + try (java.util.stream.Stream paths = Files.walk(start)) { + for (Path path : paths.collect(Collectors.toList())) { + if (Files.isSymbolicLink(path)) { + throw new IllegalStateException("Symlink in watched inputs: " + path); + } + if (Files.isRegularFile(path)) { + addInput(root, path, result); + } + } + } + } + for (String file : BUILD_INPUTS) { + if (Files.exists(root.resolve(file))) { + addInput(root, root.resolve(file), result); + } + } + return result; + } + + private static void addInput(Path root, Path path, Map result) + throws Exception { + if (Files.isSymbolicLink(path)) { + throw new IllegalStateException("Symlink in watched inputs: " + path); + } + byte[] bytes = Files.readAllBytes(path); + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + digest.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII)); + StringBuilder hash = new StringBuilder(); + for (byte b : digest.digest(bytes)) { + hash.append(String.format(Locale.ROOT, "%02x", b)); + } + result.put(root.relativize(path).toString().replace('\\', '/'), hash.toString()); + } + + private static String engine(Path root) throws Exception { + Map inputs = new TreeMap<>(); + try (java.util.stream.Stream files = Files.walk(root.resolve(TOOL))) { + for (Path path : files.filter(Files::isRegularFile).collect(Collectors.toList())) { + inputs.put(root.relativize(path).toString().replace('\\', '/'), + LookaheadAnalysis.sha(Files.readAllBytes(path))); + } + } + int generators = 0; + for (String entry : System.getProperty("java.class.path") + .split(java.util.regex.Pattern.quote(File.pathSeparator))) { + Path artifact = Path.of(entry); + String name = artifact.getFileName().toString(); + if (name.startsWith("javacc-") && name.endsWith(".jar")) { + inputs.put(name, LookaheadAnalysis.sha(Files.readAllBytes(artifact))); + generators++; + } + } + if (generators < 2) { + throw new IllegalStateException("Expected both JavaCC core and Java generator JARs"); + } + return LookaheadAnalysis.sha(LookaheadAnalysis.json(inputs)); + } + + private static String git(Path root, String... arguments) throws Exception { + List command = new ArrayList<>(List.of("git", "-C", root.toString())); + command.addAll(List.of(arguments)); + Process process = + new ProcessBuilder(command).redirectError(ProcessBuilder.Redirect.INHERIT).start(); + byte[] output = process.getInputStream().readAllBytes(); + if (process.waitFor() != 0) { + throw new IllegalStateException("Git read failed: " + command); + } + return new String(output, StandardCharsets.UTF_8); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java new file mode 100644 index 000000000..e0462a158 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java @@ -0,0 +1,183 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.util.*; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class LookaheadAuditTest { + private static final String GRAMMAR = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture { boolean enabled=true; } PARSER_END(Fixture)\n" + + "TOKEN: { | | }\n" + + "void Direct():{} { (LOOKAHEAD(2) Leaf() | ) }\n" + + "void Indirect():{} { (LOOKAHEAD(2) Bridge() | ) }\n" + + "void Bridge():{} { Leaf() }\nvoid Leaf():{} { }\n" + + "void Optional():{} { [LOOKAHEAD(2) ] }\n" + + "void Caller():{} { Optional() Tail() }\nvoid Tail():{} { [] }\n" + + "void Gate():{} { (LOOKAHEAD({enabled}) | ) }\n" + + "void Anchor():{} { [LOOKAHEAD() ] [LOOKAHEAD() ] }\n" + + "void Duplicate():{} { [LOOKAHEAD(2) ] [LOOKAHEAD(2) ] }\n" + + "void Recovery():{} { try { } catch (ParseException error) { enabled=true; } finally { enabled=true; } }\n" + + "void UnknownChoice():{} { (LOOKAHEAD(2) | { enabled=true; } ) }\n" + + "void NullableAlternative():{} { (LOOKAHEAD(2) | []) }\n"; + private static LookaheadSnapshot baseline; + private static Map evidence; + + @BeforeAll + static void baseline() throws Exception { + baseline = parse(GRAMMAR); + evidence = binding(baseline, "Direct"); + } + + private static LookaheadSnapshot parse(String grammar) throws Exception { + return LookaheadSnapshot.parse(grammar, "external", "engine"); + } + + private static Map binding(LookaheadSnapshot snapshot, String production) { + Map>> contexts = snapshot.contexts(); + String key = contexts.keySet().stream().filter(k -> k.startsWith(production + ":")) + .findFirst().orElseThrow(); + return Map.of(key, contexts.get(key).get(0).get("fingerprint").toString()); + } + + private static Map compare(String text) throws Exception { + return LookaheadSnapshot.compare(baseline, parse(text), evidence); + } + + @SuppressWarnings("unchecked") + private static List> findings(Map report) { + return (List>) report.get("changes"); + } + + @Test + void commentsLineMovesAndUnrelatedSyntaxDoNotReopenEvidence() throws Exception { + assertEquals(false, compare("// shifted line\n\n" + GRAMMAR).get("needsReview")); + Map report = compare(GRAMMAR + "void Unrelated():{} { }\n"); + assertEquals(false, report.get("needsReview")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("reusableEvidence")); + } + + @Test + void insertedProbeDoesNotRenumberExistingAnchors() throws Exception { + LookaheadSnapshot changed = parse( + GRAMMAR.replace("void Anchor():{} {", "void Anchor():{} { [LOOKAHEAD() ]")); + assertTrue(changed.contexts().keySet().containsAll(baseline.contexts().keySet())); + assertEquals(1, findings(LookaheadSnapshot.compare(baseline, changed, evidence)).stream() + .filter(r -> r.get("kind").equals("added")).count()); + } + + @Test + void indirectCalleeChangesInvalidateEvidenceEvenWhenFirstIsUnchanged() throws Exception { + Map report = compare( + GRAMMAR.replace("void Leaf():{} { }", "void Leaf():{} { }")); + Map indirect = findings(report).stream() + .filter(r -> r.get("anchor").toString().startsWith("Indirect:")).findFirst() + .orElseThrow(); + assertEquals(false, indirect.get("localFactsChanged")); + assertEquals(List.of("Leaf"), indirect.get("dependencyChanges")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + } + + @Test + void nullableAndFollowChangesReachAnUnchangedCallee() throws Exception { + for (String replacement : List.of("", "[]")) { + Map report = compare(GRAMMAR.replace("void Tail():{} { [] }", + "void Tail():{} { " + replacement + " }")); + Map finding = findings(report).stream() + .filter(r -> r.get("anchor").toString().startsWith("Optional:")).findFirst() + .orElseThrow(); + assertEquals(true, finding.get("localFactsChanged")); + assertEquals(List.of(), finding.get("dependencyChanges")); + } + } + + @Test + void changedDepthAndDeletionAreVisible() throws Exception { + assertTrue(findings(compare(GRAMMAR.replace("LOOKAHEAD(2) Leaf()", "LOOKAHEAD(3) Leaf()"))) + .stream().anyMatch(r -> r.get("anchor").toString().startsWith("Direct:"))); + assertEquals(1, findings(compare(GRAMMAR.replace("[LOOKAHEAD() ]", "[]"))).stream() + .filter(r -> r.get("kind").equals("removed")).count()); + } + + @Test + void parserPolicyLexerExternalAndEngineChangesCannotReuseEvidence() throws Exception { + for (String text : List.of( + GRAMMAR.replace("boolean enabled=true;", "boolean enabled=false;"), + GRAMMAR.replace("", ""))) { + Map report = compare(text); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + assertEquals(List.of(), report.get("changes")); + } + for (LookaheadSnapshot changed : List.of(LookaheadSnapshot.parse(GRAMMAR, "new", "engine"), + LookaheadSnapshot.parse(GRAMMAR, "external", "new"))) { + assertEquals(new ArrayList<>(evidence.keySet()), LookaheadSnapshot + .compare(baseline, changed, evidence).get("invalidatedEvidence")); + } + } + + @Test + void recoveryJavaChangesInvalidateUnrelatedEvidenceConservatively() throws Exception { + Map report = compare(GRAMMAR.replace( + "catch (ParseException error) { enabled=true; } finally { enabled=true; }", + "catch (ParseException error) { enabled=false; } finally { enabled=false; }")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + assertEquals(List.of("opaque-Java-or-predicate"), report.get("globalSignals")); + assertEquals(List.of(), report.get("changes")); + } + + @Test + void ambiguousAndMissingBindingsAreConservative() { + Map duplicate = binding(baseline, "Duplicate"); + assertEquals(new ArrayList<>(duplicate.keySet()), LookaheadSnapshot + .compare(baseline, baseline, duplicate).get("invalidatedEvidence")); + assertEquals(List.of("unbound:missing"), + LookaheadSnapshot.compare(baseline, baseline, Map.of("unbound:missing", "hash")) + .get("invalidatedEvidence")); + } + + @Test + void externalWatchScopeIncludesAllProductionModelsResourcesAndBuildInputs() { + assertTrue(LookaheadAudit.watched("src/main/java/example/AnUnrelatedModel.java")); + assertTrue(LookaheadAudit.watched("src/main/resources/dialect.properties")); + assertTrue(LookaheadAudit.watched("pom.xml")); + assertTrue(LookaheadAudit.watched("build.gradle")); + assertTrue(LookaheadAudit + .watched("src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java")); + assertFalse(LookaheadAudit.watched("build/generated/Parser.java")); + } + + @Test + void predicatesAreRetainedAndNothingIsAutomaticallyRewritten() { + Map gate = baseline.sites.stream() + .filter(r -> r.get("production").equals("Gate")).findFirst().orElseThrow(); + assertEquals("retain-state-or-policy", gate.get("triage")); + assertEquals(0, LookaheadSnapshot.compare(baseline, baseline, evidence) + .get("automaticGrammarChanges")); + } + + @Test + void competingActionsPreventAClaimOfDisjointDispatch() { + Map decision = baseline.sites.stream() + .filter(r -> r.get("production").equals("UnknownChoice")).findFirst().orElseThrow(); + assertEquals("review-shared-prefix-or-unknown", decision.get("triage")); + assertEquals(Set.of("java-action"), decision.get("competitorCaveats")); + } + + @Test + void nullableCompetitorContributesFollowToTheDecision() { + Map decision = baseline.sites.stream() + .filter(r -> r.get("production").equals("NullableAlternative")).findFirst() + .orElseThrow(); + assertEquals("review-shared-prefix-or-unknown", decision.get("triage")); + assertEquals(List.of(""), decision.get("overlap")); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java new file mode 100644 index 000000000..cbbf167c1 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java @@ -0,0 +1,430 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.StringReader; +import java.util.*; +import java.util.stream.Collectors; +import org.javacc.parser.*; + +/** Fresh snapshots deliberately avoid cached FIRST/FOLLOW or Java dependency inferences. */ +final class LookaheadSnapshot { + final Map> productions = new TreeMap<>(); + final List> sites = new ArrayList<>(); + final Map anchors = new HashMap<>(); + String globalHash; + String opaqueHash; + String externalHash; + String engineHash; + + static LookaheadSnapshot parse(String grammar, String external, String engine) + throws Exception { + Context context = new Context(); + JavaCCParser parser = new JavaCCParser(new StringReader(grammar)); + Token first = parser.getToken(1); + parser.javacc_input(context); + Semanticize.start(context); + if (context.errors().get_error_count() != 0) { + throw new IllegalArgumentException("Invalid grammar"); + } + LookaheadSnapshot snapshot = new LookaheadSnapshot(); + snapshot.externalHash = external; + snapshot.engineHash = engine; + Set owned = Collections.newSetFromMap(new IdentityHashMap<>()); + List opaque = new ArrayList<>(); + LookaheadAnalysis facts = new LookaheadAnalysis(context); + for (NormalProduction production : context.globals().bnfproductions) { + snapshot.recordProduction(production, owned, opaque, facts); + } + Collections.sort(opaque); + snapshot.globalHash = globalFingerprint(first, owned); + snapshot.opaqueHash = LookaheadAnalysis.sha(String.join("\n", opaque)); + facts.computeFirst(); + facts.computeFollows(); + for (NormalProduction production : context.globals().bnfproductions) { + facts.visit(production.getExpansion(), production.getLhs(), + Collections.newSetFromMap(new IdentityHashMap<>())); + } + snapshot.recordSites(facts.rows, grammar.startsWith("/*@bgen")); + return snapshot; + } + + private static List productionBody(NormalProduction production, Set owned) { + List body = new ArrayList<>(); + Token token = production.getFirstToken(); + if (token == null || production.getLastToken() == null) { + throw new IllegalStateException("Missing production bounds"); + } + Set end = Collections.newSetFromMap(new IdentityHashMap<>()); + end.add(production.getLastToken()); + while (true) { + owned.add(token); + body.add(token.image); + if (end.contains(token)) { + return body; + } + token = token.next; + if (token == null) { + throw new IllegalStateException("Unclosed production"); + } + } + } + + private void recordProduction(NormalProduction production, Set owned, + List opaque, LookaheadAnalysis facts) { + List body = productionBody(production, owned); + Set calls = new TreeSet<>(); + List code = new ArrayList<>(); + if (production instanceof BNFProduction) { + addCode(code, "declarations", ((BNFProduction) production).getDeclarationTokens()); + } else { + code.add("opaque-production:" + String.join(" ", body)); + } + addCode(code, "parameters", production.getParameterListTokens()); + walk(production.getExpansion(), "", production.getLhs(), null, + Collections.newSetFromMap(new IdentityHashMap<>()), calls, code); + if (!code.isEmpty()) { + opaque.add(production.getLhs() + ":" + LookaheadAnalysis.sha(String.join("\n", code))); + } + Map value = new TreeMap<>(); + value.put("hash", LookaheadAnalysis.sha(String.join(" ", body))); + value.put("calls", calls); + value.put("nullable", production.getExpansion() == null ? "unknown" + : Semanticize.emptyExpansionExists(production.getExpansion())); + productions.put(production.getLhs(), value); + facts.recordParents(production.getExpansion(), null); + } + + private static String globalFingerprint(Token first, Set owned) { + List globals = new ArrayList<>(); + for (Token token = first; token != null && token.kind != 0; token = token.next) { + if (!owned.contains(token)) { + globals.add(token.image); + } + } + return LookaheadAnalysis.sha(String.join(" ", globals)); + } + + private void recordSites(List> rows, boolean generated) { + for (Map row : rows) { + String anchor = anchors.get(location(row)); + if (anchor == null) { + throw new IllegalStateException("Unmapped explicit lookahead " + row); + } + row.put("anchor", anchor); + if (generated) { + row.put("line", ((Integer) row.get("line")) - 1); + } + sites.add(row); + } + int siteCount = sites.size(); + int anchorCount = anchors.size(); + if (siteCount != anchorCount) { + throw new IllegalStateException("Lookahead count mismatch"); + } + } + + private static String tokens(List tokens) { + return tokens.stream().map(t -> t.image).collect(Collectors.joining(" ")); + } + + private static void addCode(List target, String kind, List tokens) { + if (!tokens.isEmpty()) { + target.add(kind + ":" + tokens(tokens)); + } + } + + private static String shape(Expansion expansion) { + if (expansion instanceof Lookahead) { + return ""; + } + if (expansion instanceof RegularExpression) { + RegularExpression re = (RegularExpression) expansion; + if (!re.label.isEmpty()) { + return "T:" + re.label; + } + return re instanceof RStringLiteral + ? "L:" + LookaheadAnalysis.json(((RStringLiteral) re).image) + : "R:" + re.getClass().getSimpleName(); + } + if (expansion instanceof NonTerminal) { + NonTerminal nt = (NonTerminal) expansion; + return "N:" + nt.getName() + "(" + tokens(nt.getArgumentTokens()) + ")"; + } + if (expansion instanceof Action) { + return "Action"; + } + return expansion.getClass().getSimpleName() + LookaheadAnalysis.children(expansion).stream() + .map(LookaheadSnapshot::shape).filter(s -> !s.isEmpty()) + .collect(Collectors.joining(",", "[", "]")); + } + + private void walk(Expansion expansion, String path, String name, Expansion parent, + Set seen, Set calls, List code) { + if (expansion == null || !seen.add(expansion)) { + return; + } + recordExpansionCode(expansion, calls, code); + if (expansion instanceof Lookahead) { + Lookahead la = (Lookahead) expansion; + recordLookahead(la, path, name, parent, code); + walk(la.getLaExpansion(), path + "/probe", name, expansion, seen, calls, code); + } + for (Expansion child : LookaheadAnalysis.children(expansion)) { + walk(child, path + "/" + expansion.getClass().getSimpleName(), name, expansion, seen, + calls, code); + } + } + + private static void recordExpansionCode(Expansion expansion, Set calls, + List code) { + if (expansion instanceof NonTerminal) { + NonTerminal nt = (NonTerminal) expansion; + calls.add(nt.getName()); + addCode(code, "arguments:" + nt.getName(), nt.getArgumentTokens()); + } + if (expansion instanceof Action) { + addCode(code, "action", ((Action) expansion).getActionTokens()); + } + if (expansion instanceof TryBlock) { + recordRecoveryCode((TryBlock) expansion, code); + } + } + + private static void recordRecoveryCode(TryBlock block, List code) { + for (List type : block.types) { + addCode(code, "catch-type", type); + } + for (List body : block.catchblks) { + addCode(code, "catch-body", body); + } + if (block.finallyblk != null) { + addCode(code, "finally-body", block.finallyblk); + } + } + + private void recordLookahead(Lookahead la, String path, String name, Expansion parent, + List code) { + if (!la.isExplicit()) { + return; + } + addCode(code, "predicate", la.getActionTokens()); + String key = name + ":" + la.getLine() + ":" + la.getColumn(); + String anchor = name + ":" + + LookaheadAnalysis.sha(path + "|" + shape(parent)).substring(0, 16); + if (anchors.put(key, anchor) != null) { + throw new IllegalStateException("Duplicate lookahead location"); + } + } + + private static String location(Map row) { + return row.get("production") + ":" + row.get("line") + ":" + row.get("column"); + } + + @SuppressWarnings("unchecked") + Map>> contexts() { + Map>> result = new TreeMap<>(); + for (Map site : sites) { + Set closure = new TreeSet<>(); + Deque pending = new ArrayDeque<>(); + pending.add(site.get("production").toString()); + while (!pending.isEmpty()) { + String name = pending.removeFirst(); + if (!closure.add(name)) { + continue; + } + if (!productions.containsKey(name)) { + throw new IllegalStateException("Unresolved production " + name); + } + pending.addAll((Set) productions.get(name).get("calls")); + } + Map facts = new TreeMap<>(site); + facts.keySet().removeAll(Set.of("line", "column", "anchor")); + Map dependencies = new TreeMap<>(); + for (String name : closure) { + dependencies.put(name, productions.get(name)); + } + Map local = new TreeMap<>(); + local.put("facts", facts); + local.put("dependencies", dependencies); + Map context = new TreeMap<>(local); + context.put("global", globalHash); + context.put("opaque", opaqueHash); + context.put("external", externalHash); + context.put("engine", engineHash); + Map binding = new TreeMap<>(); + binding.put("production", site.get("production")); + binding.put("line", site.get("line")); + binding.put("triage", site.get("triage")); + binding.put("facts", facts); + binding.put("dependencies", dependencies); + binding.put("localFingerprint", LookaheadAnalysis.sha(LookaheadAnalysis.json(local))); + binding.put("fingerprint", LookaheadAnalysis.sha(LookaheadAnalysis.json(context))); + result.computeIfAbsent(site.get("anchor").toString(), k -> new ArrayList<>()) + .add(binding); + } + return result; + } + + static Map compare(LookaheadSnapshot before, LookaheadSnapshot after, + Map evidence) { + Comparison comparison = new Comparison(before.contexts(), after.contexts(), evidence); + comparison.collect(); + return comparison.report(before.sites.size(), after.sites.size(), + globalChanges(before, after)); + } + + private static List globalChanges(LookaheadSnapshot before, LookaheadSnapshot after) { + List globals = new ArrayList<>(); + if (!before.globalHash.equals(after.globalHash)) { + globals.add("parser-options-helpers-or-lexer"); + } + if (!before.opaqueHash.equals(after.opaqueHash)) { + globals.add("opaque-Java-or-predicate"); + } + if (!before.externalHash.equals(after.externalHash)) { + globals.add("production-Java-resources-or-build"); + } + if (!before.engineHash.equals(after.engineHash)) { + globals.add("generator-or-analysis-version"); + } + return globals; + } + + private static final class Comparison { + private final Map>> old; + private final Map>> current; + private final Map evidence; + private final List> changed = new ArrayList<>(); + private final List reused = new ArrayList<>(); + private final List stale = new ArrayList<>(); + private final List ambiguous = new ArrayList<>(); + private final Set direct = new TreeSet<>(); + private final Set roots = new TreeSet<>(); + + private Comparison(Map>> old, + Map>> current, Map evidence) { + this.old = old; + this.current = current; + this.evidence = evidence; + } + + private void collect() { + Set keys = new TreeSet<>(old.keySet()); + keys.addAll(current.keySet()); + keys.addAll(evidence.keySet()); + for (String key : keys) { + List> before = old.getOrDefault(key, List.of()); + List> after = current.getOrDefault(key, List.of()); + String kind = changeKind(key, before, after); + if (kind != null) { + recordFinding(key, kind, before, after); + } + if (evidence.containsKey(key)) { + recordEvidence(key, after); + } + } + } + + private String changeKind(String key, List> before, + List> after) { + if (before.isEmpty() && after.isEmpty()) { + return "unbound-evidence"; + } + if (before.size() > 1 || after.size() > 1) { + ambiguous.add(key); + return localFingerprints(before).equals(localFingerprints(after)) ? null + : "ambiguous-context-changed"; + } + if (before.isEmpty()) { + return "added"; + } + if (after.isEmpty()) { + return "removed"; + } + return before.get(0).get("localFingerprint") + .equals(after.get(0).get("localFingerprint")) + ? null + : "context-changed"; + } + + private static List localFingerprints(List> bindings) { + return bindings.stream().map(x -> x.get("localFingerprint").toString()).sorted() + .collect(Collectors.toList()); + } + + private void recordFinding(String key, String kind, List> before, + List> after) { + Map finding = new TreeMap<>(); + finding.put("anchor", key); + finding.put("kind", kind); + finding.put("beforeLines", + before.stream().map(x -> x.get("line")).collect(Collectors.toList())); + finding.put("afterLines", + after.stream().map(x -> x.get("line")).collect(Collectors.toList())); + List> available = after.isEmpty() ? before : after; + if (!available.isEmpty()) { + finding.put("triage", available.get(0).get("triage")); + } + if (before.size() == 1 && after.size() == 1) { + recordFacts(key, finding, before.get(0), after.get(0)); + } + if (kind.equals("added") || kind.equals("removed")) { + direct.add(key); + } + changed.add(finding); + } + + @SuppressWarnings("unchecked") + private void recordFacts(String key, Map finding, + Map before, + Map after) { + boolean factChange = !before.get("facts").equals(after.get("facts")); + finding.put("localFactsChanged", factChange); + if (factChange) { + direct.add(key); + } + Map oldDependencies = (Map) before.get("dependencies"); + Map newDependencies = (Map) after.get("dependencies"); + Set names = new TreeSet<>(oldDependencies.keySet()); + names.addAll(newDependencies.keySet()); + List differences = names.stream() + .filter(n -> !Objects.equals(oldDependencies.get(n), newDependencies.get(n))) + .collect(Collectors.toList()); + roots.addAll(differences); + finding.put("dependencyChanges", differences); + } + + private void recordEvidence(String key, List> bindings) { + if (bindings.size() == 1 + && evidence.get(key).equals(bindings.get(0).get("fingerprint"))) { + reused.add(key); + } else { + stale.add(key); + } + } + + private Map report(int beforeCount, int afterCount, List globals) { + Map report = new TreeMap<>(); + report.put("needsReview", !changed.isEmpty() || !stale.isEmpty() || !globals.isEmpty()); + report.put("changes", changed); + report.put("directDecisions", direct); + report.put("dependencyRoots", roots); + report.put("reusableEvidence", reused); + report.put("invalidatedEvidence", stale); + report.put("globalSignals", globals); + report.put("ambiguousAnchors", ambiguous); + report.put("beforeSites", beforeCount); + report.put("afterSites", afterCount); + report.put("automaticGrammarChanges", 0); + return report; + } + } +}