diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index c59a67b7..18296027 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -60,7 +60,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform count distinct" in { @@ -96,7 +100,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count" in { @@ -141,7 +149,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with nested criteria" in { @@ -204,7 +216,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with filter" in { @@ -278,7 +294,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with \"and not\" operator" in { @@ -352,7 +372,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with date filtering" in { @@ -420,7 +444,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested select" in { @@ -496,7 +524,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "profileId" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "exclude fields from select" in { @@ -514,7 +546,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "includes":["*"], | "excludes":["col1","col2"] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform query with group by and having" in { @@ -574,6 +610,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -836,6 +875,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll("<(\\d)", " < $1") @@ -869,6 +911,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -916,7 +961,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s", "").replaceAll("ChronoUnit", " ChronoUnit") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") + .replaceAll("ChronoUnit", " ChronoUnit") } it should "filter with date and interval" in { @@ -951,7 +1001,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s", "").replaceAll("ChronoUnit", " ChronoUnit") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") + .replaceAll("ChronoUnit", " ChronoUnit") } it should "filter with time and interval" in { @@ -988,6 +1043,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll(">=", " >= ") .replaceAll("<", " < ") @@ -1048,6 +1106,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("!=", " != ") @@ -1120,6 +1181,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") @@ -1194,6 +1258,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") @@ -1253,6 +1320,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1396,6 +1466,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1461,7 +1534,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" | } | } | } @@ -1470,6 +1543,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1487,7 +1563,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(",ZonedDateTime", ", ZonedDateTime") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") } @@ -1513,7 +1588,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1524,6 +1599,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1540,9 +1618,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(">", " > ") .replaceAll(",ZonedDateTime", ", ZonedDateTime") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") - .replaceAll("XXX", " XXX") } it should "handle date_diff function as script field" in { @@ -1570,6 +1646,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defa", "def a") @@ -1611,7 +1690,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1620,6 +1699,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defa", "def a") @@ -1637,7 +1719,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") } @@ -1680,6 +1761,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1737,6 +1821,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1788,6 +1875,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1839,6 +1929,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1878,6 +1971,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1920,6 +2016,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1966,7 +2065,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle is_notnull criteria as exists" in { @@ -1992,7 +2095,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle coalesce function as script field" in { @@ -2023,6 +2130,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll(";defp", "; defp") .replaceAll("defp", "def p") .replaceAll("defv", " def v") @@ -2078,6 +2188,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defv", " def v") .replaceAll("defa", "def a") @@ -2241,7 +2354,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c5": { | "script": { | "lang": "painless", - | "source": "def param1 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1" + | "source": "def param1 = LocalDate.parse((\"2025-09-11\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1" | } | } | }, @@ -2252,6 +2365,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll(";defp", "; defp") .replaceAll("defp", "def p") .replaceAll("defv", " def v") @@ -2317,6 +2433,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defd", " def d") .replaceAll("defe", " def e") @@ -2374,6 +2493,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defd", " def d") .replaceAll("defe", " def e") @@ -2510,6 +2632,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") @@ -2598,6 +2723,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defl", "def l") @@ -2760,6 +2888,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defp", "def p") @@ -2906,6 +3037,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3035,6 +3169,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3104,6 +3241,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3241,6 +3381,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3360,6 +3503,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defv", " def v") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3420,7 +3566,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-11\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" | } | } | }, @@ -3444,7 +3590,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3510,7 +3662,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-10\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | } @@ -3547,8 +3699,17 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3604,7 +3765,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-10\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | }, @@ -3653,8 +3814,17 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3756,6 +3926,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3823,7 +3996,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle aggregation with nested of nested context" in { @@ -3859,7 +4036,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle where filters according to scope" in { @@ -3930,7 +4111,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } // === Issue #50: HAVING COUNT(*) without alias === @@ -3992,6 +4177,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4048,6 +4236,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">=", " >= ") @@ -4104,6 +4295,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4197,6 +4391,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle HAVING COUNT(DISTINCT *) without alias" in { @@ -4243,6 +4440,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4298,6 +4498,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4365,7 +4568,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate STDDEV_SAMP(salary) to extended_stats (alias of STDDEV)" in { @@ -4413,7 +4620,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate VAR_SAMP(salary) to extended_stats (alias of VARIANCE)" in { @@ -4476,7 +4687,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate the PERCENTILE_CONT(salary, 0.95) shorthand to percentiles" in { @@ -4500,7 +4715,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY salary) to percentiles" in { @@ -4516,7 +4735,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "median": { "percentiles": { "field": "salary", "percents": [50.0] } } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate PERCENTILE_CONT(0.9) OVER (PARTITION BY department ORDER BY salary)" in { @@ -4559,7 +4782,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") "\"percentiles\"".r.findAllIn(query).length shouldBe 1 } @@ -4627,7 +4854,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "terms": { "field": "category", "size": 65536, "min_doc_count": 1 } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "size an aggregate-free GROUP BY from an explicit LIMIT" in { @@ -4658,7 +4889,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "order an aggregate-free GROUP BY by the bucket key, not by a document sort" in { @@ -4722,7 +4957,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "order buckets by the METRIC when the ordinal names an aggregate" in { diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 4e1788c9..087f211a 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -406,6 +406,31 @@ a good value with `NULL`. > **Note** — `STORED` describes *this* index only. The same column can be an executing > `SCRIPT AS` in the index that computes it and a `STORED` one in every index it flows into. +#### Conversions inside a computed column + +The expression is resolved against the **columns of the same table**, so a `CAST` over a sibling +column converts on the way in: + +```sql +CREATE TABLE orders ( + id INT, + zip_code KEYWORD, + zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT)) -- stores the NUMBER 75001 +); +``` + +`ALTER TABLE ... ALTER COLUMN ... SET SCRIPT AS (...)` resolves the same way, against the table as +it currently exists. + +> An operand that names no column of the table is left **unconverted** rather than rejected: the +> statement still succeeds and the raw value is stored. Check the column names if a computed column +> comes back with the type of its source instead of its own. + +⚠️ **Date and time FUNCTIONS are not usable in a computed column.** In an ingest script the operand +is the raw JSON value of the incoming document — for a `date` column, the string as written — so +`YEAR(created)`, `DATE_TRUNC(...)` and `DATE_DIFF(...)` fail at ingest and the column is left +absent from the document. Conversions (`CAST`) and string functions (`UPPER`, ...) are unaffected. + --- ## ALTER TABLE diff --git a/documentation/sql/functions_date_time.md b/documentation/sql/functions_date_time.md index 2fbfe3fe..a4df2296 100644 --- a/documentation/sql/functions_date_time.md +++ b/documentation/sql/functions_date_time.md @@ -890,7 +890,7 @@ The following patterns are supported in `DATE_FORMAT`, `DATE_PARSE`, `DATETIME_F | `%I` | Hour (01-12, synonym for %h) | `01` | | `%i` | Minutes (00-59) | `45` | | `%s` | Seconds (00-59) | `30` | -| `%f` | Microseconds (000000-999999) | `123456` | +| `%f` | Fractional seconds, any precision | `123456` | | `%p` | AM/PM marker | `AM` / `PM` | **Pattern Combination Examples:** @@ -907,6 +907,14 @@ The following patterns are supported in `DATE_FORMAT`, `DATE_PARSE`, `DATETIME_F -- ISO 8601 with microseconds '%Y-%m-%dT%H:%i:%s.%f' -- 2025-01-10T13:45:30.123456 +``` + +> **`%f` is variable width.** It formats a value at its actual precision — `.123456` for +> microseconds, `.123` for milliseconds — and when parsing it accepts any number of fractional +> digits, or none at all. A decimal point written immediately before it belongs to the fraction, so +> a value with no fractional part formats as `12:00:00` rather than `12:00:00.`. + +``` -- Short format '%d-%b-%y' -- 10-Jan-25 diff --git a/documentation/sql/functions_type_conversion.md b/documentation/sql/functions_type_conversion.md index ac600e6b..52a57987 100644 --- a/documentation/sql/functions_type_conversion.md +++ b/documentation/sql/functions_type_conversion.md @@ -117,22 +117,31 @@ SELECT CAST(CURRENT_TIMESTAMP AS VARCHAR) AS ts_str; **Date and Time Conversions:** ```sql --- String to DATE +-- String to DATE. Both the dash and the slash layout are accepted. SELECT CAST('2025-01-10' AS DATE) AS d; -- Result: 2025-01-10 --- String to TIMESTAMP. WARNING - the conversion is pinned to ISO_ZONED_DATE_TIME, so a --- SPACE-separated timestamp with no zone RAISES. Write it in ISO form, or use DATETIME_PARSE. +SELECT CAST('2025/01/10' AS DATE) AS d; +-- Result: 2025-01-10 + +-- String to TIMESTAMP. The ISO spelling and the SQL-standard space-separated spelling are both +-- accepted, and a missing zone defaults to UTC. An explicit offset always wins. SELECT CAST('2025-01-10T14:30:00Z' AS TIMESTAMP) AS ts; -- Result: 2025-01-10T14:30:00Z +SELECT CAST('2025-01-10 14:30:00' AS TIMESTAMP) AS ts; +-- Result: 2025-01-10T14:30:00Z + +SELECT CAST('2025-01-10T14:30:00+01:00' AS TIMESTAMP) AS ts; +-- Result: 2025-01-10T13:30:00Z + -- Timestamp to DATE SELECT CAST(CURRENT_TIMESTAMP AS DATE) AS d; -- Result: 2025-10-27 --- WARNING - a slash-separated date does NOT convert: the DATE conversion is pinned to the pattern --- yyyy-MM-dd, so this RAISES rather than returning a date. Rewrite the value in ISO form. --- SELECT CAST('2025/01/10' AS DATE) AS d; -- error +-- A DAY-FIRST date still RAISES, on purpose: `10/01/2025` could be 10 January or 1 October, and +-- guessing between them is how a date silently becomes a different date. Write it ISO-first. +-- SELECT CAST('10/01/2025' AS DATE) AS d; -- error -- Epoch MILLISECONDS to TIMESTAMP. WARNING - the operand is read as milliseconds, not seconds, so -- a seconds-precision epoch lands in 1970. Multiply by 1000, or use a millisecond epoch. @@ -142,24 +151,32 @@ SELECT CAST(1704902400000 AS TIMESTAMP) AS ts; **Boolean Conversions:** -> WARNING - a conversion TO `BOOLEAN` is currently a no-op: the value is returned unchanged -> (`CAST(1 AS BOOLEAN)` yields `1`, `CAST('true' AS BOOLEAN)` yields the string `'true'`). Only the -> conversions FROM boolean below are applied. Use a comparison (`col = 1`) instead. +A cast TO `BOOLEAN` follows the C rule: **zero is false and every other number is true**, and a +string is read by `Boolean.parseBoolean`, so `'true'` (in any case) is true and **every other +string is false**. + +> The sharp edge worth knowing: the numeric rule applies to a numeric OPERAND, not to a string that +> looks numeric. `CAST('1' AS BOOLEAN)` is **false**, because `'1'` is not the word `true`. Cast it +> to a number first — `CAST(CAST('1' AS INT) AS BOOLEAN)` — or compare instead (`col = '1'`). ```sql --- Number to BOOLEAN - NOT APPLIED, returns the number unchanged +-- Number to BOOLEAN SELECT CAST(1 AS BOOLEAN) AS b; --- Result: 1 +-- Result: true SELECT CAST(0 AS BOOLEAN) AS b; --- Result: 0 +-- Result: false --- String to BOOLEAN - NOT APPLIED, returns the string unchanged +-- String to BOOLEAN SELECT CAST('true' AS BOOLEAN) AS b; --- Result: 'true' +-- Result: true SELECT CAST('false' AS BOOLEAN) AS b; --- Result: 'false' +-- Result: false + +-- A numeric STRING is not the word 'true', so it is false +SELECT CAST('1' AS BOOLEAN) AS b; +-- Result: false -- Boolean to INT SELECT CAST(true AS INT) AS i; @@ -343,16 +360,20 @@ FROM dates_table; ``` **Safe Boolean Conversions:** + +> `TRY_CAST` catches a conversion that RAISES. A cast to `BOOLEAN` never raises — every string that +> is not the word `true` is simply `false` — so `TRY_CAST( AS BOOLEAN)` behaves exactly +> like `CAST`, and never yields `NULL`. + ```sql --- Invalid boolean returns NULL +-- Not the word 'true', so it is false - NOT null SELECT TRY_CAST('maybe' AS BOOLEAN) AS b; --- Result: NULL +-- Result: false --- Valid values work SELECT TRY_CAST('true' AS BOOLEAN) AS b1, TRY_CAST('false' AS BOOLEAN) AS b2, TRY_CAST('1' AS BOOLEAN) AS b3; --- Results: true, false, true +-- Results: true, false, false ``` **Practical Examples:** @@ -591,7 +612,7 @@ WHERE denominator != 0; | VARCHAR | INT | `CAST('123' AS INT)` | Must be valid integer string | | VARCHAR | DOUBLE | `CAST('123.45' AS DOUBLE)` | Must be valid number string | | VARCHAR | DATE | `CAST('2025-01-10' AS DATE)` | Must be valid date format | -| VARCHAR | BOOLEAN | `CAST('true' AS BOOLEAN)` | Accepts 'true'/'false', '1'/'0' | +| VARCHAR | BOOLEAN | `CAST('true' AS BOOLEAN)` | `'true'` is true, ALL else false | | INT | VARCHAR | `CAST(123 AS VARCHAR)` | Always succeeds | | INT | DOUBLE | `CAST(123 AS DOUBLE)` | Always succeeds | | INT | BOOLEAN | `CAST(1 AS BOOLEAN)` | 0=false, non-zero=true | diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index f41fca5c..7d9944b9 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -60,7 +60,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform count distinct" in { @@ -96,7 +100,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count" in { @@ -141,7 +149,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with nested criteria" in { @@ -204,7 +216,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with filter" in { @@ -278,7 +294,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with \"and not\" operator" in { @@ -352,7 +372,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested count with date filtering" in { @@ -420,7 +444,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform nested select" in { @@ -496,7 +524,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "profileId" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "exclude fields from select" in { @@ -514,7 +546,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "includes":["*"], | "excludes":["col1","col2"] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "perform query with group by and having" in { @@ -574,6 +610,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -836,6 +875,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll("<(\\d)", " < $1") @@ -869,6 +911,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -916,7 +961,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s", "").replaceAll("ChronoUnit", " ChronoUnit") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") + .replaceAll("ChronoUnit", " ChronoUnit") } it should "filter with date and interval" in { @@ -951,7 +1001,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s", "").replaceAll("ChronoUnit", " ChronoUnit") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") + .replaceAll("ChronoUnit", " ChronoUnit") } it should "filter with time and interval" in { @@ -988,6 +1043,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll(">=", " >= ") .replaceAll("<", " < ") @@ -1048,6 +1106,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("!=", " != ") @@ -1120,6 +1181,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") @@ -1194,6 +1258,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") @@ -1253,6 +1320,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1396,6 +1466,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1461,7 +1534,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" | } | } | } @@ -1470,6 +1543,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1487,7 +1563,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(",ZonedDateTime", ", ZonedDateTime") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") } @@ -1513,7 +1588,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1524,6 +1599,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("if\\(", "if (") @@ -1540,9 +1618,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(">", " > ") .replaceAll(",ZonedDateTime", ", ZonedDateTime") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") - .replaceAll("XXX", " XXX") } it should "handle date_diff function as script field" in { @@ -1570,6 +1646,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defa", "def a") @@ -1611,7 +1690,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, new DateTimeFormatterBuilder().appendPattern(\"yyyy-MM-dd HH:mm:ss\").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter().withZone(ZoneId.of('Z'))).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1620,6 +1699,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defa", "def a") @@ -1637,7 +1719,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") - .replaceAll("SSSXXX", "SSS XXX") .replaceAll("ddHH", "dd HH") } @@ -1680,6 +1761,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1737,6 +1821,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1788,6 +1875,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1839,6 +1929,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1878,6 +1971,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1920,6 +2016,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defs", "def s") @@ -1966,7 +2065,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle is_notnull criteria as exists" in { @@ -1992,7 +2095,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "*" | ] | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle coalesce function as script field" in { @@ -2023,6 +2130,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll(";defp", "; defp") .replaceAll("defp", "def p") .replaceAll("defv", " def v") @@ -2078,6 +2188,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defv", " def v") .replaceAll("defa", "def a") @@ -2232,7 +2345,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c5": { | "script": { | "lang": "painless", - | "source": "def param1 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1" + | "source": "def param1 = LocalDate.parse((\"2025-09-11\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1" | } | } | }, @@ -2243,6 +2356,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll(";defp", "; defp") .replaceAll("defp", "def p") .replaceAll("defv", " def v") @@ -2308,6 +2424,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defd", " def d") .replaceAll("defe", " def e") @@ -2365,6 +2484,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defd", " def d") .replaceAll("defe", " def e") @@ -2501,6 +2623,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") @@ -2589,6 +2714,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defe", "def e") .replaceAll("defl", "def l") @@ -2751,6 +2879,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defp", "def p") @@ -2897,6 +3028,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3026,6 +3160,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3095,6 +3232,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3232,6 +3372,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3351,6 +3494,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defv", " def v") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3411,7 +3557,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-11\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" | } | } | }, @@ -3435,7 +3581,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3501,7 +3653,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-10\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | } @@ -3538,8 +3690,17 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3595,7 +3756,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse((\"2025-09-10\").replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | }, @@ -3644,8 +3805,17 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3747,6 +3917,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3814,7 +3987,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle aggregation with nested of nested context" in { @@ -3850,7 +4027,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle where filters according to scope" in { @@ -3921,7 +4102,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } // === Issue #50: HAVING COUNT(*) without alias === @@ -3983,6 +4168,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4039,6 +4227,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">=", " >= ") @@ -4095,6 +4286,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4188,6 +4382,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "handle HAVING COUNT(DISTINCT *) without alias" in { @@ -4234,6 +4431,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4289,6 +4489,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") .replaceAll("==", " == ") .replaceAll("&&", " && ") .replaceAll(">", " > ") @@ -4429,7 +4632,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate STDDEV_SAMP(salary) to extended_stats (alias of STDDEV)" in { @@ -4477,7 +4684,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate VAR_SAMP(salary) to extended_stats (alias of VARIANCE)" in { @@ -4540,7 +4751,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate the PERCENTILE_CONT(salary, 0.95) shorthand to percentiles" in { @@ -4564,7 +4779,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY salary) to percentiles" in { @@ -4580,7 +4799,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "median": { "percentiles": { "field": "salary", "percents": [50.0] } } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "translate PERCENTILE_CONT(0.9) OVER (PARTITION BY department ORDER BY salary)" in { @@ -4618,7 +4841,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") "\"percentiles\"".r.findAllIn(query).length shouldBe 1 } @@ -4686,7 +4913,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "terms": { "field": "category", "size": 65536, "min_doc_count": 1 } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "size an aggregate-free GROUP BY from an explicit LIMIT" in { @@ -4717,7 +4948,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "order an aggregate-free GROUP BY by the bucket key, not by a document sort" in { @@ -4781,7 +5016,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s", "") + |}""".stripMargin + .replaceAll("\\s", "") + .replace("NANO_OF_SECOND,0,9,true", "NANO_OF_SECOND, 0, 9, true") + .replace(",newDateTimeFormatterBuilder", ", new DateTimeFormatterBuilder") + .replace("\\\"/\\\",\\\"-\\\"", "\\\"/\\\", \\\"-\\\"") } it should "order buckets by the METRIC when the ordinal names an aggregate" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index e68ef9ee..e76f7187 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -515,10 +515,67 @@ package object time { sealed trait FunctionWithDateTimeFormat { def format: String - def includeTimeZone: Boolean = false - - protected def param: String = "DateTimeFormatter.ofPattern(\"" + convert() + "\")" + /** The `%f` (fractional seconds) marker, left in the pattern by `convert()` and turned into a + * VARIABLE-WIDTH fraction here. It is not a letter substitution because no `ofPattern` letter + * can express one: `S` is fixed width in BOTH directions. + */ + private val FractionMarker = "%f" + + protected def param: String = { + val pattern = convert() + val at = pattern.indexOf(FractionMarker) + if (at < 0) "DateTimeFormatter.ofPattern(\"" + pattern + "\")" + else { + // A decimal point written immediately before `%f` BELONGS to the fraction: handing it to + // `appendFraction` is what makes a zero-nanosecond value format as `12:00:00` instead of + // `12:00:00.`, and what lets a value with no fraction at all still parse. + val absorbsPoint = at > 0 && pattern.charAt(at - 1) == '.' + val head = pattern.substring(0, if (absorbsPoint) at - 1 else at) + // A second `%f` in one format is meaningless; keep the historical fixed-width mapping for + // it rather than emitting a pattern `ofPattern` would reject. + val tail = pattern.substring(at + FractionMarker.length).replace(FractionMarker, "SSS") + val b = new StringBuilder("new DateTimeFormatterBuilder()") + if (head.nonEmpty) b.append(s""".appendPattern("$head")""") + b.append(s".appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, $absorbsPoint)") + if (tail.nonEmpty) b.append(s""".appendPattern("$tail")""") + b.append(".toFormatter()") + b.toString + } + } + /** The same formatter with a DEFAULT zone, for a parse that must produce a `ZonedDateTime`. + * + * 🔴 This replaces appending `" XXX"` to the user's pattern, which was wrong in BOTH + * directions and MEASURED so on real ES 8.18 (story 21.8): + * + * - formatting: `DATETIME_FORMAT(ts, 'yyyy')` emitted `ofPattern("yyyy XXX")` and returned + * `"2025 Z"` — a silent wrong answer on the ordinary script-field path, for a caller who + * asked for four characters; + * - parsing: `DATETIME_PARSE('2025-01-10 10:00:00', 'yyyy-MM-dd HH:mm:ss')` emitted + * `ofPattern("yyyy-MM-dd HH:mm:ss XXX")`, which demands a space and an offset the caller's + * format never declared, so the parse failed at runtime. It "worked" only for input that + * happened to carry ` +01:00`. + * + * `withZone` is the right tool because it does NOT rewrite the caller's pattern: at parse time + * it supplies a zone only when the text carried none, and an explicit offset still wins + * (verified — `2025-01-10T14:30:00+01:00` resolves to `13:30Z`). It is the same mechanism + * `SQLTypeUtils`' ` -> TIMESTAMP` arm uses, deliberately: one derivation, not two. + */ + protected def zonedParam: String = s"$param.withZone(ZoneId.of('Z'))" + + /** MySQL-style format letters to `java.time` pattern letters. + * + * 🔴 `%f` is deliberately ABSENT. MySQL's `%f` is fractional seconds, and it was mapped to + * `SSS` — three digits — under a comment that said "microseconds". MEASURED on real ES 8.18.3, + * `S` is FIXED WIDTH in both directions: `SSS` formats `.123456789` as `.123` and REFUSES to + * parse `.123456`, while `SSSSSS` refuses to parse `.123`. Optional sections do not rescue it + * either — `[.SSSSSS][.SSS]` formats as `.123456.123`. + * + * So no letter substitution can be correct, and picking a width would NARROW one direction to + * widen the other — an "except" inside the very rule story 21.8's temporal arms are justified + * by (widen, never narrow). `param` emits a variable-width `appendFraction` instead, which + * formats the value's real precision and parses any number of digits, including none. + */ val sqlToJava: Map[String, String] = Map( "%Y" -> "yyyy", "%y" -> "yy", @@ -534,7 +591,6 @@ package object time { "%i" -> "mm", "%s" -> "ss", "%S" -> "ss", - "%f" -> "SSS", // microseconds "%p" -> "a", "%W" -> "EEEE", "%a" -> "EEE", @@ -552,12 +608,11 @@ package object time { pattern.replace(sql, java) } - val patternWithTZ = - if (basePattern.contains("Z")) basePattern.replace("Z", "X") - else if (includeTimeZone) s"$basePattern XXX" - else basePattern - - patternWithTZ + // A literal `Z` in the caller's pattern means "zone NAME" to `ofPattern`, which does not + // accept the `Z` that ISO-8601 writes for UTC; `X` does. Unchanged, and unrelated to the + // default-zone question `zonedParam` answers. + if (basePattern.contains("Z")) basePattern.replace("Z", "X") + else basePattern } } @@ -751,8 +806,6 @@ package object time { s"$sql($base, '$format')" } - override def includeTimeZone: Boolean = true - override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = callArgs match { case arg :: Nil => @@ -760,7 +813,7 @@ package object time { case Some(ctx) => identifier.baseType match { case SQLTypes.Varchar => - ctx.addParam(LiteralParam(s"ZonedDateTime.parse($arg, $param)")) match { + ctx.addParam(LiteralParam(s"ZonedDateTime.parse($arg, $zonedParam)")) match { case Some(p) => return p case _ => } @@ -768,7 +821,7 @@ package object time { } case _ => } - s"ZonedDateTime.parse($arg, $param)" + s"ZonedDateTime.parse($arg, $zonedParam)" case _ => throw new IllegalArgumentException("DateParse requires exactly one argument") } @@ -813,8 +866,6 @@ package object time { s"$sql($base, '$format')" } - override def includeTimeZone: Boolean = true - override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = callArgs match { case arg :: Nil => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index d53d092c..fca076ab 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -169,6 +169,41 @@ sealed trait Criteria extends Updateable with PainlessScript { override def out: SQLType = SQLTypes.Boolean + /** A criterion's Painless is a comparison, so the value it emits is a BOOLEAN — and `baseType` is + * what every `SQLTypeUtils.coerce` call site reads to learn the type of the string it was handed + * (`coerce(in, to, ctx)` takes `expr = in.painless(ctx)` and `from = in.baseType`). + * + * Without this override an `Expression` inherits `FunctionChain.baseType`, which walks the + * IDENTIFIER's function chain: `1 = 1` reported `BIGINT` (the left operand), `descr = 'x'` + * reported the column's type. That has always been wrong, but it was INERT while no `(_, + * BOOLEAN)` arm existed in `coerce` — every such pair fell to the identity fallback and the + * comparison was emitted untouched, which happened to be right. + * + * 🔴 Story 21.8 part A added the `-> BOOLEAN` arms, and the lie became live. `CASE WHEN 1 = 1` + * coerced an already-boolean comparison FROM `BIGINT`, so the numeric arm fired AND — because + * the arm returns a primitive — the end-of-method null guard wrapped it, emitting + * + * {{{def param1 = (1 == 1 != null ? (def)((1 == 1 != 0)) : null); param1 ? "a" : "b"}}} + * + * which Elasticsearch refuses to compile. All five clients emitted that same script and were + * rejected with `search_phase_execution_exception: all shards failed; compile error` (HTTP 400); + * the REST-high-level legs' raw response carries the detail — `script_exception: compile error` + * with the pointer at offset 14 and `caused_by class_cast_exception: "Cannot cast from [boolean] + * to [java.lang.Object]."`. So every `CASE WHEN ` failed on every ES major — the + * shape Superset and Tableau generate constantly, and the connection handshake's own `SELECT + * CASE WHEN 1 = 1 …` probe, which is what caught it. + * + * Fixing it inside `coerce` is not possible: by the time an arm matches, a genuine `CAST(1 AS + * BOOLEAN)` and this comparison are the same `(BIGINT, BOOLEAN)` pair. The source type has to be + * right before `coerce` is called. + * + * `out` above already fixes the OUTPUT type to boolean for exactly this reason; the two facts + * were simply never stated together. `Criteria` is the one place both belong, and `baseType` + * here overrides `FunctionChain`'s by linearization for `Expression` (`Criteria` is its last + * mixin). + */ + override def baseType: SQLType = SQLTypes.Boolean + override def painless(context: Option[PainlessContext]): String = this match { case Predicate(left, op, right, maybeNot, group) => val leftStr = left.painless(context) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 63be1cc8..d3b3f8d7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -477,11 +477,50 @@ package object schema { dataType: SQLType, source: String, ignoreFailure: Boolean = true, - materialized: Boolean = false + materialized: Boolean = false, + /** The parsed expression this processor's `source` was derived FROM, when it came from a + * `SCRIPT AS (…)` in DDL rather than from a pipeline read back out of Elasticsearch. + * + * 🔴 It exists so the derivation can be RE-RUN once the sibling columns are known. At parse + * time a `CREATE TABLE`'s column list does not exist yet, so every operand's `baseType` was + * `Any` and no `SQLTypeUtils.coerce` arm fired: `CREATE TABLE t (zip_code KEYWORD, zip_n + * BIGINT SCRIPT AS (CAST(zip_code AS BIGINT)))` stored `ctx.zip_n = param1` — the UNCONVERTED + * keyword string into a `long`-mapped field. Elasticsearch never rewrites `_source` and + * `coerce` defaults true, so the mapping stayed correct while the document was wrong + * (`local-21.8-ddl-script-processor-has-no-schema.md`, the #205 silent-wrong-value family). + * + * Appended LAST and defaulted, so every positional construction keeps compiling; `None` for a + * processor loaded from Elasticsearch, which has JSON and no AST, and whose stored `source` is + * therefore left exactly as it was found. + */ + expr: Option[PainlessScript] = None ) extends IngestProcessor { override def sql: String = s"$column $dataType SCRIPT AS ($script)${if (materialized) " STORED" else ""}" + /** Re-derive `source` from [[expr]] with `schema` attached, so the conversion arms that `#306` + * made reachable for a query are reachable for an ingest script too. + * + * Returns `this` unchanged whenever there is no expression to re-derive from — which is both + * the Elasticsearch-load path and the lead's OQ-4 ruling for an operand the schema cannot + * resolve: it stays SILENT and emits the identity, exactly as before. A DDL statement that + * works today keeps working; it just stops storing the wrong value. + */ + def resolvedAgainst(schema: Schema, columnPath: String): ScriptProcessor = + expr match { + case Some(e) => + ScriptProcessor + .fromScript( + column = columnPath, + script = resolveAgainstSchema(e, schema), + dataType = Some(dataType), + pipelineType = pipelineType, + materialized = materialized + ) + .copy(description = description, ignoreFailure = ignoreFailure) + case None => this + } + override def baseType: SQLType = dataType def processorType: IngestProcessorType = IngestProcessorType.Script @@ -495,6 +534,44 @@ package object schema { } + /** Attach `schema` to a DDL `SCRIPT AS (…)` expression, so each operand resolves to its declared + * column and `SQLTypeUtils.coerce` can see a real type instead of `Any`. + * + * 🔴 It goes through the SAME `Identifier.update` every executed query uses, over a minimal + * synthetic `SingleSearch`, rather than through a second identifier-resolution walk written for + * DDL. A parallel walk would be a second derivation of "which column is this operand", and this + * area has already paid for that four times over (story 21.3). Every `scriptValue` alternative + * yields an `Identifier`, so one `update` call is the whole job. + * + * The synthetic statement carries no SELECT, no GROUP BY and one placeholder table: `update` + * reads `request.schemas.get(
)` first and falls back to `request.schema`, which is + * the schema handed in here, so the table's NAME is irrelevant and no alias, bucket or field + * alias can resolve. An operand the schema does not know simply keeps `col = None` — the lead's + * OQ-4 ruling, and byte-for-byte today's emission. + * + * 🔴 A temporal-SOURCE arm still must NOT fire here, and does not: the processor context makes + * `SQLTypeUtils.coerce`'s `isProcessorContext` guard decline them, because in an ingest script + * the operand is `ctx.` — the raw JSON scalar — not the temporal object a query's + * `doc['f'].value` yields. Attaching the schema is what makes those arms REACHABLE for the first + * time, so that guard stops being theoretical the moment this function exists. + */ + private[schema] def resolveAgainstSchema( + script: PainlessScript, + schema: Schema + ): PainlessScript = + script match { + case id: Identifier => + id.update( + SingleSearch( + select = Select(Seq.empty), + from = From(Seq(query.Table(schema.name))), + where = None, + schema = Some(schema) + ) + ) + case other => other + } + object ScriptProcessor { def fromScript( column: String, @@ -521,7 +598,8 @@ package object schema { column = column, dataType = dataType.getOrElse(script.out), source = source, - materialized = materialized + materialized = materialized, + expr = Some(script) ) } } @@ -1067,6 +1145,17 @@ package object schema { ) } + /** Re-derive this column's ingest script — and its sub-fields' — against `schema`. + * + * Runs AFTER [[update]], so `path` is settled and the derivation targets the final + * `ctx.` directly instead of patching a string that was built for a different name. + */ + def resolveScript(schema: Schema): Column = + copy( + script = script.map(_.resolvedAgainst(schema, path)), + multiFields = multiFields.map(_.resolveScript(schema)) + ) + def sql: String = { val opts = if (options.nonEmpty) { s" OPTIONS ${ObjectValue(options).ddl}" @@ -1566,8 +1655,15 @@ package object schema { else ListMap.empty[String, Value[_]]) def update(): Table = { - val updated = + val withPaths = this.copy(columns = columns.map(_.update())) // update columns first with struct info + // ...THEN re-derive every `SCRIPT AS (…)` against the settled column list. This is the single + // seam that covers BOTH DDL sites: `CreateTable.schema` builds its table from the parsed + // columns and ends here, and `Table.merge` applies `ALTER … SET SCRIPT AS` to the LIVE table + // and ends here too. The spec expected `ALTER` to need a client-side schema load, because the + // STATEMENT carries no column list — but the point where it is APPLIED already holds the + // whole table, so both are one derivation with no I/O and nothing new on the parse path. + val updated = withPaths.copy(columns = withPaths.columns.map(_.resolveScript(withPaths))) updated.copy( mappings = updated.mappings ++ ListMap( "_meta" -> diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 64080257..2784cf38 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -410,6 +410,35 @@ object SQLTypeUtils { case (SQLTypes.Boolean, SQLTypes.TinyInt) => s"(byte)($expr ? 1 : 0)" + // ---- -> BOOLEAN ---- + // 🔴 There were seven arms FROM boolean and NONE to it, so every `(_, Boolean)` pair fell + // to the identity fallback at the bottom of this match and `CAST(1 AS BOOLEAN)` emitted + // `1`, `CAST('true' AS BOOLEAN)` emitted `"true"` — a cast that silently did nothing, the + // #205 family. Recorded as `local-21.8-cast-to-boolean-is-a-no-op.md`. + // + // Semantics are the lead's PD-1 ruling (story 21.8): C-STYLE, i.e. what MySQL and SQLite do + // and what a BI tool generating `CAST(flag AS BOOLEAN)` expects. It never raises, so no + // statement that runs today starts failing — it starts returning the right answer. + // + // - numeric: zero is false, anything else true. `!= 0` compares fine against a `double` + // or a `float` too, so the whole numeric lattice is one arm. + // - string: `Boolean.parseBoolean`, i.e. case-insensitive `"true"` is true and EVERY + // other string — `"1"`, `"yes"`, `"T"` — is false. That is the documented sharp edge of + // the C-style ruling and it is written down in functions_type_conversion.md rather than + // hidden: a numeric STRING does not go through the numeric rule. + // + // The sources are ENUMERATED rather than matched with a wildcard so that `(Boolean, + // Boolean)` still reaches the identity arm below, and so a temporal or struct source keeps + // falling through instead of being silently coerced. + case ( + SQLTypes.Numeric | SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real | + SQLTypes.SmallInt | SQLTypes.TinyInt, + SQLTypes.Boolean + ) => + s"($expr != 0)" + case (_: SQLVarchar, SQLTypes.Boolean) => + s"Boolean.parseBoolean($expr)" + // ---- LITERAL (VARCHAR / TEXT / KEYWORD) -> NUMERIC ---- // 🔴 These arms matched the `SQLTypes.Varchar` case OBJECT, and no Elasticsearch mapping // ever reports VARCHAR — `SQLTypes.apply(String)` maps every string field to `Text` or @@ -445,7 +474,8 @@ object SQLTypeUtils { // applying a second time. No boxing — every one of these returns a reference. case (_: SQLVarchar, SQLTypes.Date) => val guarded = temporalGuard( - "LocalDate.parse(" + expr + ", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + "LocalDate.parse(" + temporalSeparator(expr, "/", "-") + + ", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" ) context match { case Some(ctx) => @@ -458,7 +488,7 @@ object SQLTypeUtils { return guarded case (_: SQLVarchar, SQLTypes.Time) => val guarded = temporalGuard( - "LocalTime.parse(" + expr + ", DateTimeFormatter.ofPattern(\"HH:mm:ss\"))" + "LocalTime.parse(" + expr + ", DateTimeFormatter.ISO_LOCAL_TIME)" ) context match { case Some(ctx) => @@ -471,7 +501,8 @@ object SQLTypeUtils { return guarded case (_: SQLVarchar, SQLTypes.DateTime) => val guarded = temporalGuard( - s"LocalDateTime.parse($expr, DateTimeFormatter.ISO_DATE_TIME)" + "LocalDateTime.parse(" + temporalSeparator(expr, " ", "T") + + ", DateTimeFormatter.ISO_DATE_TIME)" ) context match { case Some(ctx) => @@ -484,7 +515,8 @@ object SQLTypeUtils { return guarded case (_: SQLVarchar, SQLTypes.Timestamp) => val guarded = temporalGuard( - s"ZonedDateTime.parse($expr, DateTimeFormatter.ISO_ZONED_DATE_TIME)" + "ZonedDateTime.parse(" + temporalSeparator(expr, " ", "T") + + ", DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z')))" ) context match { case Some(ctx) => @@ -547,6 +579,40 @@ object SQLTypeUtils { else s"($expr != null ? $ret : null)" } + /** The one normalisation the four ` -> ` arms share, so that a literal written + * the way a human writes one is accepted alongside the ISO spelling. + * + * 🔴 Why a normalisation and not an ordered list of formatters. The lead's OQ-3 ruling (story + * 21.8) is "accept a small ordered format set per target". Painless has no expression-level + * `try`/`catch` — `coerce` returns an EXPRESSION that is embedded anywhere, so a + * parse-then-fall-back chain cannot be written — and `DateTimeFormatter.ofPattern`'s optional + * sections cannot express ISO's variable-length fractional seconds, so a hand-written pattern + * would NARROW what is accepted today. Rewriting the separator in front of a WIDER ISO formatter + * yields the same accepted set in one parse, and is a strict superset of the old behaviour: a + * literal that already parsed is untouched by the rewrite. + * + * The accepted sets, after this: + * + * - `DATE` — `2025-01-10` and `2025/01/10`. `10/01/2025` still fails, LOUDLY: rewriting its + * separators gives `10-01-2025`, which no pattern accepts. Day-first and month-first are + * ambiguous and guessing between them is how a date silently becomes a different date. + * - `TIME` — `ISO_LOCAL_TIME` replaces the hard-coded `HH:mm:ss`, so `14:30` and + * `14:30:00.123` join `14:30:00`. Pure widening, and included for uniformity: a constant + * justified by correctness applies with no "except". + * - `DATETIME` / `TIMESTAMP` — the space separator becomes `T`. + * + * `TIMESTAMP` additionally moves from `ISO_ZONED_DATE_TIME` to `ISO_DATE_TIME.withZone(UTC)`. + * `ISO_ZONED_DATE_TIME` REQUIRES an offset, so `2025-01-10 14:30:00` failed twice over — wrong + * separator AND no zone — while `ISO_DATE_TIME` parses the offset optionally and `withZone` + * supplies UTC only when the text carried none. An explicit `+01:00` still wins, so this widens + * without reinterpreting anything that already worked. + * + * The operand is parenthesised because it is an arbitrary expression: `a + b.replace(...)` and + * `(a + b).replace(...)` are different scripts, and only the second is this one. + */ + private def temporalSeparator(expr: String, from: String, to: String): String = + s"""($expr).replace("$from", "$to")""" + private val numericRank: Map[Class[_], Int] = Map( classOf[SQLTinyInt] -> 1, classOf[SQLSmallInt] -> 2, diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/BooleanCastSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/BooleanCastSpec.scala new file mode 100644 index 00000000..73dbea16 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/BooleanCastSpec.scala @@ -0,0 +1,194 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessScript} +import app.softnetwork.elastic.sql.function.cond.Case +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLTypeUtils, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 21.8 part A — `CAST(x AS BOOLEAN)` was a silent no-op. + * + * `SQLTypeUtils.coerce` had seven arms FROM boolean and none TO it, so every `(_, Boolean)` pair + * fell to the identity fallback: `CAST(1 AS BOOLEAN)` emitted `1` and `CAST('true' AS BOOLEAN)` + * emitted `"true"`. The cast parsed, rendered and round-tripped, and did nothing — the #205 + * silent-wrong-answer family. + * + * Semantics are the lead's PD-1 ruling: C-STYLE. Numerics compare against zero; strings go through + * `Boolean.parseBoolean`, so `'true'` (any case) is true and every other string is false. + * + * 🔴 Every emission below was EXECUTED on a real Elasticsearch 8.18.3 via `POST + * /_scripts/painless/_execute` before being pinned here — `Boolean.parseBoolean` being + * whitelisted, and `!= 0` accepting a `def`, are Painless claims, and only Elasticsearch settles + * those (this project's standing rule; story 21.5 paid for it twice). + * + * Never pin the BEFORE strings — they are the silent wrong answers this story fixes. + */ +class BooleanCastSpec extends AnyFlatSpec with Matchers { + + /** A schema-CARRYING fixture. A schema-less parse leaves every identifier's `baseType` at `Any`, + * so no conversion arm is reachable at all and a broken engine looks identical to a fixed one — + * that is exactly how the C1/C2 defects survived until story 21.5. + */ + private val schema: Table = Table( + "t", + columns = List( + Column("flag", SQLTypes.Keyword), + Column("descr", SQLTypes.Text), + Column("n", SQLTypes.Int), + Column("big", SQLTypes.BigInt), + Column("amount", SQLTypes.Double), + Column("ok", SQLTypes.Boolean), + Column("ts", SQLTypes.Timestamp) + ) + ) + + private def painlessOf(sql: String): String = + Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(schema)).select.fields.head.painless(None) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** 🔴 The PRODUCTION renderer, and the reason this file needs two. + * + * `painlessOf` above renders with `painless(None)`. That is faithful for a CAST, whose emission + * does not branch on the context. It is BLIND to `CASE`, which renders an entirely different way + * once a `PainlessContext` exists: only that branch hoists each condition into a `def paramN = + * ...` binding, and only that branch coerces the condition. So a context-free assertion cannot + * see the defect below no matter what it pins. + * + * This is the same distinction story 21.8's T0 refutation turned on — a context-free rendering + * is not a production one — and it is the reason the whole unit estate (945 sql + 924 core + 2 × + * 197 bridge) stayed green while every live client failed. + */ + private def scriptOf(sql: String): String = + Parser(sql) match { + case Right(ss: SingleSearch) => + val ctx = PainlessContext(PainlessContextType.Query) + val body = ss.update(Some(schema)).select.fields.head.painless(Some(ctx)) + s"$ctx$body" + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def conditionsOf(sql: String): List[PainlessScript] = + Parser(sql) match { + case Right(ss: SingleSearch) => + ss.update(Some(schema)) + .select + .fields + .head + .identifier + .functions + .collectFirst { case c: Case => + c.conditions.map { case (cond, _) => cond } + } + .getOrElse(fail(s"[$sql] no CASE in the function chain")) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + "a cast of a numeric LITERAL to boolean" should "compare against zero, not vanish" in { + // Executed on ES 8.18.3: `(1 != 0)` -> true, `(0 != 0)` -> false, `(1.5 != 0)` -> true. + painlessOf("SELECT CAST(1 AS BOOLEAN) FROM t") shouldBe "(1 != 0)" + painlessOf("SELECT CAST(0 AS BOOLEAN) FROM t") shouldBe "(0 != 0)" + painlessOf("SELECT CAST(1.5 AS BOOLEAN) FROM t") shouldBe "(1.5 != 0)" + } + + "a cast of a string LITERAL to boolean" should "parse it" in { + // Executed on ES 8.18.3: "true" -> true, "TRUE" -> true (parseBoolean is case-insensitive). + painlessOf("SELECT CAST('true' AS BOOLEAN) FROM t") shouldBe """Boolean.parseBoolean("true")""" + painlessOf("SELECT CAST('TRUE' AS BOOLEAN) FROM t") shouldBe """Boolean.parseBoolean("TRUE")""" + } + + it should "treat a NUMERIC string as false - the documented sharp edge of the C-style rule" in { + // 🔴 Deliberate, ruled, and written down in functions_type_conversion.md rather than hidden: + // the numeric rule applies to a numeric OPERAND, not to a string that looks numeric. + // Executed on ES 8.18.3: `Boolean.parseBoolean("1")` -> false. + painlessOf("SELECT CAST('1' AS BOOLEAN) FROM t") shouldBe """Boolean.parseBoolean("1")""" + } + + "a cast of a COLUMN to boolean" should "convert through the mapped type" in { + // The shape a BI tool generates. Before this story both emitted the raw doc value. + painlessOf("SELECT CAST(flag AS BOOLEAN) FROM t") should include("Boolean.parseBoolean") + painlessOf("SELECT CAST(descr AS BOOLEAN) FROM t") should include("Boolean.parseBoolean") + painlessOf("SELECT CAST(n AS BOOLEAN) FROM t") should include("!= 0") + painlessOf("SELECT CAST(big AS BOOLEAN) FROM t") should include("!= 0") + painlessOf("SELECT CAST(amount AS BOOLEAN) FROM t") should include("!= 0") + } + + it should "stay null-guarded, so a document missing the field does not fail the shard" in { + // The guard the four temporal arms had to grow for themselves (#306). These arms fall through + // to the end-of-method wrapper, so it applies for free -- asserted rather than assumed. + // Executed on ES 8.18.3 with a null operand: the whole expression evaluates to null. + painlessOf("SELECT CAST(flag AS BOOLEAN) FROM t") should include("!= null") + painlessOf("SELECT CAST(n AS BOOLEAN) FROM t") should include("!= null") + } + + "a boolean operand" should "still reach the identity arm, not a new conversion" in { + // The new arms ENUMERATE their sources precisely so `(Boolean, Boolean)` falls through to the + // identity arm below them. A wildcard `(_, Boolean)` would have shadowed it. + val painless = painlessOf("SELECT CAST(ok AS BOOLEAN) FROM t") + painless should not include "parseBoolean" + painless should not include "!= 0" + } + + "a temporal operand" should "keep falling through, not be coerced into a boolean" in { + val painless = painlessOf("SELECT CAST(ts AS BOOLEAN) FROM t") + painless should not include "parseBoolean" + painless should not include "!= 0" + } + + "a CASE condition" should "reach Painless as the boolean it already is" in { + // 🔴 The regression this section exists for, and the only defect in story 21.8 that reached a + // real cluster. A criterion's `baseType` reported its LEFT OPERAND's type (`1 = 1` -> BIGINT, + // `descr = 'x'` -> the column's), which was inert while `coerce` had no `(_, BOOLEAN)` arm -- + // the pair fell to the identity fallback and the comparison was emitted untouched. Part A added + // the arms, so `CASE WHEN 1 = 1` coerced an already-boolean comparison FROM BIGINT, emitting + // `def param1 = (1 == 1 != null ? (def)((1 == 1 != 0)) : null); ...` -- which Elasticsearch + // refuses to compile (`class_cast_exception: Cannot cast from [boolean] to [java.lang.Object].` + // under `all shards failed; compile error`, HTTP 400). All five clients failed identically. + // + // Every string below is byte-for-byte what `origin/main` emits, so this pins a RESTORATION, not + // a new shape. All four were executed on a real Elasticsearch 8.18.3. + scriptOf("SELECT CASE WHEN 1 = 1 THEN 'a' ELSE 'b' END AS c FROM t") shouldBe + "def param1 = 1 == 1; param1 ? \"a\" : \"b\"" + scriptOf("SELECT CASE WHEN 2 > 1 THEN 1 ELSE 0 END AS c FROM t") shouldBe + "def param1 = 2 > 1; param1 ? 1 : 0" + // a COLUMN operand -- ANY before, so no arm ever fired; pinned so it stays that way + scriptOf("SELECT CASE WHEN n > 1 THEN 1 ELSE 0 END AS c FROM t") shouldBe + "def param1 = (doc['n'].size() == 0 ? null : doc['n'].value); " + + "def param2 = param1 == null ? false : (param1 > 1); param2 ? 1 : 0" + // a VARCHAR column operand -- the `Boolean.parseBoolean` arm's source type, so this is the + // shape that would have been corrupted into `Boolean.parseBoolean()` + scriptOf("SELECT CASE WHEN descr = 'x' THEN 1 ELSE 0 END AS c FROM t") shouldBe + "def param1 = (doc['descr'].size() == 0 ? null : doc['descr'].value); " + + "def param2 = param1 == null ? false : (param1.compareTo(\"x\") == 0); param2 ? 1 : 0" + } + + it should "report BOOLEAN as its own base type, whatever it compares" in { + // The invariant behind the four pins above, stated where it can be checked without an emission: + // `baseType` is what EVERY `SQLTypeUtils.coerce` call site reads to learn the type of the + // string it was handed (`coerce(in, to, ctx)` takes `expr = in.painless(ctx)` and `from = + // in.baseType`). A criterion emits a comparison, so that type is BOOLEAN and nothing else. + // Asserting it here means the next `(_, BOOLEAN)` arm cannot re-open this hole silently. + conditionsOf("SELECT CASE WHEN 1 = 1 THEN 1 END AS c FROM t").map(_.baseType) shouldBe + List(SQLTypes.Boolean) + conditionsOf("SELECT CASE WHEN descr = 'x' THEN 1 END AS c FROM t").map(_.baseType) shouldBe + List(SQLTypes.Boolean) + conditionsOf("SELECT CASE WHEN ts > '2025-01-01' THEN 1 END AS c FROM t").map( + _.baseType + ) shouldBe + List(SQLTypes.Boolean) + } + + "canConvert" should "be untouched by this story (AC-6)" in { + // 🔴 `canConvert` gates SCHEMA EVOLUTION through `schema/TableDiff.scala` -- a different + // contract with a different blast radius (story 21.5 AD, re-confirmed by 21.8). It is a + // lookalike of `coerce` and it deliberately DISAGREES with it here: `coerce` now converts a + // numeric to a boolean, while an existing `long` column may still not be REDEFINED as + // `boolean`. Pinned so that "make them agree" is a decision someone has to take on purpose. + SQLTypeUtils.canConvert(SQLTypes.Int, SQLTypes.Boolean) shouldBe false + SQLTypeUtils.canConvert(SQLTypes.Keyword, SQLTypes.Boolean) shouldBe false + SQLTypeUtils.canConvert(SQLTypes.Boolean, SQLTypes.Boolean) shouldBe true + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ConversionTargetTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ConversionTargetTypeSpec.scala index 5ed1fde8..0e2994c0 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ConversionTargetTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ConversionTargetTypeSpec.scala @@ -362,7 +362,9 @@ class ConversionTargetTypeSpec extends AnyFlatSpec with Matchers { // alternative and the later `~` failure never backtracks. Measured; do not "clean up" there. val sql = "SELECT CAST('2025-01-01' + INTERVAL 1 DAY AS DATE) AS c FROM t" chainOf(sql) shouldBe List("Cast", "SQLAddInterval", "StringValue") + // The `.replace("/", "-")` is story 21.8's DATE format widening (it accepts `2025/01/10` too), + // not part of what this test is about; the operand-ordering claim above is. identifierOf(sql).painless(None) shouldBe - """LocalDate.parse("2025-01-01", DateTimeFormatter.ofPattern("yyyy-MM-dd")).plus(1, ChronoUnit.DAYS)""" + """LocalDate.parse(("2025-01-01").replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd")).plus(1, ChronoUnit.DAYS)""" } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/DateTimeFormatZoneSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/DateTimeFormatZoneSpec.scala new file mode 100644 index 00000000..081ad02f --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/DateTimeFormatZoneSpec.scala @@ -0,0 +1,119 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 21.8 — `DATETIME_PARSE` and `DATETIME_FORMAT` appended `" XXX"` to the CALLER'S pattern. + * + * 🔴 Found by re-measuring record 6 on the PRODUCTION path. The record claimed those functions + * emit uncompilable Painless; measured, that is true only of the context-free rendering, which no + * shipped surface applies to them. What IS live is this, and it is worse in one direction — + * MEASURED on a real Elasticsearch 8.18.3: + * + * - `DATETIME_FORMAT(ts, 'yyyy')` emitted `ofPattern("yyyy XXX")` and returned `"2025 Z"`. A + * caller who asked for four characters got seven, silently, on the ordinary script-field path; + * - `DATETIME_PARSE('2025-01-10 10:00:00', 'yyyy-MM-dd HH:mm:ss')` emitted + * `ofPattern("yyyy-MM-dd HH:mm:ss XXX")`, which demands a space and an offset the caller's + * format never declared, and THREW. It "worked" only for text that happened to carry ` + * +01:00`. + * + * The fix does not rewrite the caller's pattern. A parse that must yield a `ZonedDateTime` gets + * the zone from `withZone`, which supplies one only when the text carried none — the same + * mechanism `SQLTypeUtils`' ` -> TIMESTAMP` arm uses, deliberately shared rather than + * re-derived. + */ +class DateTimeFormatZoneSpec extends AnyFlatSpec with Matchers { + + private val schema: Table = Table( + "t", + columns = List(Column("name", SQLTypes.Keyword), Column("ts", SQLTypes.Timestamp)) + ) + + /** 🔴 Rendered WITH a `PainlessContext`, because that is the production path: the bridge builds + * one for every script field and script sort (`bridge/package.scala:563` and `:584`), and the + * extensions' materialized-view projection does the same. The context-FREE rendering of these + * four functions is separately malformed (`local-21.8-date-parse-emits-malformed-painless.md`); + * that is the deferred Part C, and asserting these claims through it would be measuring a + * surface no query uses. + */ + private def painlessOf(sql: String): String = + Parser(sql) match { + case Right(ss: SingleSearch) => + val ctx = PainlessContext(context = PainlessContextType.Query) + val body = ss.update(Some(schema)).select.fields.head.painless(Some(ctx)) + s"$ctx$body" + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + "DATETIME_FORMAT" should "format with EXACTLY the pattern the caller asked for" in { + // ES 8.18.3, same operand: before -> "2025 Z"; after -> "2025". + val painless = painlessOf("SELECT DATETIME_FORMAT(ts, 'yyyy') FROM t") + painless should include("""DateTimeFormatter.ofPattern("yyyy")""") + painless should not include "XXX" + } + + "DATETIME_PARSE" should "get its zone from the formatter, not from the caller's pattern" in { + // ES 8.18.3: `ZonedDateTime.parse("2025-01-10 10:00:00", ofPattern("yyyy-MM-dd HH:mm:ss") + // .withZone(ZoneId.of('Z')))` -> 2025-01-10T10:00Z. Before the fix the same statement threw. + val painless = painlessOf("SELECT DATETIME_PARSE(name, 'yyyy-MM-dd HH:mm:ss') FROM t") + painless should include("""DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")""") + painless should include("withZone(ZoneId.of('Z'))") + painless should not include "XXX" + } + + it should "still honour an offset the caller DID declare" in { + // `withZone` yields to a zone present in the text, so a caller who writes the offset into their + // own format keeps it. ES 8.18.3: "2025-01-10 10:00:00 +01:00" -> 2025-01-10T09:00Z. + val painless = painlessOf("SELECT DATETIME_PARSE(name, 'yyyy-MM-dd HH:mm:ss XXX') FROM t") + painless should include("""ofPattern("yyyy-MM-dd HH:mm:ss XXX")""") + painless should include("withZone(ZoneId.of('Z'))") + } + + "the DATE-only pair" should "be untouched - they never carried the suffix" in { + // `DATE_PARSE` yields a `LocalDate` and `DATE_FORMAT` formats one: no zone is involved, and + // neither ever appended anything. Pinned so the fix is shown to be narrow. + painlessOf("SELECT DATE_PARSE(name, 'yyyy-MM-dd') FROM t") should + include("""LocalDate.parse(param1, DateTimeFormatter.ofPattern("yyyy-MM-dd"))""") + painlessOf("SELECT DATE_PARSE(name, 'yyyy-MM-dd') FROM t") should not include "withZone" + painlessOf("SELECT DATE_FORMAT(ts, 'yyyy') FROM t") should + include("""DateTimeFormatter.ofPattern("yyyy")""") + } + + "a literal Z in the caller's pattern" should "still become X" in { + // Unrelated to the default-zone question and deliberately kept: to `ofPattern`, `Z` means + // "zone NAME", which does not accept the `Z` ISO-8601 writes for UTC; `X` does. + painlessOf("SELECT DATETIME_PARSE(name, 'yyyy-MM-ddZ') FROM t") should + include("""ofPattern("yyyy-MM-ddX")""") + } + + "the %f fractional-seconds format" should "emit a VARIABLE-width fraction, not a fixed one" in { + // 🔴 `%f` was mapped to `SSS` under a comment reading "microseconds". MEASURED on real ES + // 8.18.3, `S` is FIXED WIDTH in BOTH directions: `SSS` formats `.123456789` as `.123` and + // REFUSES to parse `.123456`; `SSSSSS` refuses to parse `.123`. Optional sections do not + // rescue it — `[.SSSSSS][.SSS]` formats as `.123456.123`. + // + // Picking a width would therefore NARROW one direction to widen the other — an "except" inside + // the very rule the temporal CAST arms in this same story are justified by. `appendFraction` + // is variable width, so the emission is a strict SUPERSET in both directions. Verified on ES + // 8.18.3 with the emitted script: parses `.123` (as before), `.123456`, `.123456789` and NO + // fraction at all; formats the value's real precision. + val painless = painlessOf("SELECT DATETIME_PARSE(name, '%Y-%m-%d %H:%i:%s.%f') FROM t") + painless should include("appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)") + painless should not include "SSS" + // The decimal point the caller wrote is absorbed INTO the fraction, which is what makes a + // zero-nanosecond value format as `12:00:00` rather than `12:00:00.` — measured. + painless should include("""appendPattern("yyyy-MM-dd HH:mm:ss")""") + } + + it should "leave a format with no fraction on the plain ofPattern path" in { + // The builder is emitted ONLY when a fraction is present, so nothing else moves. + painlessOf("SELECT DATETIME_PARSE(name, '%Y-%m-%d %H:%i:%s') FROM t") should + include("""DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")""") + painlessOf("SELECT DATETIME_PARSE(name, '%Y-%m-%d %H:%i:%s') FROM t") should + not include "DateTimeFormatterBuilder" + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 37f65ef8..fd0cb442 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -2286,7 +2286,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { SQLTypes.Int, source, true, - false // materialized: a DDL-declared script IS executed here + false, // materialized: a DDL-declared script IS executed here + _ // expr: the parsed AST, carried for the story-21.8 schema re-derivation ) ) => source should include( @@ -2322,7 +2323,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { SQLTypes.Int, source, true, - false // materialized: a DDL-declared script IS executed here + false, // materialized: a DDL-declared script IS executed here + _ // expr: the parsed AST, carried for the story-21.8 schema re-derivation ) ) => source should include( diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/TemporalCastFormatSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/TemporalCastFormatSpec.scala new file mode 100644 index 00000000..daded018 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/TemporalCastFormatSpec.scala @@ -0,0 +1,102 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 21.8 part B — the ` -> ` CAST arms hard-coded ONE format per target, so + * the commonest way a human writes a timestamp was rejected at script-execution time: + * + * - `CAST('2025-01-10 14:30:00' AS TIMESTAMP)` raised (`ISO_ZONED_DATE_TIME` demands both the + * `T` separator AND a zone); + * - `CAST('2025/01/10' AS DATE)` raised (`ofPattern("yyyy-MM-dd")`); + * - `CAST('14:30' AS TIME)` raised (`ofPattern("HH:mm:ss")`). + * + * And the engine disagreed with ITSELF: #276/BIDC-4 taught the WHERE path to resolve a + * space-separated literal against the mapped column's own `format`, so `WHERE ts = '2025-01-10 + * 14:30:00'` worked while the CAST of the same literal raised. + * + * The lead's OQ-3 ruling is "accept a small ordered format set per target". Painless has no + * expression-level `try`/`catch` and `ofPattern`'s optional sections cannot express ISO's + * variable-length fractional seconds, so the set is realised as a separator NORMALISATION in front + * of a WIDER ISO formatter — one parse, and a strict superset of what parsed before. + * + * 🔴 EVERY expectation below was EXECUTED on a real Elasticsearch 8.18.3 via `POST + * /_scripts/painless/_execute`, including the ISO forms that already worked (to prove the widening + * is not a swap) and the inputs that must STILL fail. A Painless claim is only provable by + * Elasticsearch. + */ +class TemporalCastFormatSpec extends AnyFlatSpec with Matchers { + + private val schema: Table = Table( + "t", + columns = List(Column("name", SQLTypes.Keyword), Column("ts", SQLTypes.Timestamp)) + ) + + private def painlessOf(sql: String): String = + Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(schema)).select.fields.head.painless(None) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + "a string cast to DATE" should "normalise the separator before parsing" in { + // ES 8.18.3: "2025/01/10" -> 2025-01-10, and "2025-01-10" -> 2025-01-10 (unchanged behaviour). + painlessOf("SELECT CAST('2025/01/10' AS DATE) FROM t") shouldBe + """LocalDate.parse(("2025/01/10").replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd"))""" + painlessOf("SELECT CAST('2025-01-10' AS DATE) FROM t") shouldBe + """LocalDate.parse(("2025-01-10").replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd"))""" + } + + it should "keep an ambiguous day-first layout a LOUD failure" in { + // 🔴 The rewrite deliberately does NOT rescue `10/01/2025`: it becomes `10-01-2025`, which no + // pattern accepts, so it still raises. Day-first and month-first are indistinguishable and + // guessing between them is how a date silently becomes a different date. Verified on ES + // 8.18.3: this script throws. Asserted on the EMISSION, since the engine's job here is to + // pass the literal through unrescued. + painlessOf("SELECT CAST('10/01/2025' AS DATE) FROM t") shouldBe + """LocalDate.parse(("10/01/2025").replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd"))""" + } + + "a string cast to TIME" should "use the ISO formatter rather than one hard-coded layout" in { + // ES 8.18.3: "14:30:00" (as before), "14:30" and "14:30:00.123" (new) all parse. + painlessOf("SELECT CAST('14:30:00' AS TIME) FROM t") shouldBe + """LocalTime.parse("14:30:00", DateTimeFormatter.ISO_LOCAL_TIME)""" + } + + "a string cast to DATETIME" should "accept the space separator" in { + // ES 8.18.3: both forms -> 2025-01-10T14:30. + painlessOf("SELECT CAST('2025-01-10 14:30:00' AS DATETIME) FROM t") shouldBe + """LocalDateTime.parse(("2025-01-10 14:30:00").replace(" ", "T"), DateTimeFormatter.ISO_DATE_TIME)""" + } + + "a string cast to TIMESTAMP" should "accept the space separator AND a missing zone" in { + // 🔴 `ISO_ZONED_DATE_TIME` REQUIRES an offset, so the space form failed twice over. `withZone` + // supplies UTC only when the text carried none -- MEASURED on ES 8.18.3: + // "2025-01-10 14:30:00" -> 2025-01-10T14:30Z + // "2025-01-10T14:30:00Z" -> 2025-01-10T14:30Z (unchanged) + // "2025-01-10T14:30:00+01:00" -> 2025-01-10T13:30Z (the explicit offset WINS) + // "2025-01-10T14:30:00.123Z" -> 2025-01-10T14:30:00.123Z (fractional seconds survive) + // "…+01:00[Europe/Paris]" -> kept as the zone region + // "not a date" -> still throws + painlessOf("SELECT CAST('2025-01-10 14:30:00' AS TIMESTAMP) FROM t") shouldBe + """ZonedDateTime.parse(("2025-01-10 14:30:00").replace(" ", "T"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z')))""" + } + + "a temporal cast over a COLUMN" should "normalise the same way" in { + // #306 made a column operand reach these arms at all; the normalisation must not be + // literal-only, or the two operand kinds would disagree about which layouts are accepted. + painlessOf("SELECT CAST(name AS DATE) FROM t") should include(""".replace("/", "-")""") + painlessOf("SELECT CAST(name AS TIMESTAMP) FROM t") should include(""".replace(" ", "T")""") + painlessOf("SELECT CAST(name AS TIMESTAMP) FROM t") should include("withZone(ZoneId.of('Z'))") + } + + it should "stay null-guarded" in { + // These four arms return from inside `ctx.addParam(...)`, so they never reach the end-of-method + // wrapper and must guard themselves (#306). The normalisation must not have moved that guard: + // `null.replace(...)` would be an NPE per document missing the field. + painlessOf("SELECT CAST(name AS DATE) FROM t") should include("!= null") + painlessOf("SELECT CAST(name AS TIMESTAMP) FROM t") should include("!= null") + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala new file mode 100644 index 00000000..594fcb06 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala @@ -0,0 +1,174 @@ +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterTable, CreateTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 21.8 part G — a DDL `SCRIPT AS (…)` built its ingest processor with NO schema attached. + * + * #306 (story 21.5) made the schema reach the QUERY execution path. It never reached the DDL one: + * the processor is derived at PARSE time, when the column list does not exist yet, so every + * operand's `baseType` was `Any`, no `SQLTypeUtils.coerce` arm fired, and the ingest script stored + * the UNCONVERTED operand. + * + * MEASURED before the fix: + * {{{ + * CREATE TABLE t (zip_code KEYWORD, zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT))) + * -> ctx.zip_n = param1 // the keyword STRING, into a `long`-mapped field + * }}} + * `zip_code KEYWORD` is declared in the same statement and was not consulted. Nothing failed: + * Elasticsearch never rewrites `_source` and `coerce` defaults true, so the mapping stayed correct + * while the document was wrong — the #205/#253 silent-wrong-value family, invisible to every + * mapping-level check. + * + * 🔴 The fix is ONE seam, `Table.update()`, and that is what covers both DDL sites. + * `CreateTable.schema` builds its table from the parsed columns and ends there; `Table.merge` + * applies `ALTER … SET SCRIPT AS` to the LIVE table and ends there too. The spec expected `ALTER` + * to need a client-side schema load because the STATEMENT carries no column list — but the point + * where it is APPLIED already holds the whole table, so neither site needs I/O and neither adds + * anything to the parse path. + */ +class DdlScriptSchemaSpec extends AnyFlatSpec with Matchers { + + private def createdColumns(sql: String): List[Column] = + Parser(sql) match { + case Right(ct: CreateTable) => ct.schema.columns + case other => fail(s"[$sql] expected a CreateTable, got $other") + } + + private def sourceOf(cols: List[Column], column: String): String = + cols + .find(_.name == column) + .flatMap(_.script) + .map(_.source) + .getOrElse(fail(s"no script processor for column [$column] in ${cols.map(_.name)}")) + + private def createSource(sql: String, column: String): String = + sourceOf(createdColumns(sql), column) + + // -- AC-G1: CREATE TABLE ---------------------------------------------------------------------- + + "a computed column over a sibling KEYWORD" should "convert on ingest" in { + // The headline shape. Verified end to end on a real Elasticsearch 8.18.3: this exact `source` + // run as an ingest pipeline over {"zip_code":"75001"} stores `"zip_n": 75001` — the NUMBER. + // 🔴 Asserted on the stored `_source` there, never on the mapping and never on a read path: a + // correct mapping and a wrong `_source` coexist silently, and a read path cannot tell a stored + // string from a read-time conversion. + val source = createSource( + "CREATE TABLE t (zip_code KEYWORD, zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT)))", + "zip_n" + ) + source should include("Long.parseLong") + source should include("ctx.zip_n =") + } + + "a computed column that NARROWS a sibling numeric" should "convert on ingest" in { + // The C2 (narrowing) arms, equally unreachable from DDL before this story. Verified on ES + // 8.18.3: {"amount":12.9} stores `"cents": 12`. + createSource( + "CREATE TABLE t (amount DOUBLE, cents BIGINT SCRIPT AS (CAST(amount AS BIGINT)))", + "cents" + ) should include("((long) param1)") + } + + // -- AC-G3: the ingest-context guard ---------------------------------------------------------- + + "a TEMPORAL-source arm" should "NOT fire in ingest context, byte for byte" in { + // 🔴 In an ingest script the operand is `ctx.` — the RAW JSON scalar — not the temporal + // object a query's `doc['f'].value` yields, so an arm keyed on a temporal SOURCE must decline. + // Attaching the schema is what makes those arms reachable for the first time, which turns that + // guard from theoretical into load-bearing. The expected bytes are the PRE-fix emission, + // captured on the baseline: this column's script must not have moved at all. + createSource( + "CREATE TABLE t (created DATE, y INTEGER SCRIPT AS (YEAR(created)))", + "y" + ) shouldBe "def param1 = ctx.created.get(ChronoField.YEAR); ctx.y = param1" + } + + "a string function over a sibling KEYWORD" should "be unchanged" in { + // A keyword is a `String` in BOTH contexts, so this arm was always correct and must stay so. + createSource( + "CREATE TABLE t (a KEYWORD, b KEYWORD SCRIPT AS (UPPER(a)))", + "b" + ) shouldBe "def param1 = ctx.a; ctx.b = (param1 == null) ? null : param1.toUpperCase()" + } + + // -- OQ-4: an operand the schema cannot resolve ----------------------------------------------- + + "an operand that names no declared column" should "stay silent and emit the identity" in { + // The lead's OQ-4 ruling. A `CREATE TABLE` that works today keeps working: the conversion + // simply does not fire, exactly as before, rather than the statement being rejected. + createSource( + "CREATE TABLE t (zip_n BIGINT SCRIPT AS (CAST(absent_column AS BIGINT)))", + "zip_n" + ) shouldBe "def param1 = ctx.absent_column; ctx.zip_n = param1" + } + + // -- the DDL text is not disturbed ------------------------------------------------------------ + + "the rendered DDL" should "round-trip unchanged" in { + // `Column.sql` renders ` SCRIPT AS ()` from the processor's `script` text, and the + // re-derivation must not perturb it: materialized-view deployment renders a stage's schema to + // DDL and runs the text, so a render that stopped re-parsing would break deployment, not a test. + val sql = "CREATE TABLE t (zip_code KEYWORD, zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT)))" + val rendered = createdColumns(sql).map(_.sql).mkString(", ") + rendered should include("zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT))") + Parser(s"CREATE TABLE t2 ($rendered)") match { + case Right(_: CreateTable) => succeed + case other => fail(s"generated DDL did not re-parse: $other") + } + } + + // -- AC-G2: the ALTER site -------------------------------------------------------------------- + + "ALTER TABLE ... SET SCRIPT AS" should "convert against the LIVE table" in { + // The site the spec expected to need a client-side schema load. `Table.merge` applies the + // statement to the table that was loaded from Elasticsearch and then calls `update()`, so the + // whole column list is already in hand. + val live = Table( + "t", + columns = List(Column("zip_code", SQLTypes.Keyword), Column("zip_n", SQLTypes.BigInt)) + ).update() + val statements = + Parser("ALTER TABLE t ALTER COLUMN zip_n SET SCRIPT AS (CAST(zip_code AS BIGINT))") match { + case Right(at: AlterTable) => at.statements + case other => fail(s"expected an AlterTable, got $other") + } + sourceOf(live.merge(statements).columns, "zip_n") should include("Long.parseLong") + } + + it should "stay silent when the live table does not know the operand" in { + val live = Table("t", columns = List(Column("zip_n", SQLTypes.BigInt))).update() + val statements = + Parser("ALTER TABLE t ALTER COLUMN zip_n SET SCRIPT AS (CAST(absent AS BIGINT))") match { + case Right(at: AlterTable) => at.statements + case other => fail(s"expected an AlterTable, got $other") + } + sourceOf(live.merge(statements).columns, "zip_n") shouldBe + "def param1 = ctx.absent; ctx.zip_n = param1" + } + + // -- the load path keeps what it read --------------------------------------------------------- + + "a processor read back from Elasticsearch" should "keep its stored source untouched" in { + // It has JSON and no AST, so `expr` is `None` and `resolvedAgainst` returns it unchanged. That + // is what keeps this change from rewriting pipelines it merely looked at. + val stored = ScriptProcessor( + script = "CAST(zip_code AS BIGINT)", + column = "zip_n", + dataType = SQLTypes.BigInt, + source = "def param1 = ctx.zip_code; ctx.zip_n = param1" + ) + stored.expr shouldBe None + val table = Table( + "t", + columns = List( + Column("zip_code", SQLTypes.Keyword), + Column("zip_n", SQLTypes.BigInt, script = Some(stored)) + ) + ).update() + sourceOf(table.columns, "zip_n") shouldBe "def param1 = ctx.zip_code; ctx.zip_n = param1" + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala index cee6a281..c3b46b3f 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala @@ -16,7 +16,17 @@ package app.softnetwork.elastic.client.repl -import app.softnetwork.elastic.client.result.{DmlResult, OutputFormat, QueryRows, QueryStructured} +import app.softnetwork.elastic.client.result.{ + DmlResult, + OutputFormat, + QueryRows, + QueryStream, + QueryStructured, + StreamResult +} +import akka.stream.scaladsl.Sink + +import scala.collection.immutable.ListMap import app.softnetwork.elastic.scalatest.ElasticTestKit import java.time.LocalDate @@ -1523,4 +1533,158 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { row should contain key "new_tier" row should contain key "expires_at" } + + // ========================================================================= + // Story 21.8 — conversion defects, on a REAL cluster, on every client + // ========================================================================= + + behavior of "REPL - story 21.8 conversions" + + /** Rows from a SELECT, whatever variant the result came back as. + * + * 🔴 `assertQueryRows` accepts `QueryRows` ONLY, and the variant is not a property of the + * statement: it depends on the client and on the statement's SHAPE. An un-LIMITed row query is + * routed through scroll (#209) and arrives as `StreamResult`; an aggregation arrives as + * `QueryStructured`; ES 9 hands back a `QueryStream`. Asserting through `assertQueryRows` here + * would fail for reasons that have nothing to do with the conversion under test — it did, on ES + * 8.18, before this helper existed. + */ + /** The scalar behind a SCRIPT FIELD's value. + * + * 🔴 Elasticsearch returns `script_fields` values inside its per-field ARRAY — a computed column + * arrives as `List(true)`, not `true` — on every path and every client. That is pre-existing and + * unrelated to this story (it is the unfiled `script_fields` wrapping recorded against #209); + * unwrapping here keeps these cases about the CONVERSION rather than about the envelope. Columns + * read from `_source` are not wrapped, so this is a no-op for them. + */ + private def scalar(v: Any): Any = v match { + case Seq(one) => one + case other => other + } + + private def conversionRows(res: ExecutionResult): Seq[ListMap[String, Any]] = { + res shouldBe a[ExecutionSuccess] + res.asInstanceOf[ExecutionSuccess].result match { + case q: QueryRows => q.rows + case q: QueryStructured => q.response.results + case QueryStream(stream, _) => stream.map(_._1).runWith(Sink.seq).futureValue + case StreamResult(_, _) => testRepl.consumeStreamSync() + case other => fail(s"Unexpected result variant for a SELECT: $other") + } + } + + /** 🔴 Part G, and the assertion is on the STORED VALUE, not on the mapping and not on a read-time + * conversion. + * + * A DDL computed column used to be compiled with no schema attached, so `CAST(zip_code AS + * BIGINT)` emitted the operand unchanged and the ingest pipeline stored the KEYWORD STRING into + * a `long`-mapped field. Nothing failed: Elasticsearch never rewrites `_source` and `coerce` + * defaults true, so the mapping stayed correct while the document was wrong. A mapping-level + * check cannot see that, and neither can a read path that converts on the way out — which is why + * this reads the value back and asserts its RUNTIME TYPE is a number. + */ + it should "store a DDL computed column CONVERTED, not as the raw operand" in { + assertDdl( + System.nanoTime(), + executeSync( + """CREATE TABLE IF NOT EXISTS conv_ddl ( + | id INT NOT NULL, + | zip_code KEYWORD, + | zip_n BIGINT SCRIPT AS (CAST(zip_code AS BIGINT)), + | amount DOUBLE, + | cents BIGINT SCRIPT AS (CAST(amount AS BIGINT)), + | PRIMARY KEY (id) + |)""".stripMargin + ) + ) + + assertDml( + System.nanoTime(), + executeSync("INSERT INTO conv_ddl (id, zip_code, amount) VALUES (1, '75001', 12.9)") + ) + + val rows = conversionRows(executeSync("SELECT zip_n, cents FROM conv_ddl WHERE id = 1")) + rows should have size 1 + val row = rows.head + + withClue(s"zip_n came back as ${row("zip_n").getClass.getName}: ") { + row("zip_n") shouldBe a[java.lang.Number] + } + row("zip_n").asInstanceOf[java.lang.Number].longValue() shouldBe 75001L + withClue(s"cents came back as ${row("cents").getClass.getName}: ") { + row("cents") shouldBe a[java.lang.Number] + } + // 12.9 narrowed to a long TRUNCATES toward zero, exactly as a Java cast does. + row("cents").asInstanceOf[java.lang.Number].longValue() shouldBe 12L + } + + /** Parts A and B: `CAST(… AS BOOLEAN)` was a silent no-op, and the temporal arms hard-coded one + * format per target so the commonest spelling of a timestamp raised at script-execution time. + * + * Executed rather than inspected: `Boolean.parseBoolean`, `String.replace(CharSequence, + * CharSequence)` and `DateTimeFormatter.withZone` are Painless whitelist claims, and only a real + * Elasticsearch settles those — on EVERY major, which is what this spec's five subclasses buy. + */ + it should "convert a cast to BOOLEAN instead of returning the operand" in { + assertDdl( + System.nanoTime(), + executeSync( + """CREATE TABLE IF NOT EXISTS conv_cast ( + | id INT NOT NULL, + | flag KEYWORD, + | n INT, + | PRIMARY KEY (id) + |)""".stripMargin + ) + ) + assertDml( + System.nanoTime(), + executeSync("INSERT INTO conv_cast (id, flag, n) VALUES (1, 'true', 0), (2, 'nope', 7)") + ) + + val rows = conversionRows( + executeSync( + "SELECT id, CAST(flag AS BOOLEAN) AS f, CAST(n AS BOOLEAN) AS b FROM conv_cast ORDER BY id ASC" + ) + ) + rows should have size 2 + val byId = rows.map(r => r("id").toString -> r).toMap + // C-style, the lead's PD-1 ruling: 'true' parses true, any other string is false; zero is + // false and every other number true. + scalar(byId("1")("f")) shouldBe true + scalar(byId("1")("b")) shouldBe false + scalar(byId("2")("f")) shouldBe false + scalar(byId("2")("b")) shouldBe true + } + + it should "accept the space-separated spelling of a timestamp in a CAST" in { + // The inconsistency a user met immediately: #276 taught the WHERE path to accept this literal + // while the CAST of the SAME literal raised. Both spellings must now yield the same instant. + val rows = conversionRows( + executeSync( + """SELECT CAST('2025-01-10 14:30:00' AS TIMESTAMP) AS spaced, + | CAST('2025-01-10T14:30:00Z' AS TIMESTAMP) AS iso, + | CAST('2025/01/10' AS DATE) AS slashed, + | CAST('2025-01-10' AS DATE) AS dashed + | FROM conv_cast WHERE id = 1""".stripMargin + ) + ) + rows should have size 1 + val row = rows.head + scalar(row("spaced")).toString shouldBe scalar(row("iso")).toString + scalar(row("slashed")).toString shouldBe scalar(row("dashed")).toString + } + + it should "format a DATETIME with EXACTLY the requested pattern" in { + // Before this story the engine appended ` XXX` to the caller's pattern, so a request for + // `'yyyy'` returned `"2025 Z"` — a silent wrong answer on the ordinary script-field path. + val rows = conversionRows( + executeSync( + "SELECT DATETIME_FORMAT(CAST('2025-01-10 14:30:00' AS TIMESTAMP), 'yyyy') AS y " + + "FROM conv_cast WHERE id = 1" + ) + ) + rows should have size 1 + scalar(rows.head("y")) shouldBe "2025" + } }