Skip to content

Commit a111ee7

Browse files
not-napoleonnot-napoleon
authored
Esql implicit casting for date nanos (#118697) (#118888)
resolves #118476 This adds an implicit cast from string to date nanos, much the same as we do for millisecond dates. In the course of working on this, I found and fixed a couple of tests that were creating pre-epoch date nanos, which are not supported in elasticsearch. I also refactored the conversion code to use the standard DateUtils functions where appropriate, which caught some of the above errors in test data.
1 parent 68f6c78 commit a111ee7

File tree

7 files changed

+179
-22
lines changed

7 files changed

+179
-22
lines changed

‎docs/changelog/118697.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 118697
2+
summary: Esql implicit casting for date nanos
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 118476

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec

+131
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,137 @@ millis:date | nanos:date_nanos | num:long
216216
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
217217
;
218218

219+
implicit casting to nanos, date only
220+
required_capability: date_nanos_type
221+
required_capability: date_nanos_implicit_casting
222+
223+
FROM date_nanos
224+
| WHERE MV_MIN(nanos) > "2023-10-23"
225+
| SORT nanos DESC
226+
| KEEP millis, nanos;
227+
228+
millis:date | nanos:date_nanos
229+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
230+
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
231+
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
232+
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
233+
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
234+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
235+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
236+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
237+
;
238+
239+
implicit casting to nanos, date only, equality test
240+
required_capability: date_nanos_type
241+
required_capability: date_nanos_implicit_casting
242+
243+
FROM date_nanos
244+
| WHERE MV_MIN(nanos) == "2023-10-23"
245+
| SORT nanos DESC
246+
| KEEP millis, nanos;
247+
248+
millis:date | nanos:date_nanos
249+
;
250+
251+
252+
implicit casting to nanos, date plus time to seconds
253+
required_capability: date_nanos_type
254+
required_capability: date_nanos_implicit_casting
255+
256+
FROM date_nanos
257+
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00"
258+
| SORT nanos DESC
259+
| KEEP millis, nanos;
260+
261+
millis:date | nanos:date_nanos
262+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
263+
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
264+
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
265+
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
266+
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
267+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
268+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
269+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
270+
;
271+
272+
implicit casting to nanos, date plus time to seconds, equality test
273+
required_capability: date_nanos_type
274+
required_capability: date_nanos_implicit_casting
275+
276+
FROM date_nanos
277+
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28"
278+
| SORT nanos DESC
279+
| KEEP millis, nanos;
280+
281+
millis:date | nanos:date_nanos
282+
;
283+
284+
implicit casting to nanos, date plus time to millis
285+
required_capability: date_nanos_type
286+
required_capability: date_nanos_implicit_casting
287+
288+
FROM date_nanos
289+
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000"
290+
| SORT nanos DESC
291+
| KEEP millis, nanos;
292+
293+
millis:date | nanos:date_nanos
294+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
295+
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
296+
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
297+
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
298+
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
299+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
300+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
301+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
302+
;
303+
304+
implicit casting to nanos, date plus time to millis, equality test
305+
required_capability: date_nanos_type
306+
required_capability: date_nanos_implicit_casting
307+
308+
FROM date_nanos
309+
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948"
310+
| SORT nanos DESC
311+
| KEEP millis, nanos;
312+
313+
millis:date | nanos:date_nanos
314+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
315+
;
316+
317+
implicit casting to nanos, date plus time to nanos
318+
required_capability: date_nanos_type
319+
required_capability: date_nanos_implicit_casting
320+
321+
FROM date_nanos
322+
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000000000"
323+
| SORT nanos DESC
324+
| KEEP millis, nanos;
325+
326+
millis:date | nanos:date_nanos
327+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
328+
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
329+
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
330+
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
331+
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
332+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
333+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
334+
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
335+
;
336+
337+
implicit casting to nanos, date plus time to nanos, equality test
338+
required_capability: date_nanos_type
339+
required_capability: date_nanos_implicit_casting
340+
341+
FROM date_nanos
342+
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948000000"
343+
| SORT nanos DESC
344+
| KEEP millis, nanos;
345+
346+
millis:date | nanos:date_nanos
347+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
348+
;
349+
219350
date nanos greater than millis
220351
required_capability: date_nanos_type
221352
required_capability: date_nanos_compare_to_millis

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,10 @@ public enum Cap {
345345
* Support for mixed comparisons between nanosecond and millisecond dates
346346
*/
347347
DATE_NANOS_COMPARE_TO_MILLIS(),
348-
348+
/**
349+
* Support implicit casting of strings to date nanos
350+
*/
351+
DATE_NANOS_IMPLICIT_CASTING(),
349352
/**
350353
* Support Least and Greatest functions on Date Nanos type
351354
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

+17-14
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE;
119119
import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
120120
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
121+
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
121122
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
122123
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
123124
import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
@@ -1050,21 +1051,23 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
10501051
/**
10511052
* Cast string literals in ScalarFunction, EsqlArithmeticOperation, BinaryComparison, In and GroupingFunction to desired data types.
10521053
* For example, the string literals in the following expressions will be cast implicitly to the field data type on the left hand side.
1053-
* date > "2024-08-21"
1054-
* date in ("2024-08-21", "2024-08-22", "2024-08-23")
1055-
* date = "2024-08-21" + 3 days
1056-
* ip == "127.0.0.1"
1057-
* version != "1.0"
1058-
* bucket(dateField, "1 month")
1059-
* date_trunc("1 minute", dateField)
1060-
*
1054+
* <ul>
1055+
* <li>date > "2024-08-21"</li>
1056+
* <li>date in ("2024-08-21", "2024-08-22", "2024-08-23")</li>
1057+
* <li>date = "2024-08-21" + 3 days</li>
1058+
* <li>ip == "127.0.0.1"</li>
1059+
* <li>version != "1.0"</li>
1060+
* <li>bucket(dateField, "1 month")</li>
1061+
* <li>date_trunc("1 minute", dateField)</li>
1062+
* </ul>
10611063
* If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if
10621064
* applicable. For example, implicit casting converts:
1063-
* Coalesce(Long, Int) to Coalesce(Long, Long)
1064-
* Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
1065-
* Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)
1066-
* Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)
1067-
*
1065+
* <ul>
1066+
* <li>Coalesce(Long, Int) to Coalesce(Long, Long)</li>
1067+
* <li>Coalesce(null, Long, Int) to Coalesce(null, Long, Long)</li>
1068+
* <li>Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)</li>
1069+
* <li>Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)</li>
1070+
* </ul>
10681071
* Coalesce(Int, Long) will NOT be converted to Coalesce(Long, Long) or Coalesce(Int, Int).
10691072
*/
10701073
private static class ImplicitCasting extends ParameterizedRule<LogicalPlan, LogicalPlan, AnalyzerContext> {
@@ -1245,7 +1248,7 @@ private static boolean supportsImplicitTemporalCasting(Expression e, BinaryOpera
12451248
}
12461249

12471250
private static boolean supportsStringImplicitCasting(DataType type) {
1248-
return type == DATETIME || type == IP || type == VERSION || type == BOOLEAN;
1251+
return type == DATETIME || type == DATE_NANOS || type == IP || type == VERSION || type == BOOLEAN;
12491252
}
12501253

12511254
private static UnresolvedAttribute unresolvedAttribute(Expression value, String type, Exception e) {

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java

+9-5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import org.elasticsearch.common.logging.LoggerMessageFormat;
1414
import org.elasticsearch.common.lucene.BytesRefs;
1515
import org.elasticsearch.common.time.DateFormatter;
16+
import org.elasticsearch.common.time.DateFormatters;
17+
import org.elasticsearch.common.time.DateUtils;
1618
import org.elasticsearch.search.DocValueFormat;
1719
import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
1820
import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
@@ -51,7 +53,6 @@
5153
import java.time.Period;
5254
import java.time.ZoneId;
5355
import java.time.temporal.ChronoField;
54-
import java.time.temporal.TemporalAccessor;
5556
import java.time.temporal.TemporalAmount;
5657
import java.util.List;
5758
import java.util.Locale;
@@ -202,6 +203,9 @@ public static Converter converterFor(DataType from, DataType to) {
202203
if (to == DataType.DATETIME) {
203204
return EsqlConverter.STRING_TO_DATETIME;
204205
}
206+
if (to == DATE_NANOS) {
207+
return EsqlConverter.STRING_TO_DATE_NANOS;
208+
}
205209
if (to == DataType.IP) {
206210
return EsqlConverter.STRING_TO_IP;
207211
}
@@ -521,13 +525,12 @@ public static long dateTimeToLong(String dateTime, DateFormatter formatter) {
521525
}
522526

523527
public static long dateNanosToLong(String dateNano) {
524-
return dateNanosToLong(dateNano, DateFormatter.forPattern("strict_date_optional_time_nanos"));
528+
return dateNanosToLong(dateNano, DEFAULT_DATE_NANOS_FORMATTER);
525529
}
526530

527531
public static long dateNanosToLong(String dateNano, DateFormatter formatter) {
528-
TemporalAccessor parsed = formatter.parse(dateNano);
529-
long nanos = parsed.getLong(ChronoField.INSTANT_SECONDS) * 1_000_000_000 + parsed.getLong(ChronoField.NANO_OF_SECOND);
530-
return nanos;
532+
Instant parsed = DateFormatters.from(formatter.parse(dateNano)).toInstant();
533+
return DateUtils.toLong(parsed);
531534
}
532535

533536
public static String dateTimeToString(long dateTime) {
@@ -646,6 +649,7 @@ public enum EsqlConverter implements Converter {
646649
STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.TIME_DURATION)),
647650
STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono),
648651
STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)),
652+
STRING_TO_DATE_NANOS(x -> EsqlDataTypeConverter.dateNanosToLong((String) x)),
649653
STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)),
650654
STRING_TO_VERSION(x -> EsqlDataTypeConverter.stringToVersion((String) x)),
651655
STRING_TO_DOUBLE(x -> EsqlDataTypeConverter.stringToDouble((String) x)),

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ private Page randomPage(List<ColumnInfoImpl> columns) {
204204
case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(randomBoolean());
205205
case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
206206
// TODO - add a random instant thing here?
207-
case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomLong());
207+
case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomNonNegativeLong());
208208
case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(new Version(randomIdentifier()).toBytesRef());
209209
case GEO_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(GEO.asWkb(GeometryTestUtils.randomPoint()));
210210
case CARTESIAN_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(CARTESIAN.asWkb(ShapeTestUtils.randomPoint()));

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77

88
package org.elasticsearch.xpack.esql.type;
99

10+
import org.elasticsearch.common.time.DateUtils;
1011
import org.elasticsearch.test.ESTestCase;
1112
import org.elasticsearch.xpack.esql.core.type.DataType;
1213

14+
import java.time.Instant;
1315
import java.util.Arrays;
1416
import java.util.List;
1517

@@ -52,11 +54,19 @@
5254
public class EsqlDataTypeConverterTests extends ESTestCase {
5355

5456
public void testNanoTimeToString() {
55-
long expected = randomLong();
57+
long expected = randomNonNegativeLong();
5658
long actual = EsqlDataTypeConverter.dateNanosToLong(EsqlDataTypeConverter.nanoTimeToString(expected));
5759
assertEquals(expected, actual);
5860
}
5961

62+
public void testStringToDateNanos() {
63+
assertEquals(
64+
DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")),
65+
EsqlDataTypeConverter.convert("2023-01-01T00:00:00.000000000", DATE_NANOS)
66+
);
67+
assertEquals(DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")), EsqlDataTypeConverter.convert("2023-01-01", DATE_NANOS));
68+
}
69+
6070
public void testCommonTypeNull() {
6171
for (DataType dataType : DataType.values()) {
6272
assertEqualsCommonType(dataType, NULL, dataType);

0 commit comments

Comments
 (0)