From 4c544697ccc61526eaef0295b975132a1ceee416 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 16 Apr 2025 14:49:55 -0700 Subject: [PATCH 1/4] wip --- .../logsdb/datageneration/FieldType.java | 12 +++- .../datasource/DataSourceHandler.java | 4 ++ .../datasource/DataSourceRequest.java | 6 ++ .../datasource/DataSourceResponse.java | 2 + .../DefaultMappingParametersHandler.java | 25 +++++++ .../ConstantKeywordFieldDataGenerator.java | 29 ++++++++ .../CountedKeywordFieldDataGenerator.java | 3 - .../leaf/KeywordFieldDataGenerator.java | 2 +- .../leaf/WildcardFieldDataGenerator.java | 28 ++++++++ .../matchers/source/FieldSpecificMatcher.java | 44 +++++++++++- .../ConstantKeywordFieldBlockLoaderTests.java | 34 +++++++++ .../xpack/versionfield/Utils.java | 33 +++++++++ .../VersionStringFieldBlockLoaderTests.java | 53 ++++++++++++++ .../VersionStringFieldMapperTests.java | 24 +------ .../VersionStringDataSourceHandler.java | 41 +++++++++++ .../VersionStringFieldDataGenerator.java | 31 +++++++++ .../mapper/WildcardFieldBlockLoaderTests.java | 69 +++++++++++++++++++ 17 files changed, 411 insertions(+), 29 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ConstantKeywordFieldDataGenerator.java create mode 100644 test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/WildcardFieldDataGenerator.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java create mode 100644 x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java create mode 100644 x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldBlockLoaderTests.java create mode 100644 x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java create mode 100644 x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringFieldDataGenerator.java create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldBlockLoaderTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 0f5cae96971ee..02ed16d22f7ee 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -12,6 +12,7 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator; @@ -26,6 +27,7 @@ import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.TextFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.fields.leaf.WildcardFieldDataGenerator; /** * Lists all leaf field types that are supported for data generation by default. @@ -46,7 +48,9 @@ public enum FieldType { DATE("date"), GEO_POINT("geo_point"), TEXT("text"), - IP("ip"); + IP("ip"), + CONSTANT_KEYWORD("constant_keyword"), + WILDCARD("wildcard"); private final String name; @@ -56,7 +60,7 @@ public enum FieldType { public FieldDataGenerator generator(String fieldName, DataSource dataSource) { return switch (this) { - case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource); + case KEYWORD -> new KeywordFieldDataGenerator(dataSource); case LONG -> new LongFieldDataGenerator(fieldName, dataSource); case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource); case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource); @@ -72,6 +76,8 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource); case TEXT -> new TextFieldDataGenerator(dataSource); case IP -> new IpFieldDataGenerator(dataSource); + case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator(); + case WILDCARD -> new WildcardFieldDataGenerator(dataSource); }; } @@ -93,6 +99,8 @@ public static FieldType tryParse(String name) { case "geo_point" -> FieldType.GEO_POINT; case "text" -> FieldType.TEXT; case "ip" -> FieldType.IP; + case "constant_keyword" -> FieldType.CONSTANT_KEYWORD; + case "wildcard" -> FieldType.WILDCARD; default -> null; }; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 5431c8410090e..ea0436fefc7be 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -78,6 +78,10 @@ default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator requ return null; } + default DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) { + return null; + } + default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 9825b9e2b092d..bba4361ff7395 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -126,6 +126,12 @@ public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) { } } + record VersionStringGenerator() implements DataSourceRequest { + public DataSourceResponse.VersionStringGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record NullWrapper() implements DataSourceRequest { public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) { return handler.handle(this); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index fd7d52d34dfe1..e5575da36108c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -53,6 +53,8 @@ record GeoPointGenerator(Supplier generator) implements DataSourceRespon record IpGenerator(Supplier generator) implements DataSourceResponse {} + record VersionStringGenerator(Supplier generator) implements DataSourceResponse {} + record NullWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 6301657465800..3d98ffe545f3b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -51,6 +51,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques case GEO_POINT -> geoPointMapping(map); case TEXT -> textMapping(request, new HashMap<>()); case IP -> ipMapping(map); + case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>()); + case WILDCARD -> wildcardMapping(new HashMap<>()); }); } @@ -225,6 +227,29 @@ private Supplier> ipMapping(Map injected) { }; } + private Supplier> constantKeywordMapping(Map injected) { + return () -> { + // value is optional and can be set from the first document + // we don't cover this case here + injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); + + return injected; + }; + } + + private Supplier> wildcardMapping(Map injected) { + return () -> { + if (ESTestCase.randomDouble() <= 0.2) { + injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); + } + if (ESTestCase.randomDouble() <= 0.2) { + injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); + } + + return injected; + }; + } + private static HashMap commonMappingParameters() { var map = new HashMap(); map.put("store", ESTestCase.randomBoolean()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ConstantKeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ConstantKeywordFieldDataGenerator.java new file mode 100644 index 0000000000000..c93c8942fa477 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ConstantKeywordFieldDataGenerator.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; + +import java.util.Map; + +public class ConstantKeywordFieldDataGenerator implements FieldDataGenerator { + @Override + public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // Dynamically mapped, skip it because it will be mapped as text, and we cover this case already + return null; + } + + var value = fieldMapping.get("value"); + assert value != null; + + return value; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java index 64a40704a64ce..b69ad33016096 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java @@ -13,14 +13,11 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.function.Supplier; public class CountedKeywordFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Set previousStrings = new HashSet<>(); public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) { var strings = dataSource.get(new DataSourceRequest.StringGenerator()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java index 4dbbc6b740dc2..1a433f20a9218 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java @@ -19,7 +19,7 @@ public class KeywordFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) { + public KeywordFieldDataGenerator(DataSource dataSource) { var strings = dataSource.get(new DataSourceRequest.StringGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/WildcardFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/WildcardFieldDataGenerator.java new file mode 100644 index 0000000000000..b88459bec700a --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/WildcardFieldDataGenerator.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; + +import java.util.Map; + +public class WildcardFieldDataGenerator implements FieldDataGenerator { + private final FieldDataGenerator keywordGenerator; + + public WildcardFieldDataGenerator(DataSource dataSource) { + this.keywordGenerator = new KeywordFieldDataGenerator(dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + return keywordGenerator.generateValue(fieldMapping); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index 147d6fa33424f..4ab2df51ff8b8 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -46,7 +46,6 @@ static Map matchers( return new HashMap<>() { { put("keyword", new KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); - put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("long", new NumberMatcher("long", actualMappings, actualSettings, expectedMappings, expectedSettings)); put("unsigned_long", new UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("integer", new NumberMatcher("integer", actualMappings, actualSettings, expectedMappings, expectedSettings)); @@ -58,11 +57,14 @@ static Map matchers( put("scaled_float", new ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("counted_keyword", new CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("boolean", new BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("geo_shape", new ExactMatcher("geo_shape", actualMappings, actualSettings, expectedMappings, expectedSettings)); put("shape", new ExactMatcher("shape", actualMappings, actualSettings, expectedMappings, expectedSettings)); put("geo_point", new GeoPointMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("text", new TextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("ip", new IpMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("constant_keyword", new ConstantKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("wildcard", new WildcardMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); } }; } @@ -691,6 +693,46 @@ Object convert(Object value, Object nullValue) { } } + class ConstantKeywordMatcher extends GenericMappingAwareMatcher { + ConstantKeywordMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + super("constant_keyword", actualMappings, actualSettings, expectedMappings, expectedSettings); + } + + @Override + Object convert(Object value, Object nullValue) { + // We just need to get rid of literal `null`s which is done in the caller. + return value; + } + } + + class WildcardMatcher extends GenericMappingAwareMatcher { + WildcardMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + super("wildcard", actualMappings, actualSettings, expectedMappings, expectedSettings); + } + + @Override + Object convert(Object value, Object nullValue) { + if (value == null) { + if (nullValue != null) { + return nullValue; + } + return null; + } + + return value; + } + } + /** * Generic matcher that supports common matching logic like null values. */ diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..67307d15878ee --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.constantkeyword.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class ConstantKeywordFieldBlockLoaderTests extends BlockLoaderTestCase { + public ConstantKeywordFieldBlockLoaderTests(Params params) { + super(FieldType.CONSTANT_KEYWORD.toString(), params); + } + + @Override + protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + return new BytesRef((String) fieldMapping.get("value")); + } + + @Override + protected Collection getPlugins() { + return List.of(new ConstantKeywordMapperPlugin()); + } +} diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java new file mode 100644 index 0000000000000..0d85278f34160 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.test.ESTestCase; + +public class Utils { + public static String randomVersionString() { + return randomVersionNumber() + (ESTestCase.randomBoolean() ? "" : randomPrerelease()); + } + + private static String randomVersionNumber() { + int numbers = ESTestCase.between(1, 3); + String v = Integer.toString(ESTestCase.between(0, 100)); + for (int i = 1; i < numbers; i++) { + v += "." + ESTestCase.between(0, 100); + } + return v; + } + + private static String randomPrerelease() { + if (ESTestCase.rarely()) { + return ESTestCase.randomFrom("alpha", "beta", "prerelease", "whatever"); + } + return ESTestCase.randomFrom("alpha", "beta", "") + randomVersionNumber(); + } + +} diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldBlockLoaderTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..a3c586c812710 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldBlockLoaderTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.versionfield.datageneration.VersionStringDataSourceHandler; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class VersionStringFieldBlockLoaderTests extends BlockLoaderTestCase { + public VersionStringFieldBlockLoaderTests(Params params) { + super("version", List.of(new VersionStringDataSourceHandler()), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + if (value == null) { + return null; + } + + if (value instanceof String s) { + return convert(s); + } + + var resultList = ((List) value).stream().map(this::convert).filter(Objects::nonNull).distinct().sorted().toList(); + return maybeFoldList(resultList); + } + + private BytesRef convert(String value) { + if (value == null) { + return null; + } + + return VersionEncoder.encodeVersion(value).bytesRef; + } + + @Override + protected Collection getPlugins() { + return Collections.singletonList(new VersionFieldPlugin(getIndexSettings())); + } +} diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 5653ed7f4302f..25ad2bb4ce771 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -148,27 +148,7 @@ public void testFailsParsingNestedList() throws IOException { @Override protected String generateRandomInputValue(MappedFieldType ft) { - return randomValue(); - } - - protected static String randomValue() { - return randomVersionNumber() + (randomBoolean() ? "" : randomPrerelease()); - } - - private static String randomVersionNumber() { - int numbers = between(1, 3); - String v = Integer.toString(between(0, 100)); - for (int i = 1; i < numbers; i++) { - v += "." + between(0, 100); - } - return v; - } - - private static String randomPrerelease() { - if (rarely()) { - return randomFrom("alpha", "beta", "prerelease", "whatever"); - } - return randomFrom("alpha", "beta", "") + randomVersionNumber(); + return Utils.randomVersionString(); } @Override @@ -212,7 +192,7 @@ public SyntheticSourceExample example(int maxValues) { } private Tuple generateValue() { - String v = randomValue(); + String v = Utils.randomVersionString(); return Tuple.tuple(v, v); } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java new file mode 100644 index 0000000000000..bb7f9d55d6003 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield.datageneration; + +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.xpack.versionfield.Utils; + +import java.util.HashMap; + +public class VersionStringDataSourceHandler implements DataSourceHandler { + @Override + public DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) { + return new DataSourceResponse.VersionStringGenerator(Utils::randomVersionString); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType().equals("version") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(HashMap::new); + } + + @Override + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + if (request.fieldType().equals("version") == false) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(new VersionStringFieldDataGenerator(request.dataSource())); + } + +} diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringFieldDataGenerator.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringFieldDataGenerator.java new file mode 100644 index 0000000000000..400c08a6a97be --- /dev/null +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringFieldDataGenerator.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield.datageneration; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.Map; +import java.util.function.Supplier; + +public class VersionStringFieldDataGenerator implements FieldDataGenerator { + private final Supplier values; + + public VersionStringFieldDataGenerator(DataSource dataSource) { + var nullWrapper = dataSource.get(new DataSourceRequest.NullWrapper()); + var versionStrings = dataSource.get(new DataSourceRequest.VersionStringGenerator()); + + this.values = nullWrapper.wrapper().apply(versionStrings.generator()::get); + } + + @Override + public Object generateValue(Map fieldMapping) { + return this.values.get(); + } +} diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldBlockLoaderTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..ca8a147805fb2 --- /dev/null +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldBlockLoaderTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.wildcard.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.wildcard.Wildcard; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class WildcardFieldBlockLoaderTests extends BlockLoaderTestCase { + public WildcardFieldBlockLoaderTests(Params params) { + super(FieldType.WILDCARD.toString(), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + var nullValue = (String) fieldMapping.get("null_value"); + + var ignoreAbove = fieldMapping.get("ignore_above") == null + ? Integer.MAX_VALUE + : ((Number) fieldMapping.get("ignore_above")).intValue(); + + if (value == null) { + return convert(null, nullValue, ignoreAbove); + } + + if (value instanceof String s) { + return convert(s, nullValue, ignoreAbove); + } + + var resultList = ((List) value).stream() + .map(s -> convert(s, nullValue, ignoreAbove)) + .filter(Objects::nonNull) + .distinct() + .sorted() + .toList(); + return maybeFoldList(resultList); + } + + private static BytesRef convert(String value, String nullValue, int ignoreAbove) { + if (value == null) { + if (nullValue != null) { + value = nullValue; + } else { + return null; + } + } + + return value.length() <= ignoreAbove ? new BytesRef(value) : null; + } + + @Override + protected Collection getPlugins() { + return Collections.singleton(new Wildcard()); + } +} From 9ea92f25dab0852d21b039fee4ce2ae848b18df6 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 16 Apr 2025 16:16:45 -0700 Subject: [PATCH 2/4] fix test dependencies --- test/framework/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/framework/build.gradle b/test/framework/build.gradle index a5bff2d0166cc..9601ee2c6a648 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -32,9 +32,12 @@ dependencies { api "org.elasticsearch:mocksocket:${versions.mocksocket}" + testImplementation project(":modules:mapper-extras") + testImplementation project(':x-pack:plugin:core') testImplementation project(':x-pack:plugin:mapper-unsigned-long') testImplementation project(':x-pack:plugin:mapper-counted-keyword') - testImplementation project(":modules:mapper-extras") + testImplementation project(':x-pack:plugin:mapper-constant-keyword') + testImplementation project(':x-pack:plugin:wildcard') } sourceSets { From e303926ae54d98e6e764017bf09e06ee386f152c Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 17 Apr 2025 09:42:22 -0700 Subject: [PATCH 3/4] fix --- .../logsdb/datageneration/DataGenerationTests.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java index ee282425b7855..9b6fad48cb69b 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java @@ -20,8 +20,10 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; +import org.elasticsearch.xpack.wildcard.Wildcard; import java.io.IOException; import java.util.Collection; @@ -111,7 +113,13 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG var mappingService = new MapperServiceTestCase() { @Override protected Collection getPlugins() { - return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin(), new CountedKeywordMapperPlugin()); + return List.of( + new UnsignedLongMapperPlugin(), + new MapperExtrasPlugin(), + new CountedKeywordMapperPlugin(), + new ConstantKeywordMapperPlugin(), + new Wildcard() + ); } }.createMapperService(mappingXContent); From ad8ecdde708d6278fc0629caa0cab3ee56290f03 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 17 Apr 2025 09:43:04 -0700 Subject: [PATCH 4/4] Naming fix --- .../xpack/versionfield/VersionStringFieldMapperTests.java | 4 ++-- .../versionfield/{Utils.java => VersionStringTestUtils.java} | 2 +- .../datageneration/VersionStringDataSourceHandler.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/{Utils.java => VersionStringTestUtils.java} (96%) diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 25ad2bb4ce771..1b2245ba3a332 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -148,7 +148,7 @@ public void testFailsParsingNestedList() throws IOException { @Override protected String generateRandomInputValue(MappedFieldType ft) { - return Utils.randomVersionString(); + return VersionStringTestUtils.randomVersionString(); } @Override @@ -192,7 +192,7 @@ public SyntheticSourceExample example(int maxValues) { } private Tuple generateValue() { - String v = Utils.randomVersionString(); + String v = VersionStringTestUtils.randomVersionString(); return Tuple.tuple(v, v); } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringTestUtils.java similarity index 96% rename from x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java rename to x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringTestUtils.java index 0d85278f34160..1f01bc4c56bf0 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/Utils.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringTestUtils.java @@ -9,7 +9,7 @@ import org.elasticsearch.test.ESTestCase; -public class Utils { +public class VersionStringTestUtils { public static String randomVersionString() { return randomVersionNumber() + (ESTestCase.randomBoolean() ? "" : randomPrerelease()); } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java index bb7f9d55d6003..663389b4ed77f 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/datageneration/VersionStringDataSourceHandler.java @@ -10,14 +10,14 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xpack.versionfield.Utils; +import org.elasticsearch.xpack.versionfield.VersionStringTestUtils; import java.util.HashMap; public class VersionStringDataSourceHandler implements DataSourceHandler { @Override public DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) { - return new DataSourceResponse.VersionStringGenerator(Utils::randomVersionString); + return new DataSourceResponse.VersionStringGenerator(VersionStringTestUtils::randomVersionString); } @Override