Skip to content

Commit 85bad19

Browse files
penghuopenghuo
authored
Support match_only_text field type (#3663)
* Support match_only_text field type Signed-off-by: Peng Huo <[email protected]> * Update doc Signed-off-by: Peng Huo <[email protected]> * Update code format Signed-off-by: Peng Huo <[email protected]> * Update Signed-off-by: Peng Huo <[email protected]> * add plugins.calcite.fallback.allowed : false Signed-off-by: Peng Huo <[email protected]> --------- Signed-off-by: Peng Huo <[email protected]>
1 parent 7863959 commit 85bad19

File tree

5 files changed

+134
-3
lines changed

5 files changed

+134
-3
lines changed

‎docs/user/ppl/general/datatypes.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and
8686
+-----------------+---------------+-----------+
8787
| text | string | VARCHAR |
8888
+-----------------+---------------+-----------+
89+
| match_only_text | string | VARCHAR |
90+
+-----------------+---------------+-----------+
8991
| date | timestamp | TIMESTAMP |
9092
+-----------------+---------------+-----------+
9193
| ip | ip | VARCHAR |

‎integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.explain.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"ppl.explain": {
3+
"documentation": {
4+
"url": "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/index.rst",
5+
"description": "OpenSearch PPL Reference Manual"
6+
},
7+
"stability" : "stable",
8+
"url": {
9+
"paths": [
10+
{
11+
"path" : "/_plugins/_ppl/_explain",
12+
"methods" : ["POST"]
13+
}
14+
]
15+
},
16+
"params": {},
17+
"body": {
18+
"description": "PPL Explain Query",
19+
"required":true
20+
}
21+
}
22+
}

‎integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3655.yml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
setup:
2+
- skip:
3+
features:
4+
- headers
5+
- do:
6+
indices.create:
7+
index: test
8+
body:
9+
settings:
10+
number_of_replicas: 0
11+
number_of_shards: 1
12+
mappings:
13+
properties:
14+
id:
15+
type: integer
16+
body:
17+
type: match_only_text
18+
19+
- do:
20+
bulk:
21+
index: test
22+
refresh: true
23+
body:
24+
- '{"index": {}}'
25+
- '{"id": 1, "body": "This is a match_only_text field 1"}'
26+
- '{"index": {}}'
27+
- '{"id": 2, "body": "This is a match_only_text field 2"}'
28+
29+
---
30+
teardown:
31+
- do:
32+
query.settings:
33+
body:
34+
transient:
35+
plugins.calcite.enabled : false
36+
plugins.calcite.fallback.allowed : true
37+
38+
---
39+
"Support match_only_text field type with Calcite enabled":
40+
- do:
41+
query.settings:
42+
body:
43+
transient:
44+
plugins.calcite.enabled : true
45+
plugins.calcite.fallback.allowed : false
46+
47+
- do:
48+
headers:
49+
Content-Type: 'application/json'
50+
ppl:
51+
body:
52+
query: 'source=test | sort id | fields body | head 1'
53+
- match: {"total": 1}
54+
- match: {"schema": [{"name": "body", "type": "string"}]}
55+
- match: {"datarows": [["This is a match_only_text field 1"]]}
56+
57+
- do:
58+
headers:
59+
Content-Type: 'application/json'
60+
ppl:
61+
body:
62+
query: "source=test | where like(body, '%field 2%') | fields body"
63+
- match: {"total": 1}
64+
- match: {"schema": [{"name": "body", "type": "string"}]}
65+
- match: {"datarows": [["This is a match_only_text field 2"]]}
66+
67+
---
68+
"Support match_only_text field type with Calcite disabled":
69+
- do:
70+
headers:
71+
Content-Type: 'application/json'
72+
ppl:
73+
body:
74+
query: 'source=test | sort id | fields body | head 1'
75+
- match: {"total": 1}
76+
- match: {"schema": [{"name": "body", "type": "string"}]}
77+
- match: {"datarows": [["This is a match_only_text field 1"]]}
78+
79+
- do:
80+
headers:
81+
Content-Type: 'application/json'
82+
ppl:
83+
body:
84+
query: "source=test | where match_phrase(body, 'field 2') | fields body"
85+
- match: {"total": 1}
86+
- match: {"schema": [{"name": "body", "type": "string"}]}
87+
- match: {"datarows": [["This is a match_only_text field 2"]]}
88+
89+
---
90+
"Support match_only_text field type function push down with Calcite disabled":
91+
- do:
92+
headers:
93+
Content-Type: 'application/json'
94+
ppl.explain:
95+
body:
96+
query: "source=test | where match_phrase(body, 'field 2') | fields body"
97+
- match: { root.children.0.description.request: "OpenSearchQueryRequest(indexName=test, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"match_phrase\":{\"body\":{\"query\":\"field 2\",\"slop\":0,\"zero_terms_query\":\"NONE\",\"boost\":1.0}}},\"_source\":{\"includes\":[\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"}
98+
99+
100+
- do:
101+
headers:
102+
Content-Type: 'application/json'
103+
ppl.explain:
104+
body:
105+
query: "source=test | where like(body, '%field 2%') | fields body"
106+
- match: { root.children.0.description.request: "OpenSearchQueryRequest(indexName=test, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"body\":{\"wildcard\":\"*field 2*\",\"case_insensitive\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public class OpenSearchDataType implements ExprType, Serializable {
2525
public enum MappingType {
2626
Invalid(null, ExprCoreType.UNKNOWN),
2727
Text("text", ExprCoreType.UNKNOWN),
28+
MatchOnlyText("match_only_text", ExprCoreType.UNKNOWN),
2829
Keyword("keyword", ExprCoreType.STRING),
2930
Ip("ip", ExprCoreType.IP),
3031
GeoPoint("geo_point", ExprCoreType.UNKNOWN),
@@ -172,6 +173,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object>
172173
OpenSearchDataType objectDataType = res.cloneEmpty();
173174
objectDataType.properties = properties;
174175
return objectDataType;
176+
case MatchOnlyText:
175177
case Text:
176178
// TODO update these 2 below #1038 https://github.com/opensearch-project/sql/issues/1038
177179
Map<String, OpenSearchDataType> fields =

‎opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType;
6565
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
6666
import org.opensearch.sql.opensearch.data.type.OpenSearchDateType;
67+
import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
6768
import org.opensearch.sql.opensearch.data.utils.Content;
6869
import org.opensearch.sql.opensearch.data.utils.ObjectContent;
6970
import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent;
@@ -117,9 +118,7 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) {
117118
.put(
118119
OpenSearchDataType.of(OpenSearchDataType.MappingType.Double),
119120
(c, dt) -> new ExprDoubleValue(c.doubleValue()))
120-
.put(
121-
OpenSearchDataType.of(OpenSearchDataType.MappingType.Text),
122-
(c, dt) -> new OpenSearchExprTextValue(c.stringValue()))
121+
.put(OpenSearchTextType.of(), (c, dt) -> new OpenSearchExprTextValue(c.stringValue()))
123122
.put(
124123
OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword),
125124
(c, dt) -> new ExprStringValue(c.stringValue()))

0 commit comments

Comments
 (0)