diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 5afb4ff55e..cc68eeab4a 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -3,7 +3,7 @@ name: Release Drafter on: push: branches: - - develop + - main jobs: update_release_draft: @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 1.0.0.0-rc1 + version: 1.0.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 8ad294d853..816056bd33 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,7 +25,7 @@ jobs: - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false + run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false - name: Build with Gradle run: ./gradlew build assemble diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index b22f7649b3..0e8e712690 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -8,7 +8,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1 + OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 jobs: @@ -38,7 +38,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.23.1' + node-version: '10.24.1' - name: Move Workbench to Plugins Dir run: | diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 05a6541e34..cad24e6899 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1 + OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 jobs: @@ -27,7 +27,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.23.1' + node-version: '10.24.1' - name: Move Workbench to Plugins Dir run: | diff --git a/README.md b/README.md index bf76415459..0a66325424 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,22 @@ [![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/sql/) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) -# OpenSearch SQL + +- [OpenSearch SQL](#opensearch-sql) +- [Highlights](#highlights) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [Attribution](#attribution) +- [Code of Conduct](#code-of-conduct) +- [Security](#security) +- [License](#license) +- [Copyright](#copyright) -OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you. +# OpenSearch SQL -## SQL Related Projects +OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL or Piped Processing Language (PPL) query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you. The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for OpenSearch. @@ -21,197 +30,39 @@ The following projects have been merged into this repository as separate folders * [Query Workbench](https://github.com/opensearch-project/sql/tree/main/workbench) -## Documentation - -Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. Looking to contribute? Read the instructions on [Developer Guide](./DEVELOPER_GUIDE.rst) and then submit a patch! - -## SQL Engine V2 - -Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported the new released Piped Processing Language. However, it was experimental and disabled by default for SQL query processing. With most important features and full testing complete, now we're ready to promote it as our default SQL query engine. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md). - - -## Setup - -Install as plugin: build plugin from source code by following the instruction in Build section and install it to your OpenSearch. - -After doing this, you need to restart the OpenSearch server. Otherwise you may get errors like `Invalid index name [sql], must not start with '']; ","status":400}`. - - -## Build - -The package uses the [Gradle](https://docs.gradle.org/4.10.2/userguide/userguide.html) build system. - -1. Checkout this package from version control. -2. To build from command line set `JAVA_HOME` to point to a JDK >=14 -3. Run `./gradlew build` - - -## Basic Usage - -To use the feature, send requests to the `_plugins/_sql` URI. You can use a request parameter or the request body (recommended). Note that for backward compatibility, old `_opendistro/_sql` endpoint is still available, though any future API will be only accessible by new OpenSearch endpoint. - -* Simple query - -``` -POST https://:/_plugins/_sql -{ - "query": "SELECT * FROM my-index LIMIT 50" -} -``` - -* Explain SQL to OpenSearch query DSL -``` -POST _plugins/_sql/_explain -{ - "query": "SELECT * FROM my-index LIMIT 50" -} -``` - -* For a sample curl command with the OpenSearch Security plugin, try: -``` -curl -XPOST https://localhost:9200/_plugins/_sql -u admin:admin -k -d '{"query": "SELECT * FROM my-index LIMIT 10"}' -H 'Content-Type: application/json' -``` - - -## SQL Usage - -* Query - - SELECT * FROM bank WHERE age >30 AND gender = 'm' +## Highlights -* Aggregation +Besides basic filtering and aggregation, OpenSearch SQL also supports complex queries, such as querying semi-structured data, JOINs, set operations, sub-queries etc. Beyond the standard functions, OpenSearch functions are provided for better analytics and visualization. Please check our [documentation](#documentation) for more details. - SELECT COUNT(*),SUM(age),MIN(age) as m, MAX(age),AVG(age) - FROM bank - GROUP BY gender - HAVING m >= 20 - ORDER BY SUM(age), m DESC +Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md). -* Join - SELECT b1.firstname, b1.lastname, b2.age - FROM bank b1 - LEFT JOIN bank b2 - ON b1.age = b2.age AND b1.state = b2.state - -* Show - - SHOW TABLES LIKE ban% - DESCRIBE TABLES LIKE bank - -* Delete - - DELETE FROM bank WHERE age >30 AND gender = 'm' - - -## Beyond SQL - -* Search - - SELECT address FROM bank WHERE address = matchQuery('880 Holmes Lane') ORDER BY _score DESC LIMIT 3 - -* Nested Field - - + - - SELECT address FROM bank b, b.nestedField e WHERE b.state = 'WA' and e.name = 'test' - - + - SELECT address, nested(nestedField.name) - FROM bank - WHERE nested(nestedField, nestedField.state = 'WA' AND nestedField.name = 'test') - OR nested(nestedField.state) = 'CA' - -* Aggregations - - + range age group 20-25,25-30,30-35,35-40 - - SELECT COUNT(age) FROM bank GROUP BY range(age, 20,25,30,35,40) - - + range date group by day - - SELECT online FROM online GROUP BY date_histogram(field='insert_time','interval'='1d') - - + range date group by your config - - SELECT online FROM online GROUP BY date_range(field='insert_time','format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now') - -* OpenSearch Geographic - - SELECT * FROM locations WHERE GEO_BOUNDING_BOX(fieldname,100.0,1.0,101,0.0) - -* Select type or pattern - - SELECT * FROM indexName/type - SELECT * FROM index* - - -## SQL Features +## Documentation -* SQL Select -* SQL Delete -* SQL Where -* SQL Order By -* SQL Group By -* SQL Having -* SQL Inner Join -* SQL Left Join -* SQL Show -* SQL Describe -* SQL AND & OR -* SQL Like -* SQL COUNT distinct -* SQL In -* SQL Between -* SQL Aliases -* SQL Not Null -* SQL(OpenSearch) Date -* SQL avg() -* SQL count() -* SQL max() -* SQL min() -* SQL sum() -* SQL Nulls -* SQL isnull() -* SQL floor -* SQL trim -* SQL log -* SQL log10 -* SQL substring -* SQL round -* SQL sqrt -* SQL concat_ws -* SQL union and minus +Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. -## JDBC Support -Please check out JDBC driver repository for more details. +## Contributing -## Beyond SQL features +See [developer guide](DEVELOPER_GUIDE.rst) and [how to contribute to this project](CONTRIBUTING.md). -* OpenSearch TopHits -* OpenSearch MISSING -* OpenSearch STATS -* OpenSearch GEO_INTERSECTS -* OpenSearch GEO_BOUNDING_BOX -* OpenSearch GEO_DISTANCE -* OpenSearch GEOHASH_GRID aggregation ## Attribution This project is based on the Apache 2.0-licensed [elasticsearch-sql](https://github.com/NLPchina/elasticsearch-sql) project. Thank you [eliranmoyal](https://github.com/eliranmoyal), [shi-yuan](https://github.com/shi-yuan), [ansjsun](https://github.com/ansjsun) and everyone else who contributed great code to that project. Read this for more details [Attributions](./docs/attributions.md). + ## Code of Conduct This project has adopted an [Open Source Code of Conduct](./CODE_OF_CONDUCT.md). -## Security issue notifications +## Security If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. -## Licensing +## License See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/build.gradle b/build.gradle index 1b69cf810b..85523b8859 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ buildscript { ext { - opensearch_version = "1.0.0-rc1" + opensearch_version = "1.0.0" } repositories { @@ -60,7 +60,7 @@ ext { } allprojects { - version = "${opensearchVersion}.0-rc1" + version = "${opensearchVersion}.0" plugins.withId('java') { sourceCompatibility = targetCompatibility = "1.8" diff --git a/integtest.sh b/integtest.sh index 8be8831a60..f5822fb49a 100755 --- a/integtest.sh +++ b/integtest.sh @@ -16,11 +16,12 @@ function usage() { echo -e "-p BIND_PORT\t, defaults to 9200 or 5601 depends on OpenSearch or Dashboards, can be changed to any port for the cluster location." echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." + echo -e "-d DIRECTORY\tSome Repo has more than 1 directory / component, use this to give that directory for separate tests." echo -e "-h\tPrint this message." echo "--------------------------------------------------------------------------" } -while getopts ":hb:p:s:c:" arg; do +while getopts ":hb:p:s:c:d:" arg; do case $arg in h) usage @@ -38,6 +39,9 @@ while getopts ":hb:p:s:c:" arg; do c) CREDENTIAL=$OPTARG ;; + d) + DIRECTORY=$OPTARG + ;; :) echo "-${OPTARG} requires an argument" usage @@ -73,5 +77,23 @@ then PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` fi -./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain +if [ -z "$DIRECTORY" ] +then + DIRECTORY="root" +fi + +if [ "$DIRECTORY" = "workbench" ] +then + mv -v workbench ../ + cd ../ + rm -rf sql + cd workbench + yarn osd bootstrap + curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/accounts.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/accounts/_bulk?pretty' --data-binary @- > /dev/null 2>&1 + curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/employee_nested.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/employee_nested/_bulk?pretty' --data-binary @- > /dev/null 2>&1 + npx cypress run +else + ./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain + +fi diff --git a/release-notes/opensearch-sql.release-notes-1.0.0.0.md b/release-notes/opensearch-sql.release-notes-1.0.0.0.md new file mode 100644 index 0000000000..9f4ba69278 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-1.0.0.0.md @@ -0,0 +1,54 @@ +## 2021-07-12 Version 1.0.0.0 + +Compatible with OpenSearch and OpenSearch Dashboards Version 1.0.0 + +### Enhancements + +* Support querying a data stream ([#56](https://github.com/opensearch-project/sql/pull/56)) + +### Bug Fixes + +* Bug Fix: Enable legacy settings in new setting action ([#97](https://github.com/opensearch-project/sql/pull/97)) +* Fix NPE for SHOW statement without filter ([#150](https://github.com/opensearch-project/sql/pull/150)) + +### OpenSearch Migration + +* Remove debug logging in ODBC driver ([#27](https://github.com/opensearch-project/sql/pull/27)) +* Update workbench nav category to opensearch ([#28](https://github.com/opensearch-project/sql/pull/28)) +* fix opendistro related renaming for sql-cli ([#29](https://github.com/opensearch-project/sql/pull/29)) +* Fix issue of workbench not outputting errors ([#32](https://github.com/opensearch-project/sql/pull/32)) +* Update issue template with multiple labels ([#34](https://github.com/opensearch-project/sql/pull/34)) +* SQL/PPL and JDBC package renaming ([#54](https://github.com/opensearch-project/sql/pull/54)) +* Upgrade dependencies to address high severity CVE-2021-20270 ([#61](https://github.com/opensearch-project/sql/pull/61)) +* ODBC folder, file and code renaming ([#62](https://github.com/opensearch-project/sql/pull/62)) +* Update workbench documentation links, complete renaming ([#67](https://github.com/opensearch-project/sql/pull/67)) +* Update sqli-cli documentation links to OpenSearch ([#72](https://github.com/opensearch-project/sql/pull/72)) +* Remove opensearch.sql.engine.new.enabled setting ([#70](https://github.com/opensearch-project/sql/pull/70)) +* SQL/PPL API endpoint backward compatibility ([#66](https://github.com/opensearch-project/sql/pull/66)) +* Remove opensearch.sql.query.analysis.* related settings ([#76](https://github.com/opensearch-project/sql/pull/76)) +* Remove opensearch.sql.query.response.format setting ([#77](https://github.com/opensearch-project/sql/pull/77)) +* Migrate #1097: Adding support to NOT REGEXP_QUERY ([#79](https://github.com/opensearch-project/sql/pull/79)) +* Migrate #1083: Support long literals in SQL/PPL ([#80](https://github.com/opensearch-project/sql/pull/80)) +* Change strategy to test connectivity between ODBC driver and SQL plugin ([#69](https://github.com/opensearch-project/sql/pull/69)) +* Remove cursor enabling and fetch size setting ([#75](https://github.com/opensearch-project/sql/pull/75)) +* Disable DELETE clause by defaut and add opensearch.sql.delete.enabled setting ([#81](https://github.com/opensearch-project/sql/pull/81)) +* Support Plugin Settings Backwards Compatibility ([#82](https://github.com/opensearch-project/sql/pull/82)) +* Updated icon and background images in ODBC installers ([#84](https://github.com/opensearch-project/sql/pull/84)) +* Build SQL/PPL against OpenSearch rc1 and rename artifacts ([#83](https://github.com/opensearch-project/sql/pull/83)) +* Support text functions ASCII, LEFT, LOCATE, REPLACE in new engine ([#88](https://github.com/opensearch-project/sql/pull/88)) +* Update PowerBI custom connector .mez file for ODBC driver ([#90](https://github.com/opensearch-project/sql/pull/90)) +* Rename remaining beta1 references in sql-cli/workbench ([#91](https://github.com/opensearch-project/sql/pull/91)) +* Build SQL/PPL against OpenSearch 1.0 branch ([#94](https://github.com/opensearch-project/sql/pull/94)) +* Bump OpenSearch Dashboards version to 1.0 in Workbench ([#98](https://github.com/opensearch-project/sql/pull/98)) +* Add Integtest.sh for OpenSearch integtest setups ([#128](https://github.com/opensearch-project/sql/pull/128)) +* Merge develop into main ([#142](https://github.com/opensearch-project/sql/pull/142)) +* Build against OpenSearch 1.0.0 and bump artifact version to 1.0.0.0 ([#146](https://github.com/opensearch-project/sql/pull/146)) + +### Documentation + +* Migrate SQL/PPL, JDBC, ODBC docs to OpenSearch ([#68](https://github.com/opensearch-project/sql/pull/68)) +* Level up README markdown ([#148](https://github.com/opensearch-project/sql/pull/148)) + +### Infrastructure + +* Bump glob-parent from 5.1.1 to 5.1.2 in /workbench ([#125](https://github.com/opensearch-project/sql/pull/125)) diff --git a/sql-cli/src/opensearch_sql_cli/__init__.py b/sql-cli/src/opensearch_sql_cli/__init__.py index 2ffe495725..e9f870deba 100644 --- a/sql-cli/src/opensearch_sql_cli/__init__.py +++ b/sql-cli/src/opensearch_sql_cli/__init__.py @@ -22,4 +22,4 @@ express or implied. See the License for the specific language governing permissions and limitations under the License. """ -__version__ = "1.0.0.0-rc1" +__version__ = "1.0.0.0" diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 955a75e366..548b34a9be 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -43,7 +43,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '1.0.0.0-rc1' +version '1.0.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 18c75b94ff..fe5526621f 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -72,7 +72,7 @@ adminStatement ; showStatement - : SHOW TABLES tableFilter? + : SHOW TABLES tableFilter ; describeStatement diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 53de19a0fd..cf1ca36f02 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -160,4 +160,9 @@ public void canParseOrderByClause() { "SELECT name, age FROM test ORDER BY name ASC NULLS FIRST, age DESC NULLS LAST")); } + @Test + public void canNotParseShowStatementWithoutFilterClause() { + assertThrows(SyntaxCheckException.class, () -> parser.parse("SHOW TABLES")); + } + } diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index a873b67a9f..0ee32f9b98 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "1.0.0.0-rc1", - "opensearchDashboardsVersion": "1.0.0-rc1", + "version": "1.0.0.0", + "opensearchDashboardsVersion": "1.0.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 54d009704e..3e78f6a26b 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,13 +1,13 @@ { "name": "opensearch-query-workbench", - "version": "1.0.0.0-rc1", + "version": "1.0.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", "homepage": "https://github.com/opensearch-project/sql/tree/main/workbench", "opensearchDashboards": { - "version": "1.0.0-rc1", - "templateVersion": "1.0.0-rc1" + "version": "1.0.0", + "templateVersion": "1.0.0" }, "repository": { "type": "git", @@ -47,7 +47,7 @@ "tslint-plugin-prettier": "^2.0.1" }, "engines": { - "node": "10.23.1", + "node": "10.24.1", "yarn": "^1.21.1" }, "resolutions": {