From e175c623c23552576e580715f3ace3fdbb24ff72 Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Thu, 1 Feb 2024 17:10:47 -0800 Subject: [PATCH] add integration tests (#31) --- .github/workflows/tests.yml | 64 ++++++++++- Makefile | 14 ++- integration-tests/batch_quickstart_test.go | 123 +++++++++++++++++++++ scripts/start-pinot-quickstart.sh | 74 +++++++++++++ 4 files changed, 269 insertions(+), 6 deletions(-) create mode 100644 integration-tests/batch_quickstart_test.go create mode 100755 scripts/start-pinot-quickstart.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25cd087..937f51b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ on: jobs: golangci: runs-on: ubuntu-latest - name: lint + name: Linter steps: - uses: actions/checkout@v3 @@ -27,9 +27,9 @@ jobs: - name: Linter uses: golangci/golangci-lint-action@v3 - test: + unit-tests: runs-on: ubuntu-latest - name: Test + name: Unit Tests steps: - uses: actions/checkout@v3 @@ -42,7 +42,7 @@ jobs: run: make setup - name: Build - run: make + run: make build - name: Test run: make test @@ -54,7 +54,61 @@ jobs: outfile: coverage.lcov - name: Publish to coveralls.io - uses: coverallsapp/github-action@v1 + uses: coverallsapp/github-action@v2 with: github-token: ${{ github.token }} path-to-lcov: coverage.lcov + flag-name: unit + allow-empty: true + + integration-tests: + runs-on: ubuntu-latest + name: Integration Tests + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Before Install + run: make setup + + - name: Build + run: make build + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: "adopt" + + - name: Set up Pinot Cluster + run: make run-pinot-dist + env: + PINOT_VERSION: 1.0.0 + PINOT_HOME: /tmp/pinot + CONTROLLER_PORT_FORWARD: 9000 + BROKER_PORT_FORWARD: 8000 + + - name: Integration Test + run: make integration-test + env: + ZOOKEEPER_PORT: 2123 + CONTROLLER_PORT: 9000 + BROKER_PORT: 8000 + + - name: Convert coverage to lcov + uses: jandelgado/gcov2lcov-action@v1.0.9 + with: + infile: profile.cov + outfile: coverage.lcov + + - name: Publish to coveralls.io + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ github.token }} + path-to-lcov: coverage.lcov + flag-name: integration + allow-empty: true diff --git a/Makefile b/Makefile index 07e8d84..578225b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ # make file to hold the logic of build and test setup -PACKAGES := $(shell go list ./... | grep -v examples) +PACKAGES := $(shell go list ./... | grep -v examples| grep -v integration-tests) +INTEGRATION_TESTS_PACKAGES := $(shell go list ./... | grep integration-tests) .DEFAULT_GOAL := test @@ -30,3 +31,14 @@ build: test: build go test -timeout 500s -v -race -covermode atomic -coverprofile=profile.cov $(PACKAGES) +.PHONY: run-pinot-dist +run-pinot-dist: + ./scripts/start-pinot-quickstart.sh + +.PHONY: run-pinot-docker +run-pinot-docker: + docker run --name pinot-quickstart -p 2123:2123 -p 9000:9000 -p 8000:8000 apachepinot/pinot:latest QuickStart -type MULTI_STAGE + +.PHONY: integration-test +integration-test: build + go test -timeout 500s -v -race -covermode atomic -coverprofile=profile.cov $(INTEGRATION_TESTS_PACKAGES) diff --git a/integration-tests/batch_quickstart_test.go b/integration-tests/batch_quickstart_test.go new file mode 100644 index 0000000..510876c --- /dev/null +++ b/integration-tests/batch_quickstart_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "net" + "net/http" + "os" + "testing" + "time" + + pinot "github.com/startreedata/pinot-client-go/pinot" + "github.com/stretchr/testify/assert" + + log "github.com/sirupsen/logrus" +) + +// getEnv retrieves the value of the environment variable named by the key. +// It returns the value, which will be the default value if the variable is not present. +func getEnv(key, defaultValue string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + return defaultValue +} + +var ( + zookeeperPort = getEnv("ZOOKEEPER_PORT", "2123") + controllerPort = getEnv("CONTROLLER_PORT", "9000") + brokerPort = getEnv("BROKER_PORT", "8000") +) + +func getPinotClientFromZookeeper() *pinot.Connection { + pinotClient, err := pinot.NewFromZookeeper([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster") + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +func getPinotClientFromController() *pinot.Connection { + pinotClient, err := pinot.NewFromController("localhost:" + controllerPort) + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +func getPinotClientFromBroker() *pinot.Connection { + pinotClient, err := pinot.NewFromBrokerList([]string{"localhost:" + brokerPort}) + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +func getCustomHttpClient() *http.Client { + httpClient := &http.Client{ + Timeout: 15 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, // Max idle connections in total + MaxIdleConnsPerHost: 10, // Max idle connections per host + IdleConnTimeout: 90 * time.Second, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + // You may add other settings like TLS configuration, Proxy, etc. + }, + } + return httpClient +} + +func getPinotClientFromZookeeperAndCustomHttpClient() *pinot.Connection { + pinotClient, err := pinot.NewFromZookeeperAndClient([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster", getCustomHttpClient()) + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +func getPinotClientFromControllerAndCustomHttpClient() *pinot.Connection { + pinotClient, err := pinot.NewFromControllerAndClient("localhost:"+controllerPort, getCustomHttpClient()) + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +func getPinotClientFromBrokerAndCustomHttpClient() *pinot.Connection { + pinotClient, err := pinot.NewFromBrokerListAndClient([]string{"localhost:" + brokerPort}, getCustomHttpClient()) + if err != nil { + log.Fatalln(err) + } + return pinotClient +} + +// TestSendingQueriesToPinot tests sending queries to Pinot using different Pinot clients. +// This test requires a Pinot cluster running locally with binary not docker. +// You can change the ports by setting the environment variables ZOOKEEPER_PORT, CONTROLLER_PORT, and BROKER_PORT. +func TestSendingQueriesToPinot(t *testing.T) { + pinotClients := []*pinot.Connection{ + getPinotClientFromZookeeper(), + getPinotClientFromController(), + getPinotClientFromBroker(), + getPinotClientFromZookeeperAndCustomHttpClient(), + getPinotClientFromControllerAndCustomHttpClient(), + getPinotClientFromBrokerAndCustomHttpClient(), + } + + table := "baseballStats" + pinotQueries := []string{ + "select count(*) as cnt from baseballStats limit 1", + } + + log.Printf("Querying SQL") + for _, query := range pinotQueries { + for i := 0; i < 100; i++ { + log.Printf("Trying to query Pinot: %v\n", query) + brokerResp, err := pinotClients[i%len(pinotClients)].ExecuteSQL(table, query) + assert.Nil(t, err) + assert.Equal(t, int64(97889), brokerResp.ResultTable.GetLong(0, 0)) + } + } +} diff --git a/scripts/start-pinot-quickstart.sh b/scripts/start-pinot-quickstart.sh new file mode 100755 index 0000000..60b7998 --- /dev/null +++ b/scripts/start-pinot-quickstart.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Set the Pinot version +if [ -z "${PINOT_VERSION}" ]; then + echo "PINOT_VERSION is not set. Using default version 1.0.0" + PINOT_VERSION="1.0.0" +fi + +# Set the download URL +DOWNLOAD_URL="https://archive.apache.org/dist/pinot/apache-pinot-${PINOT_VERSION}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" + +# Set the destination directory +if [ -z "${PINOT_HOME}" ]; then + echo "PINOT_HOME is not set. Using default directory /tmp/pinot" + PINOT_HOME="/tmp/pinot" +fi + +# Create the destination directory +mkdir -p "${PINOT_HOME}" + +# Download the Pinot package +curl -L "${DOWNLOAD_URL}" -o "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" + +# Extract the downloaded package +tar -xzf "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" -C "${PINOT_HOME}" + +# Remove the downloaded package +rm "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" + +# Start the Pinot cluster +${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin/bin/pinot-admin.sh QuickStart -type MULTI_STAGE & +PID=$! + +# Print the JVM settings +jps -lvm + +### --------------------------------------------------------------------------- +### Ensure Pinot cluster started correctly. +### --------------------------------------------------------------------------- + +echo "Ensure Pinot cluster started correctly" + +# Wait at most 5 minutes to reach the desired state +for i in $(seq 1 150) +do + SUCCEED_TABLE=0 + for table in "airlineStats" "baseballStats" "dimBaseballTeams" "githubComplexTypeEvents" "githubEvents" "starbucksStores"; + do + QUERY="select count(*) from ${table} limit 1" + QUERY_REQUEST='curl -s -X POST -H '"'"'Accept: application/json'"'"' -d '"'"'{"sql": "'${QUERY}'"}'"'"' http://localhost:'${BROKER_PORT_FORWARD}'/query/sql' + echo ${QUERY_REQUEST} + QUERY_RES=`eval ${QUERY_REQUEST}` + echo ${QUERY_RES} + + if [ $? -eq 0 ]; then + COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'` + if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -gt 0 ]; then + SUCCEED_TABLE=$((SUCCEED_TABLE+1)) + fi + fi + echo "QUERY: ${QUERY}, QUERY_RES: ${QUERY_RES}" + done + echo "SUCCEED_TABLE: ${SUCCEED_TABLE}" + if [ "${SUCCEED_TABLE}" -eq 6 ]; then + break + fi + sleep 2 +done + +if [ "${SUCCEED_TABLE}" -lt 6 ]; then + echo 'Quickstart failed: Cannot confirmed count-star result from quickstart table in 5 minutes' + exit 1 +fi +echo "Pinot cluster started correctly"