add integration tests (#31)

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   golangci:
     runs-on: ubuntu-latest
-    name: lint
+    name: Linter
     steps:
       - uses: actions/checkout@v3
 
@@ -27,9 +27,9 @@ jobs:
       - name: Linter
         uses: golangci/golangci-lint-action@v3
 
-  test:
+  unit-tests:
     runs-on: ubuntu-latest
-    name: Test
+    name: Unit Tests
     steps:
       - uses: actions/checkout@v3
 
@@ -42,7 +42,7 @@ jobs:
         run: make setup
 
       - name: Build
-        run: make
+        run: make build
 
       - name: Test
         run: make test
@@ -54,7 +54,61 @@ jobs:
           outfile: coverage.lcov
 
       - name: Publish to coveralls.io
-        uses: coverallsapp/github-action@v1
+        uses: coverallsapp/github-action@v2
         with:
           github-token: ${{ github.token }}
           path-to-lcov: coverage.lcov
+          flag-name: unit
+          allow-empty: true
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    name: Integration Tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Before Install
+        run: make setup
+
+      - name: Build
+        run: make build
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: "adopt"
+
+      - name: Set up Pinot Cluster
+        run: make run-pinot-dist
+        env:
+          PINOT_VERSION: 1.0.0
+          PINOT_HOME: /tmp/pinot
+          CONTROLLER_PORT_FORWARD: 9000
+          BROKER_PORT_FORWARD: 8000
+
+      - name: Integration Test
+        run: make integration-test
+        env:
+          ZOOKEEPER_PORT: 2123
+          CONTROLLER_PORT: 9000
+          BROKER_PORT: 8000
+
+      - name: Convert coverage to lcov
+        uses: jandelgado/[email protected]
+        with:
+          infile: profile.cov
+          outfile: coverage.lcov
+
+      - name: Publish to coveralls.io
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ github.token }}
+          path-to-lcov: coverage.lcov
+          flag-name: integration
+          allow-empty: true

Makefile

@@ -1,5 +1,6 @@
 # make file to hold the logic of build and test setup
-PACKAGES := $(shell go list ./... | grep -v examples)
+PACKAGES := $(shell go list ./... | grep -v examples| grep -v integration-tests)
+INTEGRATION_TESTS_PACKAGES := $(shell go list ./... | grep integration-tests)
 
 .DEFAULT_GOAL := test
 
@@ -30,3 +31,14 @@ build:
 test: build
 	go test -timeout 500s -v -race -covermode atomic -coverprofile=profile.cov $(PACKAGES)
 
+.PHONY: run-pinot-dist
+run-pinot-dist:
+	./scripts/start-pinot-quickstart.sh
+
+.PHONY: run-pinot-docker
+run-pinot-docker:
+	docker run --name pinot-quickstart -p 2123:2123 -p 9000:9000 -p 8000:8000 apachepinot/pinot:latest QuickStart -type MULTI_STAGE
+
+.PHONY: integration-test
+integration-test: build
+	go test -timeout 500s -v -race -covermode atomic -coverprofile=profile.cov $(INTEGRATION_TESTS_PACKAGES)

integration-tests/batch_quickstart_test.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"net"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	pinot "github.com/startreedata/pinot-client-go/pinot"
+	"github.com/stretchr/testify/assert"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// getEnv retrieves the value of the environment variable named by the key.
+// It returns the value, which will be the default value if the variable is not present.
+func getEnv(key, defaultValue string) string {
+	if value, exists := os.LookupEnv(key); exists {
+		return value
+	}
+	return defaultValue
+}
+
+var (
+	zookeeperPort  = getEnv("ZOOKEEPER_PORT", "2123")
+	controllerPort = getEnv("CONTROLLER_PORT", "9000")
+	brokerPort     = getEnv("BROKER_PORT", "8000")
+)
+
+func getPinotClientFromZookeeper() *pinot.Connection {
+	pinotClient, err := pinot.NewFromZookeeper([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster")
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+func getPinotClientFromController() *pinot.Connection {
+	pinotClient, err := pinot.NewFromController("localhost:" + controllerPort)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+func getPinotClientFromBroker() *pinot.Connection {
+	pinotClient, err := pinot.NewFromBrokerList([]string{"localhost:" + brokerPort})
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+func getCustomHttpClient() *http.Client {
+	httpClient := &http.Client{
+		Timeout: 15 * time.Second,
+		Transport: &http.Transport{
+			MaxIdleConns:        100, // Max idle connections in total
+			MaxIdleConnsPerHost: 10,  // Max idle connections per host
+			IdleConnTimeout:     90 * time.Second,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			// You may add other settings like TLS configuration, Proxy, etc.
+		},
+	}
+	return httpClient
+}
+
+func getPinotClientFromZookeeperAndCustomHttpClient() *pinot.Connection {
+	pinotClient, err := pinot.NewFromZookeeperAndClient([]string{"localhost:" + zookeeperPort}, "", "QuickStartCluster", getCustomHttpClient())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+func getPinotClientFromControllerAndCustomHttpClient() *pinot.Connection {
+	pinotClient, err := pinot.NewFromControllerAndClient("localhost:"+controllerPort, getCustomHttpClient())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+func getPinotClientFromBrokerAndCustomHttpClient() *pinot.Connection {
+	pinotClient, err := pinot.NewFromBrokerListAndClient([]string{"localhost:" + brokerPort}, getCustomHttpClient())
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return pinotClient
+}
+
+// TestSendingQueriesToPinot tests sending queries to Pinot using different Pinot clients.
+// This test requires a Pinot cluster running locally with binary not docker.
+// You can change the ports by setting the environment variables ZOOKEEPER_PORT, CONTROLLER_PORT, and BROKER_PORT.
+func TestSendingQueriesToPinot(t *testing.T) {
+	pinotClients := []*pinot.Connection{
+		getPinotClientFromZookeeper(),
+		getPinotClientFromController(),
+		getPinotClientFromBroker(),
+		getPinotClientFromZookeeperAndCustomHttpClient(),
+		getPinotClientFromControllerAndCustomHttpClient(),
+		getPinotClientFromBrokerAndCustomHttpClient(),
+	}
+
+	table := "baseballStats"
+	pinotQueries := []string{
+		"select count(*) as cnt from baseballStats limit 1",
+	}
+
+	log.Printf("Querying SQL")
+	for _, query := range pinotQueries {
+		for i := 0; i < 100; i++ {
+			log.Printf("Trying to query Pinot: %v\n", query)
+			brokerResp, err := pinotClients[i%len(pinotClients)].ExecuteSQL(table, query)
+			assert.Nil(t, err)
+			assert.Equal(t, int64(97889), brokerResp.ResultTable.GetLong(0, 0))
+		}
+	}
+}

scripts/start-pinot-quickstart.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Set the Pinot version
+if [ -z "${PINOT_VERSION}" ]; then
+  echo "PINOT_VERSION is not set. Using default version 1.0.0"
+  PINOT_VERSION="1.0.0"
+fi
+
+# Set the download URL
+DOWNLOAD_URL="https://archive.apache.org/dist/pinot/apache-pinot-${PINOT_VERSION}/apache-pinot-${PINOT_VERSION}-bin.tar.gz"
+
+# Set the destination directory
+if [ -z "${PINOT_HOME}" ]; then
+  echo "PINOT_HOME is not set. Using default directory /tmp/pinot"
+  PINOT_HOME="/tmp/pinot"
+fi
+
+# Create the destination directory
+mkdir -p "${PINOT_HOME}"
+
+# Download the Pinot package
+curl -L "${DOWNLOAD_URL}" -o "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz"
+
+# Extract the downloaded package
+tar -xzf "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz" -C "${PINOT_HOME}"
+
+# Remove the downloaded package
+rm "${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin.tar.gz"
+
+# Start the Pinot cluster
+${PINOT_HOME}/apache-pinot-${PINOT_VERSION}-bin/bin/pinot-admin.sh QuickStart -type MULTI_STAGE &
+PID=$!
+
+# Print the JVM settings
+jps -lvm
+
+### ---------------------------------------------------------------------------
+### Ensure Pinot cluster started correctly.
+### ---------------------------------------------------------------------------
+
+echo "Ensure Pinot cluster started correctly"
+
+# Wait at most 5 minutes to reach the desired state
+for i in $(seq 1 150)
+do
+  SUCCEED_TABLE=0
+  for table in "airlineStats" "baseballStats" "dimBaseballTeams" "githubComplexTypeEvents" "githubEvents" "starbucksStores";
+  do
+    QUERY="select count(*) from ${table} limit 1"
+    QUERY_REQUEST='curl -s -X POST -H '"'"'Accept: application/json'"'"' -d '"'"'{"sql": "'${QUERY}'"}'"'"' http://localhost:'${BROKER_PORT_FORWARD}'/query/sql'
+    echo ${QUERY_REQUEST}
+    QUERY_RES=`eval ${QUERY_REQUEST}`
+    echo ${QUERY_RES}
+
+    if [ $? -eq 0 ]; then
+      COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
+      if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -gt 0 ]; then
+        SUCCEED_TABLE=$((SUCCEED_TABLE+1))
+      fi
+    fi
+    echo "QUERY: ${QUERY}, QUERY_RES: ${QUERY_RES}"
+  done
+  echo "SUCCEED_TABLE: ${SUCCEED_TABLE}"
+  if [ "${SUCCEED_TABLE}" -eq 6 ]; then
+    break
+  fi
+  sleep 2
+done
+
+if [ "${SUCCEED_TABLE}" -lt 6 ]; then
+  echo 'Quickstart failed: Cannot confirmed count-star result from quickstart table in 5 minutes'
+  exit 1
+fi
+echo "Pinot cluster started correctly"