diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml new file mode 100644 index 0000000..b3631d8 --- /dev/null +++ b/.github/workflows/build-and-push.yml @@ -0,0 +1,184 @@ +name: Build and Push Grafana with Quickwit Plugin + +on: + push: + branches: + - disable-field-caps-all-fields + - main + pull_request: + types: [closed] + workflow_dispatch: + inputs: + force_publish: + description: 'Force publish image' + required: false + type: boolean + default: false + +env: + GRAFANA_VERSION: 12.4.0 + AWS_REGION_MGT: us-east-1 + DOCKER_REGISTRY_MGT: 337909757619.dkr.ecr.us-east-1.amazonaws.com + ECR_REPOSITORY: grafana-quickwit + +jobs: + build-and-publish: + name: Build and Publish Grafana Quickwit Image + runs-on: gha-runner-ecr-publish + outputs: + githash: ${{ steps.metadata.outputs.githash }} + image_tag: ${{ steps.metadata.outputs.image_tag }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate metadata + id: metadata + run: | + SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + + # Use git tag if available, otherwise use short SHA + if git describe --exact-match --tags HEAD 2>/dev/null; then + GIT_TAG=$(git describe --exact-match --tags HEAD) + # Strip 'v' prefix if present + VERSION=${GIT_TAG#v} + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-${VERSION}" + else + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" + fi + + echo "githash=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + + echo "Image will be tagged as: ${IMAGE_TAG}" + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Install dependencies + run: npm ci + + - name: Build frontend + run: npm run build + + - name: Build backend binaries + run: | + # Try using mage if available, otherwise use go build directly + if command -v mage &> /dev/null; then + mage -v buildAll + else + echo "Mage not available, building with go directly" + cd pkg + GOOS=linux GOARCH=amd64 go build -o ../dist/gpx_quickwit_linux_amd64 . + GOOS=linux GOARCH=arm64 go build -o ../dist/gpx_quickwit_linux_arm64 . + cd .. + fi + + - name: Remove signature files for patched plugin + run: | + cd dist + rm -f MANIFEST.txt + if [ -f plugin.json ]; then + # Remove signature field from plugin.json + jq 'del(.signature)' plugin.json > plugin.json.tmp && mv plugin.json.tmp plugin.json + fi + + - name: Package plugin + run: | + cd dist + zip -r quickwit-quickwit-datasource-patched.zip . -x "*.zip" + ls -lh quickwit-quickwit-datasource-patched.zip + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create Dockerfile + run: | + cat > Dockerfile <<'EOF' + FROM grafana/grafana:${{ env.GRAFANA_VERSION }} + + USER root + + # Install patched Quickwit plugin + COPY dist/quickwit-quickwit-datasource-patched.zip /tmp/plugin.zip + RUN set -ex && \ + mkdir -p /var/lib/grafana/plugins && \ + cd /var/lib/grafana/plugins && \ + unzip -q /tmp/plugin.zip -d quickwit-quickwit-datasource && \ + rm /tmp/plugin.zip && \ + chown -R 472:0 /var/lib/grafana/plugins + + USER grafana + + ENV GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=quickwit-quickwit-datasource + + LABEL org.opencontainers.image.source="https://github.com/Iterable/quickwit-datasource" + LABEL org.opencontainers.image.description="Grafana with patched Quickwit datasource plugin (field_caps disabled)" + LABEL grafana.version="${{ env.GRAFANA_VERSION }}" + LABEL quickwit.plugin.version="0.6.0-patched" + LABEL githash="${{ steps.metadata.outputs.githash }}" + + EXPOSE 3000 + + HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1 + EOF + + - name: Build Docker image + run: | + docker buildx build \ + --platform linux/amd64 \ + --load \ + --tag ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} \ + --tag ${{ env.ECR_REPOSITORY }}:latest \ + -f Dockerfile . + + - name: Publish to ECR + id: publish + if: | + github.event_name == 'workflow_dispatch' && github.event.inputs.force_publish == 'true' || + github.event.action == 'closed' && github.event.pull_request.merged == true || + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/disable-field-caps-all-fields' || + startsWith(github.ref, 'refs/tags/') + run: | + aws ecr get-login-password --region $AWS_REGION_MGT | docker login --username AWS --password-stdin $DOCKER_REGISTRY_MGT + + docker tag ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} \ + $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} + + docker push $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} + + SUMMARY=$'# Published Grafana Quickwit Image to ECR\n' + SUMMARY+=$'## Image\n' + SUMMARY+=$'```\n' + SUMMARY+=$''$DOCKER_REGISTRY_MGT'/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }}\n' + SUMMARY+=$'```\n' + SUMMARY+=$'\n## Usage in Deployments\n' + SUMMARY+=$'**Preprod**: Update gitops to use this tag for testing\n' + SUMMARY+=$'**Prod**: Promote this tag after preprod validation\n' + SUMMARY+=$'\n## Details\n' + SUMMARY+=$'* **Grafana Version**: ${{ env.GRAFANA_VERSION }}\n' + SUMMARY+=$'* **Quickwit Plugin**: 0.6.0-patched (field_caps disabled)\n' + SUMMARY+=$'* **Git Hash**: ${{ steps.metadata.outputs.githash }}\n' + echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY + + - name: Build Summary (No Publish) + if: steps.publish.outcome == 'skipped' + run: | + SUMMARY=$'# Built Grafana Quickwit Image (Not Published)\n' + SUMMARY+=$'## Image Tag\n' + SUMMARY+=$'* ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }}\n' + SUMMARY+=$'\n_Image was built but not published to ECR. Publish occurs on PR merge or manual workflow dispatch._\n' + echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY diff --git a/CI-CD-SETUP.md b/CI-CD-SETUP.md new file mode 100644 index 0000000..4233dbc --- /dev/null +++ b/CI-CD-SETUP.md @@ -0,0 +1,160 @@ +# CI/CD Setup for Grafana Quickwit Image + +This repository includes a GitHub Actions workflow that automatically builds and pushes a Grafana Docker image with the patched Quickwit datasource plugin to ECR. + +## Overview + +The workflow uses the **`gha-runner-ecr-publish`** self-hosted runner which already has AWS credentials configured. **No additional secrets are required.** + +## Workflow Triggers + +The workflow runs and **publishes to ECR** on: +- **Push** to `disable-field-caps-all-fields` or `main` branches +- **PR merge** to these branches +- **Manual** workflow dispatch with `force_publish` option + +The workflow **builds but does not publish** on: +- Pull request events (for testing) +- Other branch pushes + +## Image Details + +### Target ECR Repository +- **Repository**: `grafana-quickwit` +- **Region**: `us-east-1` +- **Registry**: `337909757619.dkr.ecr.us-east-1.amazonaws.com` + +### Image Tags + +Images use **immutable tags only** (no `latest` tag for production safety): + +**For Git Tags** (e.g., `v0.6.0-patched-1`): +``` +12.4.0-quickwit-0.6.0-patched-1 +``` + +**For Untagged Commits**: +``` +12.4.0-quickwit-0.6.0-patched-a1b2c3d +``` + +Where: +- `12.4.0` = Grafana version +- `0.6.0-patched` = Quickwit plugin version (patched) +- `a1b2c3d` = Short git SHA (7 chars) + +### Image Contents +- **Base**: Grafana 12.4.0 +- **Plugin**: Quickwit datasource v0.6.0 (patched to disable field_caps) +- **Platform**: linux/amd64 + +## What the Workflow Does + +1. **Build Plugin** + - Installs Node.js and Go dependencies + - Builds frontend (TypeScript → JavaScript) + - Builds backend (Go binaries for Linux) + - Removes signature files (since plugin is patched) + - Packages as ZIP + +2. **Build Docker Image** + - Creates Dockerfile dynamically + - Copies patched plugin into Grafana base image + - Configures unsigned plugin loading + - Adds metadata labels + +3. **Publish to ECR** (conditional) + - Authenticates to ECR using runner's AWS credentials + - Tags image with git hash and `latest` + - Pushes both tags to ECR + - Generates build summary + +## Running the Workflow + +### Automatic (Recommended) +Just push commits to `disable-field-caps-all-fields` branch: +```bash +git push origin disable-field-caps-all-fields +``` + +The workflow will automatically build and push to ECR. + +### Manual Trigger +1. Go to the **Actions** tab in GitHub +2. Select **Build and Push Grafana with Quickwit Plugin** +3. Click **Run workflow** +4. Select branch: `disable-field-caps-all-fields` +5. Check **force_publish** if you want to publish to ECR +6. Click **Run workflow** + +## Verifying the Build + +After the workflow completes: + +1. **Check GitHub Actions**: The workflow summary will show the published image tags +2. **Check ECR**: + ```bash + aws ecr describe-images \ + --repository-name grafana-quickwit \ + --region us-east-1 \ + --query 'sort_by(imageDetails,& imagePushedAt)[-5:]' \ + --output table + ``` + +## Using the Image + +### Deployment Strategy + +1. **Find the latest tag** from the workflow output or ECR +2. **Deploy to preprod** for testing +3. **Promote to prod** after validation + +```yaml +# Preprod - test new builds +image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:12.4.0-quickwit-0.6.0-patched-a1b2c3d + +# Prod - promote after preprod validation +image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:12.4.0-quickwit-0.6.0-patched-a1b2c3d +``` + +### Creating Release Tags + +To create a versioned release: + +```bash +# Create and push a version tag +git tag -a v0.6.0-patched-1 -m "Release v0.6.0-patched-1" +git push origin v0.6.0-patched-1 + +# This will create image tag: 12.4.0-quickwit-0.6.0-patched-1 +``` + +## Troubleshooting + +### Build Fails on Plugin Build +- Check Node.js and Go versions in the workflow match requirements +- Review build logs for npm or go errors + +### Docker Build Fails +- Verify the Grafana base image version exists +- Check that plugin ZIP was created successfully + +### ECR Push Fails +- Verify the `gha-runner-ecr-publish` runner has ECR write permissions +- Check that the ECR repository `grafana-quickwit` exists +- Verify AWS credentials on the runner are valid + +### Workflow Doesn't Trigger +- Ensure you're pushing to the correct branch +- Check workflow file syntax in `.github/workflows/build-and-push.yml` +- Verify GitHub Actions are enabled for the repository + +## Comparing with Backstage Setup + +This workflow follows the same pattern as `Iterable/backstage`: +- Uses `gha-runner-ecr-publish` runner +- Authenticates with `aws ecr get-login-password` +- Conditionally publishes based on event type +- Generates summary with published tags + +No IAM roles or GitHub secrets are required because the self-hosted runner already has the necessary AWS permissions. diff --git a/src/datasource/base.ts b/src/datasource/base.ts index b0abba7..224c791 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -16,7 +16,7 @@ import { TimeRange, ToggleFilterAction, } from '@grafana/data'; -import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; +import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery } from '@/types'; import { DataSourceWithBackend, getTemplateSrv, @@ -29,7 +29,7 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap, hasWhiteSpace, isSimpleToken } from 'utils'; +import { hasWhiteSpace, isSimpleToken } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; import { normalizeInternalLinkQuery } from '@/queryModel'; @@ -332,65 +332,68 @@ export class BaseQuickwitDataSource } getFields(spec: FieldCapsSpec = {}): Observable { - const range = spec.range || getDefaultTimeRange(); - return from( - this.getResource('_elastic/' + this.index + '/_field_caps', { - start_timestamp: Math.floor(range.from.valueOf() / SECOND), - end_timestamp: Math.ceil(range.to.valueOf() / SECOND), - }) - ).pipe( - map((field_capabilities_response: FieldCapabilitiesResponse) => { - // Cache field → type on the datasource for modifyQuery to consult. - // Quickwit routes phrase queries to the text variant first on multi-indexed - // fields (text+keyword), and text fields don't index positions by default. - // So prefer 'text' when present — it drives safer operator choices downstream. - for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { - const typeKeys = Object.keys(caps); - const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; - if (chosen) { - this.fieldTypes[name] = chosen; - } - } - - const shouldAddField = (field: any) => { - if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { - return false; - } - if (spec.searchable !== undefined && field.searchable !== spec.searchable) { - return false; - } - if ( - spec.type && - spec.type.length !== 0 && - !(spec.type.includes(field.type) || spec.type.includes(fieldTypeMap[field.type])) - ) { - return false; - } - return true; - }; - const fieldCapabilities = Object.entries(field_capabilities_response.fields) - .flatMap(([field_name, field_capabilities]) => { - return Object.values(field_capabilities).map((field_capability) => { - field_capability.field_name = field_name; - return field_capability; - }); - }) - .filter(shouldAddField) - .map((field_capability) => { - return { - text: field_capability.field_name, - type: fieldTypeMap[field_capability.type], - }; - }); - const uniquefieldCapabilities = fieldCapabilities - .filter( - (field_capability, index, self) => - index === self.findIndex((t) => t.text === field_capability.text && t.type === field_capability.type) - ) - .sort((a, b) => a.text.localeCompare(b.text)); - return uniquefieldCapabilities; - }) - ); + // PATCH: Disable field_caps to avoid large responses on indices with many dynamic fields + return of([]); + + // DISABLED: const range = spec.range || getDefaultTimeRange(); + // DISABLED: return from( + // DISABLED: this.getResource('_elastic/' + this.index + '/_field_caps', { + // DISABLED: start_timestamp: Math.floor(range.from.valueOf() / SECOND), + // DISABLED: end_timestamp: Math.ceil(range.to.valueOf() / SECOND), + // DISABLED: }) + // DISABLED: ).pipe( + // DISABLED: map((field_capabilities_response: FieldCapabilitiesResponse) => { + // DISABLED: // Cache field → type on the datasource for modifyQuery to consult. + // DISABLED: // Quickwit routes phrase queries to the text variant first on multi-indexed + // DISABLED: // fields (text+keyword), and text fields don't index positions by default. + // DISABLED: // So prefer 'text' when present — it drives safer operator choices downstream. + // DISABLED: for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { + // DISABLED: const typeKeys = Object.keys(caps); + // DISABLED: const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; + // DISABLED: if (chosen) { + // DISABLED: this.fieldTypes[name] = chosen; + // DISABLED: } + // DISABLED: } + // DISABLED: + // DISABLED: const shouldAddField = (field: any) => { + // DISABLED: if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: if (spec.searchable !== undefined && field.searchable !== spec.searchable) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: if ( + // DISABLED: spec.type && + // DISABLED: spec.type.length !== 0 && + // DISABLED: !(spec.type.includes(field.type) || spec.type.includes(fieldTypeMap[field.type])) + // DISABLED: ) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: return true; + // DISABLED: }; + // DISABLED: const fieldCapabilities = Object.entries(field_capabilities_response.fields) + // DISABLED: .flatMap(([field_name, field_capabilities]) => { + // DISABLED: return Object.values(field_capabilities).map((field_capability) => { + // DISABLED: field_capability.field_name = field_name; + // DISABLED: return field_capability; + // DISABLED: }); + // DISABLED: }) + // DISABLED: .filter(shouldAddField) + // DISABLED: .map((field_capability) => { + // DISABLED: return { + // DISABLED: text: field_capability.field_name, + // DISABLED: type: fieldTypeMap[field_capability.type], + // DISABLED: }; + // DISABLED: }); + // DISABLED: const uniquefieldCapabilities = fieldCapabilities + // DISABLED: .filter( + // DISABLED: (field_capability, index, self) => + // DISABLED: index === self.findIndex((t) => t.text === field_capability.text && t.type === field_capability.type) + // DISABLED: ) + // DISABLED: .sort((a, b) => a.text.localeCompare(b.text)); + // DISABLED: return uniquefieldCapabilities; + // DISABLED: }) + // DISABLED: ); } /**