feat(*): Init project (#1)

* feat(*): Add connector skeleton * feat(*): Import for 1 IP * feat(*): Avoid database request for speedup import * feat(*): Use batch to send bundles to OpenCTI * feat(config): Make config variable consistant * feat(*): Store last run on first loop to avoid multiple works to run * feat(config): Rename config variable for consistency with other connectors * fix(None value): Handle None value returned by CTI * feat(errors): Improve error management * feat(*): Add Ipv6 support * feat(*): Set is_running state to avoid concurrent runs * feat(acl): Handle max observable tlp * feat(*): Add enrichment threshold per import * feat(log): Improve date logs * feat(log): Improve date logs * feat(log): Improve date logs * style(*): Refactor importer * feat(*): Add settings to allow light enrichment * feat(*): Remove option to add external ref and always add it * fix(import): Avoid multiple run even if not first run * test(*): Add tests and ci actions * docs(*): Add docs * docs(*): Add TOC * fix(import): Fix division by zero error * feat(*): Add log for total number of enrichments * feat(*): Add log for current number of enrichments * docs(user guide): Add benchmarks
crowdsecurity · Jun 27, 2024 · 4b8862b · 4b8862b
1 parent d5cf40c
commit 4b8862b
Show file tree

Hide file tree

Showing 44 changed files with 3,350 additions and 1 deletion.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,2 @@
+src/config.yml
+src/__pycache__
diff --git a/.github/workflows/code-format.yml b/.github/workflows/code-format.yml
@@ -0,0 +1,37 @@
+name: Code format
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  code-format-check:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r tests/test-requirements.txt
+
+    - name: Code format check
+      run: |
+        black --check -v ./
diff --git a/.github/workflows/doc-links.yml b/.github/workflows/doc-links.yml
@@ -0,0 +1,39 @@
+name: Documentation links
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  markdown-test:
+    name: Markdown files test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Clone sources
+        uses: actions/checkout@v4
+        with:
+          path: extension
+
+      - name: Launch localhost server
+        run: |
+          sudo npm install --global http-server
+          http-server ./extension &
+
+      - name: Set up Ruby 2.6
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+
+      - name: Check links in Markdown files
+        run: |
+          gem install awesome_bot
+          cd extension
+          awesome_bot --files README.md --allow-dupe --allow-redirect --allow 401 --skip-save-results --base-url http://localhost:8080/
+          awesome_bot docs/*.md --allow-dupe --allow-redirect --allow 401 --white-list http://opencti --skip-save-results --base-url http://localhost:8080/docs/
diff --git a/.github/workflows/keepalive.yml b/.github/workflows/keepalive.yml
@@ -0,0 +1,25 @@
+name: Keep Alive
+on:
+
+  schedule:
+    - cron: '0 3 * * 4'
+
+permissions:
+  actions: write
+
+jobs:
+  keep-alive:
+
+    name: Keep Alive
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Clone project files
+        uses: actions/checkout@v4
+
+      # keepalive-workflow keeps GitHub from turning off tests after 60 days
+      - uses: gautamkrishnar/keepalive-workflow@v2
+        with:
+          time_elapsed: 50
+          workflow_files: 'unittests.yml'
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,120 @@
+name: Create release
+# example: gh workflow run release.yml -f tag_name=v1.1.4
+on:
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        type: string
+        required: true
+        description: Tag name
+      first-release:
+        type: boolean
+        description: First release
+        default: false
+
+jobs:
+  create-release:
+    name: Create release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      EXTENSION_ZIP_NAME: "crowdsec-opencti-external-import-connector"
+
+    steps:
+      - name: Check naming convention
+        run: |
+          VERIF=$(echo ${{ github.event.inputs.tag_name }} | grep -E "^v([0-9]{1,}\.)([0-9]{1,}\.)([0-9]{1,})(-(alpha|beta)\.[0-9]{1,})?$")
+          if [ ! ${VERIF} ]
+          then
+            echo "Tag name '${{ github.event.inputs.tag_name }}' does not comply with naming convention vX.Y.Z"
+            exit 1
+          fi
+
+      - name: Set version number without v
+        run: |
+          echo "VERSION_NUMBER=$(echo ${{ github.event.inputs.tag_name }} | sed 's/v//g' )" >> $GITHUB_ENV
+
+      - name: Set version sources
+        run: |
+          echo "VERSION_SOURCES=$(echo ${{ env.EXTENSION_ZIP_NAME }}-${{ env.VERSION_NUMBER }} )" >> $GITHUB_ENV    
+
+      - name: Clone sources
+        uses: actions/checkout@v4
+        with:
+          path: ${{ env.VERSION_SOURCES }}
+
+      - name: Check version ${{ env.VERSION_NUMBER }} consistency in files
+        # CHANGELOG.md, src/crowdsec.py
+        run: |
+          cd ${{ env.VERSION_SOURCES }}
+          # Check top ## [VERSION_NUMBER](GITHUB_URL/releases/tag/vVERSION_NUMBER) - yyyy-mm-dd in CHANGELOG.md
+          # Example: ## [0.0.2](https://github.com/crowdsecurity/python-capi-sdk/releases/tag/v0.0.2) - 2024-02-07
+          CURRENT_DATE=$(date +'%Y-%m-%d')
+          echo $CURRENT_DATE
+          CHANGELOG_VERSION=$(grep -o -E "## \[(.*)\].* - $CURRENT_DATE"  CHANGELOG.md | head -1 | sed 's/ //g')
+          echo $CHANGELOG_VERSION
+          if [[ $CHANGELOG_VERSION == "##[${{ env.VERSION_NUMBER }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/v${{ env.VERSION_NUMBER }})-$CURRENT_DATE" ]]
+          then
+              echo "CHANGELOG VERSION OK"
+          else
+              echo "CHANGELOG VERSION KO"
+              echo $CHANGELOG_VERSION
+              exit 1
+          fi
+          # Check top [_Compare with previous release_](GITHUB_URL/compare/vLAST_TAG...vVERSION_NUMBER) in CHANGELOG.md
+          # Example: [_Compare with previous release_](https://github.com/crowdsecurity/python-capi-sdk/compare/v0.0.1...v0.0.2)
+          if [[ ${{ github.event.inputs.first-release }} != "true" ]]
+          then
+            COMPARISON=$(grep -oP "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/\K(.*)$" CHANGELOG.md | head -1)
+            LAST_TAG=$(curl -Ls -o /dev/null -w %{url_effective} $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/latest | grep -oP "\/tag\/\K(.*)$")
+            if [[ $COMPARISON == "$LAST_TAG...v${{ env.VERSION_NUMBER }})" ]]
+            then
+              echo "VERSION COMPARISON OK"
+            else
+              echo "VERSION COMPARISON KO"
+              echo $COMPARISON
+              echo "$LAST_TAG...v${{ env.VERSION_NUMBER }})"
+              exit 1
+            fi
+          fi
+          # Check "User-Agent": "crowdsec-import-opencti/vVERSION_NUMBER", in src/crowdsec/client.py
+          UA_VERSION=$(grep -E "\"User-Agent\": \"crowdsec-import-opencti\/v(.*)\""  src/crowdsec/client.py | sed 's/ //g' | sed 's/"//g')
+          if [[ $UA_VERSION == "User-Agent:crowdsec-import-opencti/v${{ env.VERSION_NUMBER }}," ]]
+          then
+              echo "USER AGENT VERSION OK"
+          else
+              echo "USER AGENT VERSION KO"
+              echo $UA_VERSION 
+              exit 1
+          fi
+
+      - name: Create zip for OpenCTI pull request
+        run: |
+          zip -r ${{ env.VERSION_SOURCES }}.zip ${{ env.VERSION_SOURCES }}/ -x '${{ env.VERSION_SOURCES }}/.git*'    
+
+      - name: Create Tag ${{ github.event.inputs.tag_name }}
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: "refs/tags/${{ github.event.inputs.tag_name }}",
+              sha: context.sha
+            })
+
+      - name: Prepare release notes
+        run: |
+          # Retrieve release body and remove ---
+          VERSION_RELEASE_NOTES=$(awk -v ver="[${{ env.VERSION_NUMBER }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/v${{ env.VERSION_NUMBER }})" '/^## / { if (p) { exit }; if ($2 == ver) { p=1; next} } p && NF' CHANGELOG.md | sed ':a;N;$!ba;s/\n---/ /g')
+          echo "$VERSION_RELEASE_NOTES" >> CHANGELOG.txt
+
+      - name: Create release ${{ env.VERSION_NUMBER }}
+        uses: softprops/action-gh-release@v2
+        with:
+          files: ${{ env.VERSION_SOURCES }}.zip
+          body_path: CHANGELOG.txt
+          name: ${{ env.VERSION_NUMBER }}
+          tag_name: ${{ github.event.inputs.tag_name }}
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -0,0 +1,40 @@
+name: Unit tests
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+  schedule:
+    - cron: '25 02 * * THU'
+  workflow_dispatch:
+
+jobs:
+  unittests:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r tests/test-requirements.txt
+
+    - name: Tests
+      run: |
+        python -m pytest -s -v
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+config.yml
+__pycache__
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,16 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## SemVer public API
+
+The [public API](https://semver.org/spec/v2.0.0.html#spec-item-1)  for this project is defined by the set of 
+functions provided by the `src` folder and the following files: `docker-compose.yml`, `Dockerfile`, `entrypoint.sh`
+
+---
+
+
+## [0.0.1](https://github.com/crowdsecurity/cs-opencti-external-import-connector/releases/tag/v1.0.0) - 2024-??-??
+
+- Initial release
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,18 @@
+FROM python:3.11-alpine
+ENV CONNECTOR_TYPE=EXTERNAL_IMPORT
+
+# Copy the connector
+COPY src /opt/opencti-crowdsec-import
+
+# Install Python modules
+# hadolint ignore=DL3003
+RUN apk --no-cache add git build-base libmagic libffi-dev && \
+    cd /opt/opencti-crowdsec-import && \
+    pip3 install --no-cache-dir -r requirements.txt && \
+    apk del git build-base
+
+# Expose and entrypoint
+COPY entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
diff --git a/README.md b/README.md
@@ -1 +1,25 @@
-# cs-opencti-external-import-connector
+![CrowdSec Logo](https://raw.githubusercontent.com/crowdsecurity/cs-opencti-external-import-connector/main/docs/images/logo_crowdsec.png)
+
+# OpenCTI CrowdSec external import connector
+
+> Connector which enriches your knowledge by using CrowdSec's CTI API.
+
+[![Version](https://img.shields.io/github/v/release/crowdsecurity/cs-opencti-external-import-connector)](https://github.com/crowdsecurity/cs-opencti-external-import-connector/releases/)
+[![Unit Tests](https://github.com/crowdsecurity/cs-opencti-external-import-connector/actions/workflows/unittests.yml/badge.svg)](https://github.com/crowdsecurity/cs-opencti-external-import-connector/actions/workflows/unittests.yml)
+[![Code format](https://github.com/crowdsecurity/cs-opencti-external-import-connector/actions/workflows/code-format.yml/badge.svg)](https://github.com/crowdsecurity/cs-opencti-external-import-connector/actions/workflows/code-format.yml)
+
+## Usage
+
+See [User Guide](https://github.com/crowdsecurity/cs-opencti-external-import-connector/blob/main/docs/USER_GUIDE.md)
+
+## Installation
+
+See [Installation Guide](https://github.com/crowdsecurity/cs-opencti-external-import-connector/blob/main/docs/INSTALLATION_GUIDE.md)
+
+## Developer guide
+
+See [Developer guide](https://github.com/crowdsecurity/cs-opencti-external-import-connector/blob/main/docs/DEVELOPER.md)
+
+
+
+
diff --git a/dev/Dockerfile-dev b/dev/Dockerfile-dev
@@ -0,0 +1,13 @@
+FROM python:3.11-alpine
+ENV CONNECTOR_TYPE=EXTERNAL_IMPORT
+
+# Install Python modules
+# hadolint ignore=DL3003
+RUN apk --no-cache add git build-base libmagic libffi-dev
+
+# Expose and entrypoint
+COPY entrypoint-dev.sh /
+RUN chmod +x /entrypoint-dev.sh
+# DO NOT RUN crowdsec.py process, it has to be done manually
+ENTRYPOINT ["/entrypoint-dev.sh"]
+
diff --git a/dev/docker-compose.crowdsec-dev.yml b/dev/docker-compose.crowdsec-dev.yml
@@ -0,0 +1,25 @@
+services:
+  connector-crowdsec-import:
+    build:
+      context: ../cs-opencti-external-import-connector/dev
+      dockerfile: Dockerfile-dev
+    volumes:
+      - ../cs-opencti-external-import-connector/src:/opt/opencti-crowdsec-import
+    environment:
+      - OPENCTI_URL=http://opencti:8080
+      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
+      - CONNECTOR_ID='ChangeMe' # Add connector ID (any valid UUID v4)
+      - CROWDSEC_MAX_TLP=TLP:AMBER
+      - CONNECTOR_TYPE=EXTERNAL_IMPORT
+      - "CONNECTOR_NAME=CrowdSec Import"
+      - CONNECTOR_SCOPE=CrowdSecImport
+      - CROWDSEC_NAME=CrowdSec
+      - "CROWDSEC_DESCRIPTION=CrowdSec CTI enrichment"
+      - CONNECTOR_CONFIDENCE_LEVEL=100 # From 0 (Unknown) to 100 (Fully trusted)
+      - CONNECTOR_LOG_LEVEL=debug
+      - CROWDSEC_KEY='ChangeMe' # Add API Key
+      - CROWDSEC_VERSION=v2 # v2 is the only supported version for now
+    restart: always
+    depends_on:
+      - opencti
+
diff --git a/dev/entrypoint-dev.sh b/dev/entrypoint-dev.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Correct working directory
+cd /opt/opencti-crowdsec-import || exit
+
+pip3 install --no-cache-dir -r requirements.txt
+
+# Idle indefinitely
+while true; do
+    sleep 60 # Sleeps for 60 seconds and then loops again
+done