Skip to content

Commit 0348c93

Browse files
janheinrichmerkerjanheinrichmerker
authored
Merge pull request #25 from webis-de/elastic
Elasticsearch storage backend
2 parents b897a56 + fbd3c6f commit 0348c93

File tree

230 files changed

+15240
-15543
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

230 files changed

+15240
-15543
lines changed

‎.dockerignore‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
venv
2+
*.egg-info
3+
dist
4+
data
5+
docs
6+
integrations
7+
notebooks
8+
scripts
9+
.gitattributes
10+
.gitignore
11+
config.override.yml
12+
build
13+
.pytest_cache
14+
.github
15+
.idea
16+
*.iml
17+
examples
18+
Dockerfile
19+
coverage.xml
20+
.coverage

‎.github/workflows/ci.yml‎

Lines changed: 226 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,241 @@
1-
name: "CI"
1+
name: CI
22

33
on:
44
push:
55

66
jobs:
7-
code-format:
8-
name: "🔍 Python code format"
7+
python-build:
8+
name: 🏗️ Build Python wheels
9+
strategy:
10+
matrix:
11+
python:
12+
- '3.10'
913
runs-on: ubuntu-latest
1014
steps:
11-
- name: "📥 Check-out"
12-
uses: actions/checkout@v4
13-
- name: "🧰 Install Python"
14-
uses: actions/setup-python@v5
15-
with:
16-
python-version: "3.10"
17-
cache: pip
18-
cache-dependency-path: pyproject.toml
19-
- name: "🧰 Install Protoc"
20-
run: sudo apt install protobuf-compiler
21-
- name: "🧰 Install dependencies"
22-
run: |
23-
python -m pip install --upgrade pip
24-
pip install .
25-
pip install .[tests]
26-
- name: "🔍 Check Python code format"
27-
run: flake8 archive_query_log/
28-
lint:
29-
name: "🔍 Python Lint"
15+
- name: 📥 Check-out
16+
uses: actions/checkout@v3
17+
- name: 🧰 Install Protoc
18+
run: sudo apt install protobuf-compiler
19+
- name: 🧰 Install Python
20+
uses: actions/setup-python@v4
21+
with:
22+
python-version: ${{ matrix.python }}
23+
cache: pip
24+
cache-dependency-path: pyproject.toml
25+
- name: 🧰 Install dependencies
26+
run: pip install build twine
27+
- name: 🏗️ Build Python wheels
28+
run: python -m build
29+
- name: 🧪 Check package bundles
30+
run: twine check dist/*
31+
- name: 📤 Upload Python wheels
32+
uses: actions/upload-artifact@v4
33+
if: matrix.python == '3.10'
34+
with:
35+
name: wheels
36+
path: dist
37+
python-code-check:
38+
name: 🔍 Check Python code
39+
strategy:
40+
matrix:
41+
python:
42+
- '3.10'
3043
runs-on: ubuntu-latest
3144
steps:
32-
- name: "📥 Check-out"
33-
uses: actions/checkout@v4
34-
- name: "🧰 Install Python"
35-
uses: actions/setup-python@v5
36-
with:
37-
python-version: "3.10"
38-
cache: pip
39-
cache-dependency-path: pyproject.toml
40-
- name: "🧰 Install Protoc"
41-
run: sudo apt install protobuf-compiler
42-
- name: "🧰 Install dependencies"
43-
run: |
44-
python -m pip install --upgrade pip
45-
pip install .
46-
pip install .[tests]
47-
- name: "🔍 Lint Python code"
48-
run: pylint -E archive_query_log
49-
unit-tests:
50-
name: "🧪 Python unit tests"
45+
- name: 📥 Check-out
46+
uses: actions/checkout@v3
47+
- name: 🧰 Install Protoc
48+
run: sudo apt install protobuf-compiler
49+
- name: 🧰 Install Python
50+
uses: actions/setup-python@v4
51+
with:
52+
python-version: ${{ matrix.python }}
53+
cache: pip
54+
cache-dependency-path: pyproject.toml
55+
- name: 🧰 Install dependencies
56+
run: pip install .[tests]
57+
- name: 🔍 Check Python code
58+
run: ruff .
59+
python-typing:
60+
name: 🔍 Check Python static typing
61+
strategy:
62+
matrix:
63+
python:
64+
- '3.10'
5165
runs-on: ubuntu-latest
5266
steps:
53-
- name: "📥 Check-out"
54-
uses: actions/checkout@v4
55-
- name: "🧰 Install Python"
56-
uses: actions/setup-python@v5
57-
with:
58-
python-version: "3.10"
59-
cache: pip
60-
cache-dependency-path: pyproject.toml
61-
- name: "🧰 Install Protoc"
62-
run: sudo apt install protobuf-compiler
63-
- name: "🧰 Install dependencies"
64-
run: |
65-
python -m pip install --upgrade pip
66-
pip install .
67-
pip install .[tests]
68-
- name: "🧪 Test Python code"
69-
run: pytest --cov=./ --cov-report=xml --capture=no archive_query_log/
70-
- name: "📤 Upload coverage to Codecov"
71-
uses: codecov/codecov-action@v5
72-
with:
73-
token: ${{ secrets.CODECOV_TOKEN }}
74-
release:
75-
name: "🚀 Create GitHub release"
67+
- name: 📥 Check-out
68+
uses: actions/checkout@v3
69+
- name: 🧰 Install Protoc
70+
run: sudo apt install protobuf-compiler
71+
- name: 🧰 Install Python
72+
uses: actions/setup-python@v4
73+
with:
74+
python-version: ${{ matrix.python }}
75+
cache: pip
76+
cache-dependency-path: pyproject.toml
77+
- name: 🧰 Install dependencies
78+
run: pip install .[tests]
79+
- name: 🔍 Check Python static typing
80+
run: mypy .
81+
python-security:
82+
name: 🔍 Check Python code security
83+
strategy:
84+
matrix:
85+
python:
86+
- '3.10'
87+
runs-on: ubuntu-latest
88+
steps:
89+
- name: 📥 Check-out
90+
uses: actions/checkout@v3
91+
- name: 🧰 Install Protoc
92+
run: sudo apt install protobuf-compiler
93+
- name: 🧰 Install Python
94+
uses: actions/setup-python@v4
95+
with:
96+
python-version: ${{ matrix.python }}
97+
cache: pip
98+
cache-dependency-path: pyproject.toml
99+
- name: 🧰 Install dependencies
100+
run: pip install .[tests]
101+
- name: 🔍 Check Python code security
102+
run: bandit -c pyproject.toml -r .
103+
python-test:
104+
name: 🧪 Test Python code
105+
strategy:
106+
matrix:
107+
python:
108+
- '3.10'
109+
runs-on: ubuntu-latest
110+
steps:
111+
- name: 📥 Check-out
112+
uses: actions/checkout@v3
113+
- name: 🧰 Install Protoc
114+
run: sudo apt install protobuf-compiler
115+
- name: 🧰 Install Python
116+
uses: actions/setup-python@v4
117+
with:
118+
python-version: ${{ matrix.python }}
119+
cache: pip
120+
cache-dependency-path: pyproject.toml
121+
- name: 🧰 Install dependencies
122+
run: pip install .[tests]
123+
- name: 🧪 Test Python code
124+
run: pytest --cov --cov-report=xml archive_query_log
125+
- name: 📤 Upload coverage to Codecov
126+
uses: codecov/codecov-action@v3
127+
if: matrix.python == '3.10'
128+
with:
129+
fail_ci_if_error: true
130+
token: ${{ secrets.CODECOV_TOKEN }}
131+
docker-build:
132+
name: 🏗️ Build Docker image
133+
runs-on: ubuntu-latest
134+
steps:
135+
- name: 📥 Check-out
136+
uses: actions/checkout@v3
137+
- name: 🧰 Set up QEMU
138+
uses: docker/setup-qemu-action@v2
139+
- name: 🧰 Set up Docker Buildx
140+
uses: docker/setup-buildx-action@v2
141+
- name: 🏗️ Build Docker image
142+
uses: docker/build-push-action@v4
143+
with:
144+
context: .
145+
push: false
146+
cache-from: type=gha
147+
cache-to: type=gha,mode=max
148+
python-publish:
149+
name: 🚀 Publish Python wheels
150+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
151+
needs:
152+
- python-build
153+
- python-code-check
154+
- python-typing
155+
- python-security
156+
- python-test
157+
- docker-build
158+
runs-on: ubuntu-latest
159+
permissions:
160+
id-token: write
161+
steps:
162+
- name: 📥 Check-out
163+
uses: actions/checkout@v3
164+
- name: 📥 Download Python wheels
165+
uses: actions/download-artifact@v4
166+
with:
167+
name: wheels
168+
path: dist
169+
- name: 🚀 Publish Python wheels
170+
uses: pypa/gh-action-pypi-publish@release/v1
171+
docker-publish:
172+
name: 🚀 Publish Docker image
173+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
174+
permissions:
175+
packages: write
176+
needs:
177+
- python-build
178+
- python-code-check
179+
- python-typing
180+
- python-security
181+
- python-test
182+
- docker-build
183+
runs-on: ubuntu-latest
184+
steps:
185+
- name: 📥 Check-out
186+
uses: actions/checkout@v3
187+
- name: 🧰 Set up QEMU
188+
uses: docker/setup-qemu-action@v2
189+
- name: 🧰 Set up Docker Buildx
190+
uses: docker/setup-buildx-action@v2
191+
- name: 🔑 Login to GitHub Packages
192+
uses: docker/login-action@v2
193+
with:
194+
registry: ghcr.io
195+
username: ${{ github.actor }}
196+
password: ${{ secrets.GITHUB_TOKEN }}
197+
- name: ℹ️ Extract image metadata
198+
id: meta
199+
uses: docker/metadata-action@v4
200+
with:
201+
images: |
202+
ghcr.io/${{ github.repository }}
203+
- name: 🚀 Build and push image
204+
uses: docker/build-push-action@v4
205+
with:
206+
context: .
207+
push: true
208+
tags: ${{ steps.meta.outputs.tags }}
209+
labels: ${{ steps.meta.outputs.labels }}
210+
cache-from: type=gha
211+
cache-to: type=gha,mode=max
212+
github-release:
213+
name: 🚀 Create GitHub release
76214
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
77215
needs:
78-
- code-format
79-
- lint
80-
- unit-tests
216+
- python-build
217+
- python-code-check
218+
- python-typing
219+
- python-security
220+
- python-test
221+
- docker-build
222+
permissions:
223+
contents: write
81224
runs-on: ubuntu-latest
82225
steps:
83-
- name: "📥 Check-out"
84-
uses: actions/checkout@v4
85-
- name: "🏷️ Get version tag"
86-
id: get-version
87-
run: echo ::set-output name=tag::${GITHUB_REF/refs\/tags\//}
88-
- name: "📥 Download Python wheel"
89-
uses: actions/upload-artifact@v4
90-
with:
91-
name: wheel
92-
path: dist/*
93-
- name: "🚀 Create GitHub release"
94-
uses: softprops/action-gh-release@v2
95-
with:
96-
name: Release ${{ steps.get-version.outputs.tag }}
97-
files: dist/*
98-
fail_on_unmatched_files: true
99-
draft: false
100-
prerelease: false
101-
generate_release_notes: true
226+
- name: 📥 Check-out
227+
uses: actions/checkout@v3
228+
- name: 📥 Download Python wheels
229+
uses: actions/download-artifact@v4
230+
with:
231+
name: wheels
232+
path: dist
233+
- name: 🚀 Create GitHub release
234+
uses: softprops/action-gh-release@v1
235+
with:
236+
name: Release ${{ github.ref_name }}
237+
files: dist/*
238+
fail_on_unmatched_files: true
239+
draft: false
240+
prerelease: false
241+
generate_release_notes: true

‎.gitignore‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
### JetBrains ###
2-
.idea
32
out/
43
*.iml
54

@@ -120,9 +119,6 @@ acs-*.bib
120119
*.slo
121120
*.sls
122121

123-
# uncomment this for glossaries-extra (will ignore makeindex's style files!)
124-
# *.ist
125-
126122
# gnuplot
127123
*.gnuplot
128124
*.table
@@ -471,14 +467,6 @@ cython_debug/
471467
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
472468
#.idea/
473469

474-
### VisualStudioCode ###
475-
.vscode/*
476-
!.vscode/settings.json
477-
!.vscode/tasks.json
478-
!.vscode/launch.json
479-
!.vscode/extensions.json
480-
!.vscode/*.code-snippets
481-
482470
# Local History for Visual Studio Code
483471
.history/
484472

@@ -498,5 +486,7 @@ cython_debug/
498486

499487
# Custom exclusions
500488
*.lop
501-
reimer*.pdf
502-
cd-cover.pdf
489+
config.override.yml
490+
491+
.vscode
492+
.idea

0 commit comments

Comments
 (0)