Merge remote-tracking branch 'upstream/main' into feature/vote-decrypt

.github/workflows/autoupdate.yml

@@ -6,12 +6,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: go fmt
       run: test -z $(gofmt -l .)

.github/workflows/project-automation.yml

@@ -0,0 +1,32 @@
+name: Project automation
+on:
+  workflow_call:
+    inputs:
+      resource_node_id:
+        required: true
+        type: string
+      status_value:
+        required: true
+        type: string
+    secrets:
+      AUTOMATION_APP_ID:
+        required: true
+      AUTOMATION_APP_INSTALLATION_ID:
+        required: true
+      AUTOMATION_APP_PRIVATE_KEY:
+        required: true
+
+jobs:
+  workflow_call:
+    name: Set status
+    runs-on: ubuntu-latest
+    steps:
+      - uses: leonsteinhaeuser/[email protected]
+        with:
+          gh_app_ID: ${{ secrets.AUTOMATION_APP_ID }}
+          gh_app_installation_ID: ${{ secrets.AUTOMATION_APP_INSTALLATION_ID }}
+          gh_app_secret_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+          organization: OpenSlides
+          project_id: 2
+          resource_node_id: ${{ inputs.resource_node_id }}
+          status_value: ${{ inputs.status_value }}

.github/workflows/project-issue-closed.yml

@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  issues:
+    types:
+      - closed
+
+jobs:
+  issue_closed:
+    name: Issue closed
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.issue.node_id }}
+      status_value: "Done"

.github/workflows/project-issue-opened.yml

@@ -0,0 +1,15 @@
+name: Project automation
+on:
+  issues:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  issue_opened:
+    name: Issue opened
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.issue.node_id }}
+      status_value: "Backlog"

.github/workflows/project-pull-request-closed.yml

@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - closed
+
+jobs:
+  pull_request_closed:
+    name: Pull request closed
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Done"

.github/workflows/project-pull-request-opened.yml

@@ -0,0 +1,15 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  pull_request_opened:
+    name: Pull request opened
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Work in progress"

.github/workflows/project-pull-request-review-requested.yml

@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - review_requested
+
+jobs:
+  pull_request_review_requested:
+    name: Pull request review requested
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Review in progress"

.github/workflows/readme.yml

@@ -27,13 +27,13 @@ jobs:
 
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.20'
+        go-version: '1.21'
       id: go
 
     - name: Check out code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: create secrets
       run: |
@@ -45,7 +45,9 @@ jobs:
     - name: Start With golang
       run: |
         go build
-        export SECRETS_PATH=secrets
+        export DATABASE_PASSWORD_FILE=secrets/postgres_password
+        export AUTH_TOKEN_KEY_FILE=secrets/auth_token_key
+        export AUTH_COOKIE_KEY_FILE=secrets/auth_cookie_key
         timeout --preserve-status --signal=SIGINT 2s ./openslides-autoupdate-service
 
     - name: Start With Docker

.github/workflows/update-generated-files.yml

@@ -8,25 +8,25 @@ jobs:
     name: generate files
     runs-on: ubuntu-latest
     steps:
-    - uses: tibdex/github-app-token@v1
+    - uses: tibdex/github-app-token@v2
       id: generate-token
       with:
         app_id: ${{ secrets.AUTOMATION_APP_ID }}
         private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Check out code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run go generate
       run: go generate ./...
 
     - name: Create or update PR
-      uses: peter-evans/create-pull-request@v5
+      uses: peter-evans/create-pull-request@v6
       with:
         token: ${{ steps.generate-token.outputs.token }}
         commit-message: "update generated files"

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.6-alpine as base
+FROM golang:1.21.6-alpine as base
 WORKDIR /root/
 
 RUN apk add git

README.md

@@ -24,38 +24,32 @@ It also needs a running postgres and redis instance. You can start one with:
 
 ```
 docker run  --network host -e POSTGRES_PASSWORD=password -e POSTGRES_USER=openslides -e POSTGRES_DB=openslides postgres:13
-```
-
-and
-
-```
 docker run --network host redis
 ```
 
 
 ### With Golang
 
 ```
-export SECRETS_PATH=secrets
+export DATABASE_PASSWORD_FILE=secrets/postgres_password
+export AUTH_TOKEN_KEY_FILE=secrets/auth_token_key
+export AUTH_COOKIE_KEY_FILE=secrets/auth_cookie_key
 go build
 ./autoupdate
 ```
 
 
 ### With Docker
 
-The docker build uses the auth token as default. Either configure it to use the
-auth-fake services (see environment variables below) or make sure the service
-inside the docker container can connect to the auth service. For example with
-the docker argument --network host. The auth-secrets have to given as a file.
+Make sure the service inside the docker container can connect to the auth
+service, postgres and redis, for example with the docker argument `--network
+host`.
 
 ```
 docker build . --tag openslides-autoupdate
 docker run --network host -v $PWD/secrets:/run/secrets openslides-autoupdate
 ```
 
-It uses the host network to connect to redis and postgres.
-
 
 ### With Auto Restart
 
@@ -121,7 +115,7 @@ A request can have a body and the `k`-query parameter.
 After the request is send, the values to the keys are returned as a json-object
 without a newline:
 ```
-{"user/1/name":"value","user/2/name":"value"}
+{"user/1/username":"value","user/2/username":"value"}
 ```
 
 With the query parameter `single` the server writes the first response and
@@ -137,7 +131,7 @@ specific position from the datastore. This implieds `single`:
 
 ### Updates via redis
 
-Keys are updated via redis:
+Values are updated via redis:
 
 `xadd ModifiedFields * user/1/username newName user/1/password newPassword`
 
@@ -192,13 +186,82 @@ To get the data at a position, use the normal autoupdate request with the
 attribute `position`. See above.
 
 
-### Internal Restrict FQIDs
+### Internal Autoupdate
+
+The autoupdate service provides an internal route to return fields for a defined user.
+
+`curl "localhost:9012/internal/autoupdate?user_id=42&k=user/1/username"`
+
+It also supports the attributes `single=1` and the normal autoupdate body.
+
+
+### Connection Count
+
+The autoupdate services saves how many connections are currently open to each user.
+The save interval can be defined with the environment variable
+`METRIC_SAVE_INTERVAL`. The default is 5 minutes.
+
+The values are saved for each instance of the autoupdate service. So it is
+possible to access all open connection for every instance of the autoupdate service in
+the same cloud.
 
-The autoupdate service provides an internal route to restrict a list of fqids.
+`curl "localhost:9012/service/autoupdate/connection_count"`
 
-`curl localhost:9012/internal/autoupdate -d '{"user_id":23,"fqids":["user/1","motion/6"]}'`
+It returns a JSON dictonary like this:
+
+`{"0":15,"1":4,"2":3}`
+
+The key is a user ID and the value is the amount of currently open connections. User ID
+`0` is the anonymous user. It the example above, the anonymous user has 15 open
+connections, the user with the ID 1 has 4 open connections and the user with the
+ID 2 has 3 open connection.
+
+Users can only access this page if they have the organization management level
+or higher.
+
+
+## Metric
+
+The autoupdate service logs some metric values. The interval can be set with the
+environment variable `METRIC_INTERVAL`.
+
+The logged metric is a json dictonary like:
+
+```json
+{
+    "connected_users_anonymous_connections": 1,
+    "connected_users_average_connections": 3,
+    "connected_users_current": 3,
+    "connected_users_current_local": 3,
+    "connected_users_total": 3,
+    "connected_users_total_local": 3,
+    "current_connections": 8,
+    "current_connections_local": 8,
+    "datastore_cache_key_len": 343,
+    "datastore_cache_size": 3114,
+    "runtime_goroutines": 42
+}
+```
 
-It returns all fields for the given objects restricted for the given user id.
+The values are:
+
+* `connected_users_anonymous_connections`: Number of connections from the
+  anonymous users from all autoupdate instances.
+* `connected_users_average_connections`: Average connection count for each user
+  except for anonymous user.
+* `connected_users_current`: Amount of connected users that have at least one
+  open connection.
+* `connected_users_current_local`: Amount of connected users that have at least
+  one open connection of this instance.
+* `connected_users_total`: Amount of different users that are currently
+  connected or were connected since the autoupdate service was started.
+* `connected_users_total_local`: Same as `connected_users_total`, but only for this
+instance.
+* `current_connections`: Amount of all connections.
+* `current_connections_local`: Amount of all connections of this instance.
+* `datastore_cache_key_len`: Amount of keys in the cache.
+* `datastore_cache_size`: Combined size of all values in the cache.
+* `runtime_goroutines`: Current goroutines used by the instance.
 
 
 ## Configuration