Filmlist Merger

Uses a database to merge multiple film lists into one.

Settings

Spring Boot setting	Environment variable	Default value	Description
spring.datasource.url	SPRING_DATASOURCE_URL	jdbc:postgresql://localhost:5432/filmlistmerger?useServerPrepStmts=true&cachePrepStmts=true&rewriteBatchedStatements=true	The JDCB url to connect to the database including the database name
spring.datasource.username	SPRING_DATASOURCE_USERNAME	filmlistmerger	The database user username
spring.datasource.password	SPRING_DATASOURCE_PASSWORD	filmlistmerger	The database user passwort
filmlistmerger.input.path	FILMLISTMERGER_INPUT_PATH	input	The path to the input folder from which the film list will be read
filmlistmerger.output.format	FILMLISTMERGER_OUTPUT_FORMAT	NEW	The format of the merged film list. Can be NEW or OLD
filmlistmerger.output.file	FILMLISTMERGER_OUTPUT_FILE	output/filmlist.json	The path where the merged film list should be written to including the file name.
filmlistmerger.database.startup.enabled	FILMLISTMERGER_DATABASE_STARTUP_ENABLED	true	Waits the timeout of filmlistmerger.database.startup.timeout in seconds for the database to startup when true.
filmlistmerger.database.startup.timeout	FILMLISTMERGER_DATABASE_STARTUP_TIMEOUT	10	A timeout in seconds to wait for the database to startup. Only when filmlistmerger.database.startup.enabled is true.
management.metrics.export.influx.enabled	MANAGEMENT_METRICS_EXPORT_INFLUX_ENABLED	true	Sends the metrics to a configured InfluxDB when true
management.metrics.export.influx.bucket	MANAGEMENT_METRICS_EXPORT_INFLUX_BUCKET	filmlistmerger	The InfluxDB bucket name to use
management.metrics.export.influx.org	MANAGEMENT_METRICS_EXPORT_INFLUX_ORG	mediathekview	The InfluxDB organisation
management.metrics.export.influx.token	MANAGEMENT_METRICS_EXPORT_INFLUX_TOKEN		The InfluxDB authentication token
management.metrics.export.influx.uri	MANAGEMENT_METRICS_EXPORT_INFLUX_URI	http://localhost:8086	The InfluxDB URI

How to run

Docker

1. Start a postgres Database like this:
   docker run --network="host" -p 5432:5432 --env POSTGRES_DB=filmlistmerger --env POSTGRES_USER=filmlistmerger --env POSTGRES_PASSWORD=filmlistmerger --name filmlistmerger-db --shm-size 4GB postgres -c max_connections=515

2. Create a folder for your input film lists and one for the merged output

3. Start the application:
   docker run --network="host" -v $PWD/input:/workspace/input:rw -v $PWD/output:/workspace/output:rw --env SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/filmlistmerger?useServerPrepStmts=true&cachePrepStmts=true&rewriteBatchedStatements=true" --name filmlistmerger mediathekview/filmlistmerger:0.0.1-SNAPSHOT

Docker compose

Source: docker-compose.yml

services:
  database:
    image: postgres
    command:
    - -N 515
    environment:
      POSTGRES_DB: mv_filmlistmerger
      POSTGRES_USER: mediathekview
      POSTGRES_PASSWORD: mediathekview
    shm_size: 4GB

  fimlistmerger:
    image: mediathekview/filmlistmerger:0.0.1-SNAPSHOT
    depends_on:
      - database
    links:
      - database
    environment:
      SPRING_DATASOURCE_URL: jdbc:postgresql://database:5432/mv_filmlistmerger?useServerPrepStmts=true&cachePrepStmts=true&rewriteBatchedStatements=true
      SPRING_DATASOURCE_USERNAME: mediathekview
      SPRING_DATASOURCE_PASSWORD: mediathekview
      FILMLISTMERGER_INPUT_PATH: input
      FILMLISTMERGER_OUTPUT_FORMAT: NEW
      FILMLISTMERGER_OUTPUT_FILE: output/filmlist.json
      MANAGEMENT_METRICS_EXPORT_INFLUX_ENABLED: 'true'
      MANAGEMENT_METRICS_EXPORT_INFLUX_BUCKET: filmlistmerger
      MANAGEMENT_METRICS_EXPORT_INFLUX_ORG: mediathekview
      MANAGEMENT_METRICS_EXPORT_INFLUX_TOKEN: filmlistmerger
      MANAGEMENT_METRICS_EXPORT_INFLUX_URI: http://influx:8086
    volumes:
      - $PWD/input:/workspace/input:rw
      - $PWD/output:/workspace/output:rw
      -
  influx:
    container_name: influx
    image: influxdb:alpine
    ports:
      - 8086:8086
    environment:
      DOCKER_INFLUXDB_INIT_MODE: setup
      DOCKER_INFLUXDB_INIT_USERNAME: filmlistmerger
      DOCKER_INFLUXDB_INIT_PASSWORD: filmlistmerger
      DOCKER_INFLUXDB_INIT_ORG: mediathekview
      DOCKER_INFLUXDB_INIT_BUCKET: filmlistmerger
      DOCKER_INFLUXDB_INIT_RETENTION: 1w
      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: filmlistmerger

  grafana:
    image: grafana/grafana
    container_name: grafana
    ports:
      - 3000:3000

Kubernetes

Source: kubernetes.yml

apiVersion: v1
kind: Namespace
metadata:
  name: mediathekview
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: filmlistmerger-inputfiles-claim
  namespace: mediathekview
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 3Gi
  storageClassName: local-path
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: filmlistmerger-database-claim
  namespace: mediathekview
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 10Gi
  storageClassName: local-path
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: filmlistmerger-metrics-grafana-claim
  namespace: mediathekview
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 1Gi
  storageClassName: local-path
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  namespace: mediathekview
  name: filmlist-download
  labels:
    app: filmlistmerger
spec:
  schedule: "30 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
            - name: filmlist-downloader
              image: mediathekview/filmlistdownloader:v1.0.1
              env:
                - name: OUTPUT_PATH
                  value: /output
              volumeMounts:
                - mountPath: /output
                  name: filmlist-input
              imagePullPolicy: Always
          restartPolicy: OnFailure
          volumes:
            - name: filmlist-input
              persistentVolumeClaim:
                claimName: filmlistmerger-inputfiles-claim
---
apiVersion: v1
kind: Service
metadata:
  name: filmlistmerger-metrics
  namespace: mediathekview

spec:
  ports:
    - name: grafana
      protocol: TCP
      port: 3000
    - name: influxdb
      protocol: TCP
      port: 8086
  selector:
    app: metrics
---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: mediathekview
  name: filmlistmerger-metrics
  labels:
    app: grafana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: metrics
  template:
    metadata:
      labels:
        app: metrics
    spec:
      containers:
        - name: grafana
          image: grafana/grafana
          ports:
            - containerPort: 3000
          volumeMounts:
            - mountPath: /var/lib/grafana
              name: grafana-pv
        - name: influx
          image: influxdb:alpine
          ports:
            - containerPort: 8086
          env:
            - name: DOCKER_INFLUXDB_INIT_MODE
              value: setup
            - name: DOCKER_INFLUXDB_INIT_USERNAME
              value: filmlistmerger
            - name: DOCKER_INFLUXDB_INIT_PASSWORD
              value: filmlistmerger
            - name: DOCKER_INFLUXDB_INIT_ORG
              value: mediathekview
            - name: DOCKER_INFLUXDB_INIT_BUCKET
              value: filmlistmerger
            - name : DOCKER_INFLUXDB_INIT_RETENTION
              value: 1w
            - name: DOCKER_INFLUXDB_INIT_ADMIN_TOKEN
              value: filmlistmerger
      volumes:
        - name: grafana-pv
          persistentVolumeClaim:
            claimName: filmlistmerger-metrics-grafana-claim
---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: mediathekview
  name: filmlistmerger
  labels:
    app: filmlistmerger
spec:
  replicas: 1
  selector:
    matchLabels:
      app: filmlistmerger
  template:
    metadata:
      labels:
        app: filmlistmerger
    spec:
      containers:
        - name: filmlistmerger-database
          image: postgres
          args:
           - -c
           - max_connections=515
          env:
            - name: POSTGRES_DB
              value: mv_filmlistmerger
            - name: POSTGRES_USER
              value: mediathekview
            - name: POSTGRES_PASSWORD
              value: mediathekview
            - name: PGDATA
              value: /data
          volumeMounts:
            - mountPath: /dev/shm
              name: shared-memory
            - mountPath: /data
              name: filmlist-database
        - name: filmlistmerger
          image: mediathekview/filmlistmerger:0.0.1-SNAPSHOT
          env:
            - name: SPRING_DATASOURCE_URL
              value: jdbc:postgresql://localhost:5432/mv_filmlistmerger?useServerPrepStmts=true&cachePrepStmts=true&rewriteBatchedStatements=true
            - name: SPRING_DATASOURCE_USERNAME
              value: mediathekview
            - name: SPRING_DATASOURCE_PASSWORD
              value: mediathekview
            - name: LOGGING_LEVEL_ROOT
              value: INFO
            - name: FILMLISTMERGER_INPUT_PATH
              value: input
            - name: FILMLISTMERGER_OUTPUT_FORMAT
              value: NEW
            - name: FILMLISTMERGER_OUTPUT_FILE
              value: output/filmlist.json
            - name: FILMLISTMERGER_DATABASE_STARTUP_TIMEOUT
              value: '60'
            - name: FILMLISTMERGER_DATABASE_STARTUP_ENABLED
              value: 'true'
            - name: MANAGEMENT_METRICS_EXPORT_INFLUX_ENABLED
              value: 'true'
            - name: MANAGEMENT_METRICS_EXPORT_INFLUX_BUCKET
              value: filmlistmerger
            - name: MANAGEMENT_METRICS_EXPORT_INFLUX_ORG
              value: mediathekview
            - name: MANAGEMENT_METRICS_EXPORT_INFLUX_TOKEN
              value: filmlistmerger
            - name: MANAGEMENT_METRICS_EXPORT_INFLUX_URI
              value: http://filmlistmerger-metrics:8086
          volumeMounts:
            - name: filmlist-input
              mountPath: /workspace/input
            - name: filmlist-output
              mountPath: /workspace/output
          imagePullPolicy: Always
      volumes:
        - name: filmlist-input
          persistentVolumeClaim:
            claimName: filmlistmerger-inputfiles-claim
        - name: filmlist-database
          persistentVolumeClaim:
            claimName: filmlistmerger-database-claim
        - name: filmlist-output
          emptyDir: { }
        - name: shared-memory
          emptyDir: { }
---
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
  name: filmlistmerger-metrics-ingress
  namespace: mediathekview

spec:
  rules:
    - host: metrics.filmlistmerger.mediathekview.de
      http:
        paths:
          - path: /
            backend:
              serviceName: filmlistmerger-metrics
              servicePort: 3000

Deployment View

Maven Buildfile

To gain a good maven pom we use Spring Boots Parent Pom to gain the advantages of the Spring Boot build system for dependencies and also for versions.

Our pom.xml will be sorted with sortpom-maven-plugin. If you change something please run the plugin before commit. Here is the commandline you need:

Example 1. Maven CLI to get the pom in correct order

mvn com.github.ekryd.sortpom:sortpom-maven-plugin:sort -Dsort.keepBlankLines -Dsort.predefinedSortOrder=recommended_2008_06