feat: generate data for indicated region, district and disease

.env

@@ -1,5 +1,9 @@
 CASE_COUNT=4
 EVENT_COUNT=3
+REGION=Niedersachsen
+DISTRICT=Wolfsburg
+DISEASE=CORONAVIRUS
+
 OPENAPI_GENERATOR_VERSION=4.3.1
 PERFORMANCE_LOG_DIR=/srv/timings
 LOGLEVEL=info
@@ -97,5 +101,3 @@ SORMAS_ORG_ID=HZI_TEST
 SORMAS_ORG_NAME="HZI Braunschweig"
 SORMAS_S2S_CERT_PASS=password
 SORMAS_S2S_REST_PASSWORD=passwordpassword
-
-

.gitignore

@@ -1,3 +1,4 @@
 docker-data/
 .idea
-timings/*
+timings/*
+.vscode

README.md

@@ -11,4 +11,3 @@ Goal of this project is to generate and import credible test data into SORMAS. T
 
 # Run it
 `docker-compose up -d`: This will start a minimal stack of [SORMAS-Docker](https://github.com/hzi-braunschweig/SORMAS-Docker) which receives the generated data. For more options see [here](src/importer/README.md).
-

docker-compose.yml

@@ -89,17 +89,23 @@ services:
       context: src/
       args:
         - DOCKERIZED=false
-        - SORMAS_VERSION=${SORMAS_VERSION}
         - OPENAPI_GENERATOR_VERSION=${OPENAPI_GENERATOR_VERSION}
         - PERFORMANCE_LOG_DIR=${PERFORMANCE_LOG_DIR}
+        - SORMAS_VERSION=${SORMAS_VERSION}
     environment:
+      - DOMAIN_NAME=${DOMAIN_NAME}
+      - SORMAS_REST_USERNAME=${SORMAS_REST_USERNAME}
+      - SORMAS_REST_PASSWORD=${SORMAS_REST_PASSWORD}
       - SORMAS_POSTGRES_USER=${SORMAS_POSTGRES_USER}
       - SORMAS_POSTGRES_PASSWORD=${SORMAS_POSTGRES_PASSWORD}
-      - SORMAS_SERVER_URL=${SORMAS_SERVER_URL}
       - DB_HOST=${DB_HOST}
-      - DOMAIN_NAME=${DOMAIN_NAME}
+      - DB_NAME=${DB_NAME}
+      - SUPERVISOR_UUID=${SUPERVISOR_UUID}
       - CASE_COUNT=${CASE_COUNT}
       - EVENT_COUNT=${EVENT_COUNT}
+      - REGION=${REGION}
+      - DISTRICT=${DISTRICT}
+      - DISEASE=${DISEASE}
       - PERFORMANCE_LOG_DIR=${PERFORMANCE_LOG_DIR}
       - ANALYZE_PERFORMANCE=${ANALYZE_PERFORMANCE}
       - LOGLEVEL=${LOGLEVEL}

src/Dockerfile

@@ -1,22 +1,28 @@
 FROM debian:buster-slim
-ARG SORMAS_VERSION
-ARG OPENAPI_GENERATOR_VERSION
+
 ARG DOCKERIZED=false
+ARG OPENAPI_GENERATOR_VERSION
 ARG PERFORMANCE_LOG_DIR
+ARG SORMAS_VERSION
+
 # see https://github.com/debuerreotype/debuerreotype/issues/10
 RUN mkdir -p /usr/share/man/man1
-RUN apt update && apt upgrade -y
-RUN apt install --no-install-recommends -y \
-    curl wget unzip vim gcc \
-    python3 python3-pip python3-venv python3-dev \
-    maven jq \
-    postgresql libpq-dev
+RUN apt update > /dev/null && \
+    apt upgrade -y > /dev/null && \
+    apt install --no-install-recommends -y \
+        curl wget unzip vim gcc \
+        python3 python3-pip python3-venv python3-dev \
+        maven jq \
+        postgresql libpq-dev > /dev/null
 
 WORKDIR /srv
-RUN mkdir $PERFORMANCE_LOG_DIR
-COPY config/setup_and_run.sh setup_and_run.sh
-COPY importer/gen-client.sh /srv/importer/
-COPY config/requirements.txt /srv/importer/
-RUN  importer/gen-client.sh
-COPY . /srv
 CMD ["./setup_and_run.sh"]
+
+COPY config/setup_and_run.sh  setup_and_run.sh
+COPY config/requirements.txt  /srv/importer/
+COPY importer/gen-client.sh   /srv/importer/
+
+RUN mkdir $PERFORMANCE_LOG_DIR && \
+    importer/gen-client.sh
+
+COPY . /srv

src/config/requirements.txt

@@ -1,6 +1,6 @@
-psycopg2==2.8.6
+click
 numpy==1.19.4
 pandas==1.1.5
-click
-zeep
+psycopg2-binary==2.8.6
 pygrok
+zeep

src/config/setup_and_run.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 source venv/bin/activate
+
 echo "Starting import"
+
 cd importer || exit
-python3 main.py --case-count $CASE_COUNT --event-count $EVENT_COUNT
+
+python3 main.py \
+    --case-count $CASE_COUNT \
+    --event-count $EVENT_COUNT \
+    --region $REGION \
+    --district $DISTRICT \
+    --disease $DISEASE

src/generator/README.md

@@ -4,7 +4,7 @@ An overview can be found in the presentation [presentation/sormas-oegd-credible-
 
 The scripts consists essentially in three parts:
 1. the parameters and functions
-2. the reading of input data in [data/in/](data/in/) (the case counts from RKI) and the generating of the data set, wich is stored as RDS, CSV and Excel files in [data/out/](data/out/)
+2. the reading of input data in [data/in/](data/in/) (the case counts from RKI) and the generating of the data set, which is stored as RDS, CSV and Excel files in [data/out/](data/out/)
 3. the plots, which are stored in [img/](img/)
 
 Parts 2 and 3 are largely independent. Part to is within `if (generate_dataset) {...}`, if `generate_dataset` is `FALSE`, the data set is loaded from the RDS files to be used for the plots.
@@ -24,7 +24,7 @@ Not simulations or scenarios! Rather "play" and demonstration  data: also readab
 ### Adding new fields for a more complete data set
 
 Addresses:
-- assign a proper address to each geolocation, e.g., through the Google Maps API 
+- assign a proper address to each geolocation, e.g., through the Google Maps API
 
 Tests, quarantine, isolation:
 - tested persons as fourth person category besides case, contact and participant (one person can be any number of those at the same time)
@@ -33,15 +33,15 @@ Tests, quarantine, isolation:
 - whether a case is in isolation
 - dates of isolation/quarantine start and end
 
-Occupation/setting of a person, an infection, an event, especially according "household", screening, and to IfSG definitions (), see RKI's [Epidemiologisches Bulletin](https://www.rki.de/DE/Content/Infekt/EpidBull/epid_bull_node.html) 38/20 and RKI's Lagebricht 
+Occupation/setting of a person, an infection, an event, especially according "household", screening, and to IfSG definitions (), see RKI's [Epidemiologisches Bulletin](https://www.rki.de/DE/Content/Infekt/EpidBull/epid_bull_node.html) 38/20 and RKI's Lagebricht
 
 Context:
-- place of infection (address, location ,type) 
-- type of venue/setting for infection, contact, event  venue, 
+- place of infection (address, location ,type)
+- type of venue/setting for infection, contact, event  venue,
 - name of event
 - date of contact
 
-Hospitalization and death: 
+Hospitalization and death:
 - whether in ICU
 - dates of hospitalization start and end
 - date of death
@@ -78,7 +78,7 @@ Transmission and infection dynamics:
 - age-dependent susceptibility
 - age-dependent infectiosity
 - more realistic period of infectiosity
-- age-dependent probability of being in a type of place of infection, of contact or event setting 
+- age-dependent probability of being in a type of place of infection, of contact or event setting
 - transmission risk dependent on contact and event setting type
 - contacts between cases follow the same social-contact-matrix based probabilities
 
@@ -87,8 +87,8 @@ Age (and sex?) dependent symptoms:
 - given that one has symptoms, age (and sex?) dependent symptom and age specific probabilities from the literature
 - non-cases may also have symptoms (?)
 
-Spread dynamics: 
-- not just one retrospective snapshot, but day-to-day changes in reporting, contacts, etc.: a contact can get sick, contacted, be tested (negtive or positive > case), go into quarantine...
+Spread dynamics:
+- not just one retrospective snapshot, but day-to-day changes in reporting, contacts, etc.: a contact can get sick, contacted, be tested (negative or positive > case), go into quarantine...
 
 Tests:
 - take positive rate into account
@@ -137,4 +137,4 @@ Modelling:
 
 ## Authors
 
-## License
+## License

src/importer/generator/contacts.py

@@ -59,11 +59,11 @@ def gen_contact_dto(person_uuid, case_uuid, disease):
         creation_date=dnow(),  # todo  required missing
         change_date=dnow(),  # todo  required missing
         reporting_user=surv_sup_user_ref(),
-        last_contact_date=datetime.date.fromisoformat('2020-02-01'),
+        last_contact_date=datetime.date.fromisoformat('2021-05-01'),
         disease=disease,
         caze=case_ref(case_uuid),  # todo validation exception talking about region
         health_conditions=gen_health_condition_dto(),  # todo ContactFacadeRjb:1092 nullpointer if missing
-        # todo contact classifaction is required in UI
+        # todo contact classification is required in UI
         multi_day_contact=False  # todo check required
     )
     return contact_dto

src/importer/generator/district.py

@@ -10,7 +10,7 @@
 def default_district():
     with sormas_db_connect() as conn:
         with conn.cursor() as cur:
-            cur.execute("SELECT uuid FROM district")
+            cur.execute("SELECT uuid FROM district LIMIT 1")
             uuid = cur.fetchone()[0]
             return DistrictReferenceDto(uuid=uuid)
 
@@ -34,8 +34,10 @@ def insert_district(district, region_id):
             _id = max(all_ids) + 1
             date = datetime.date.today()
             uuid = duuid()
-            cur.execute("INSERT INTO district (id, changedate, creationdate, name, uuid, region_id, epidcode, archived)"
-                        "VALUES (%s,%s, %s, %s, %s, %s, %s, %s)",
-                        [_id, date, date, district, uuid, region_id, 'DIS', False])
+            cur.execute(
+                "INSERT INTO district"
+                " (id, changedate, creationdate, name, uuid, region_id, epidcode, archived)"
+                " VALUES (%s,%s, %s, %s, %s, %s, %s, %s)",
+                [_id, date, date, district, uuid, region_id, 'DIS', False])
 
             return _id, uuid

src/importer/generator/event.py

@@ -20,7 +20,7 @@
 random.seed(42)
 
 
-def gen_event_dto(event_desc=None, start_date=None, location=None,disease= None):
+def gen_event_dto(event_desc=None, start_date=None, location=None, disease=None):
     if event_desc is None:
         raise NotImplementedError
 
@@ -41,8 +41,7 @@ def gen_event_dto(event_desc=None, start_date=None, location=None,disease= None)
         reporting_user=surv_sup_user_ref(),
         start_date=start_date,
         event_location=location,
-        disease=Disease.CORONAVIRUS
-        # todo region + district is required in the UI!
+        disease=disease,
     )
     return event_dto
 

src/importer/generator/region.py

@@ -10,7 +10,7 @@
 def default_region():
     with sormas_db_connect() as conn:
         with conn.cursor() as cur:
-            cur.execute("SELECT uuid FROM region")
+            cur.execute("SELECT uuid FROM region LIMIT 1")
             uuid = cur.fetchone()[0]
             return RegionReferenceDto(uuid=uuid)
 
@@ -28,13 +28,15 @@ def insert_region(region):
             if exists:
                 logging.info(f'{region} already exists in the DB, value was {exists}')
                 return exists[0], exists[1]
+
             cur.execute("SELECT id FROM region")
             all_ids = list(chain.from_iterable(cur.fetchall()))
             _id = max(all_ids) + 1
             date = datetime.date.today()
             uuid = duuid()
-            cur.execute("INSERT INTO region (id,changedate, creationdate, name, uuid, epidcode, archived)"
-                        "VALUES (%s,%s, %s, %s, %s, %s, %s)",
-                        [_id, date, date, region, uuid, 'REG', False]
-                        )
+            cur.execute(
+                "INSERT INTO region"
+                " (id,changedate, creationdate, name, uuid, epidcode, archived)"
+                " VALUES (%s,%s, %s, %s, %s, %s, %s)",
+                [_id, date, date, region, uuid, 'REG', False])
             return _id, uuid

src/importer/generator/user.py

@@ -1,12 +1,26 @@
+import os
+
 from sormas import UserReferenceDto
 
 from generator.utils import sormas_db_connect
 
 
 # todo make this deterministic
 def surv_sup_user_ref():
-    with sormas_db_connect() as conn:
-        with conn.cursor() as cur:
-            cur.execute("SELECT uuid FROM users WHERE firstname = 'Surveillance' AND lastname = 'Supervisor'")
-            uuid = cur.fetchone()[0]
-            return UserReferenceDto(uuid=uuid)
+    sup_uuid = os.getenv('SUPERVISOR_UUID')
+    if sup_uuid:
+        return UserReferenceDto(uuid=sup_uuid)
+    else:
+        sup_first_name = os.getenv('SUPERVISOR_FIRSTNAME', 'Surveillance')
+        sup_last_name = os.getenv('SUPERVISOR_LASTNAME', 'Supervisor')
+
+        with sormas_db_connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(f"""
+                    SELECT uuid
+                      FROM users
+                     WHERE firstname = '{sup_first_name}'
+                       AND lastname = '{sup_last_name}'
+                """)
+                uuid = cur.fetchone()[0]
+                return UserReferenceDto(uuid=uuid)

src/importer/main.py

@@ -5,6 +5,8 @@
 
 from performance.evaluate import analyze_performance
 from universe.world import World
+from sormas import Disease
+
 
 # noinspection PyArgumentList
 logging.basicConfig(
@@ -16,22 +18,20 @@
 @click.command()
 @click.option('--case-count', default=1, help='Number of cases you want to import.')
 @click.option('--event-count', default=1, help='Number of events you want to import.')
-def main(case_count, event_count):
+@click.option('--region', default='Niedersachsen', help='Region location.')
+@click.option('--district', default='Wolfsburg', help='District location.')
+@click.option('--disease', default=Disease.CORONAVIRUS, help='Type of disease.')
+def main(case_count, event_count, region, district, disease):
     logging.info(f'Importing {case_count} cases')
     logging.info(f'Importing {event_count} events')
     # Set everything up
     # Create our world where we simulate a pandemic. This is our playground.
     # Set a beginning for our world
 
-    world = World()
+    world = World(disease=disease)
 
-    # Populate default entities in our world
-    # Counties of interest
-    lower_saxony = 'Niedersachsen'
-    world.add_region(lower_saxony)
-    world.add_district('Braunschweig', lower_saxony)
-    world.add_district('Salzgitter', lower_saxony)
-    world.add_district('Wolfsburg', lower_saxony)
+    # # Populate default entities in our world
+    world.add_district(district, region)
 
     world.pre_populate_cases_and_contacts(n=case_count)
 
@@ -46,6 +46,8 @@ def main(case_count, event_count):
     if os.environ.get('ANALYZE_PERFORMANCE', 'False').upper() == 'TRUE':
         analyze_performance()
 
+    logging.info('Done')
+
 
 if __name__ == '__main__':
     main()