From d5945727a1a94450ff73810c5d2ad8c87aed915b Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Tue, 8 Apr 2025 17:15:31 +0100 Subject: [PATCH 1/2] upgrade neo4j to 2025.03.0 --- README.md | 3 +-- dataload/Dockerfile | 6 +++--- dataload/README.md | 5 ++--- dataload/create_neo4j_indexes.sh | 2 +- dataload/load_into_neo4j.sh | 5 ++--- dev-testing/load_test_into_neo4j.sh | 5 ++--- docker-compose.yml | 2 +- frontend/src/pages/Downloads.tsx | 2 +- k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml | 2 +- k8chart/ols4/templates/ols4-neo4j-deployment.yaml | 2 +- 10 files changed, 15 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index eff483525..6133f2359 100644 --- a/README.md +++ b/README.md @@ -241,12 +241,11 @@ First, make sure the configuration files (that determine which ontologies to loa Run Neo4j `import` command: - ./neo4j-admin import \ + ./neo4j-admin database import full \ --ignore-empty-strings=true \ --legacy-style-quoting=false \ --array-delimiter="|" \ --multiline-fields=true \ - --database=neo4j \ --read-buffer-size=134217728 \ $(/make_csv_import_cmd.sh) diff --git a/dataload/Dockerfile b/dataload/Dockerfile index 49bad6398..8f3f1e648 100644 --- a/dataload/Dockerfile +++ b/dataload/Dockerfile @@ -3,7 +3,7 @@ # This image, once built, should allow any arbitrary config to be loaded and output neo4j/solr datafiles -FROM maven:3.8-jdk-11 +FROM maven:3.8-eclipse-temurin-21 RUN addgroup --system --gid 1000 ols && adduser --system --uid 1000 --ingroup ols ols @@ -13,13 +13,13 @@ RUN addgroup --system --gid 1000 ols && adduser --system --uid 1000 --ingroup ol # and Solr servers are the standard images specified in docker-compose.yml # RUN mkdir /opt/neo4j && \ - curl https://dist.neo4j.org/neo4j-community-4.4.9-unix.tar.gz | tar -xz --strip-components=1 -C /opt/neo4j + curl https://dist.neo4j.org/neo4j-community-2025.03.0-unix.tar.gz | tar -xz --strip-components=1 -C /opt/neo4j RUN mkdir /opt/solr && \ curl https://archive.apache.org/dist/solr/solr/9.0.0/solr-9.0.0.tgz | tar -xz --strip-components=1 -C /opt/solr RUN echo "dbms.security.auth_enabled=false" >> /opt/neo4j/conf/neo4j.conf -RUN echo "dbms.jvm.additional=-Dorg.neo4j.kernel.impl.index.schema.GenericNativeIndexPopulator.blockBasedPopulation=true" >> /opt/neo4j/conf/neo4j.conf +RUN echo "server.jvm.additional=-Dorg.neo4j.kernel.impl.index.schema.GenericNativeIndexPopulator.blockBasedPopulation=true" >> /opt/neo4j/conf/neo4j.conf # Copy all the code for dataload into /opt/dataload and build the JARs # diff --git a/dataload/README.md b/dataload/README.md index 109a7858e..b0f94226a 100644 --- a/dataload/README.md +++ b/dataload/README.md @@ -25,14 +25,13 @@ You can now convert this huge JSON file to a CSV file ready for Neo4j, using jso ## Step 3: CSV to Neo4j -Now (after 5-10 mins) you should have a directory full of CSV files. These files are formatted especially for Neo4j. You can load them using `neo4j-admin import`, but you'll need to provide the filename of every single CSV file on the command line, which is boring, so included in this repo is a script called `make_csv_import_cmd.sh` that generates the command line for you. +Now (after 5-10 mins) you should have a directory full of CSV files. These files are formatted especially for Neo4j. You can load them using `neo4j-admin database import full`, but you'll need to provide the filename of every single CSV file on the command line, which is boring, so included in this repo is a script called `make_csv_import_cmd.sh` that generates the command line for you. - neo4j-admin import \ + neo4j-admin database import full \ --ignore-empty-strings=true \ --legacy-style-quoting=false \ --multiline-fields=true \ --array-delimiter="|" \ - --database=neo4j \ $(./make_csv_import_cmd.sh) Now you should have a Neo4j database ready to start! diff --git a/dataload/create_neo4j_indexes.sh b/dataload/create_neo4j_indexes.sh index 9ee4bce3b..df6b692c1 100755 --- a/dataload/create_neo4j_indexes.sh +++ b/dataload/create_neo4j_indexes.sh @@ -5,7 +5,7 @@ if [ $# == 0 ]; then exit 1 fi -$1/bin/neo4j start +$1/bin/neo4j start --verbose sleep 20 echo Creating neo4j indexes... diff --git a/dataload/load_into_neo4j.sh b/dataload/load_into_neo4j.sh index 21bc5b286..4daa707ae 100755 --- a/dataload/load_into_neo4j.sh +++ b/dataload/load_into_neo4j.sh @@ -8,14 +8,13 @@ fi rm -rf $1/data/databases/neo4j rm -rf $1/data/transactions/neo4j -$1/bin/neo4j-admin import \ +$1/bin/neo4j-admin database import full \ --ignore-empty-strings=true \ --legacy-style-quoting=false \ --multiline-fields=true \ --read-buffer-size=16777216 \ --array-delimiter="|" \ - --database=neo4j \ - --processors=16 \ + --threads=32 \ $(./make_csv_import_cmd.sh $2) diff --git a/dev-testing/load_test_into_neo4j.sh b/dev-testing/load_test_into_neo4j.sh index 1d811ac71..286f3e81c 100755 --- a/dev-testing/load_test_into_neo4j.sh +++ b/dev-testing/load_test_into_neo4j.sh @@ -5,12 +5,11 @@ if [ $# == 0 ]; then exit 1 fi -$NEO4J_HOME/bin/neo4j-admin import \ +$NEO4J_HOME/bin/neo4j-admin database import full \ --ignore-empty-strings=true \ --legacy-style-quoting=false \ --multiline-fields=true \ --array-delimiter="|" \ - --database=neo4j \ - --processors=2 \ + --threads=4 \ --read-buffer-size=134217728 \ $($OLS4_HOME/dev-testing/make_csv_import_cmd.sh $1) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b31bbf1a6..fb80e9bb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: ols4-dataload: condition: service_completed_successfully ols4-neo4j: - image: neo4j:4.4.9-community + image: neo4j:2025.03.0-community ports: - 7474:7474 - 7687:7687 diff --git a/frontend/src/pages/Downloads.tsx b/frontend/src/pages/Downloads.tsx index 48074d146..4c4a4ff60 100644 --- a/frontend/src/pages/Downloads.tsx +++ b/frontend/src/pages/Downloads.tsx @@ -98,7 +98,7 @@ const data: any[] = [ }, { description: - "The neo4j database generated from ontologies_linked.json by the OLS json2neo tool, after creating db indexes (Requires Neo4j community 4.4.9)", + "The neo4j database generated from ontologies_linked.json by the OLS json2neo tool, after creating db indexes (Requires Neo4j community 2025.03.0)", downloadLabel: "neo4j.tgz", downloadLink: "https://ftp.ebi.ac.uk/pub/databases/spot/ols/latest/neo4j.tgz", diff --git a/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml b/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml index 0260ce520..eab8dcfa4 100644 --- a/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml +++ b/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml @@ -29,7 +29,7 @@ spec: claimName: {{ .Release.Name }}-neo4j-pvc containers: - name: neo4j - image: ghcr.io/ebispot/ols4-neo4j:4.4.9-community + image: ghcr.io/ebispot/ols4-neo4j:2025.03.0-community volumeMounts: - mountPath: "/data" name: {{ .Release.Name }}-neo4j-storage diff --git a/k8chart/ols4/templates/ols4-neo4j-deployment.yaml b/k8chart/ols4/templates/ols4-neo4j-deployment.yaml index 859757d72..b69e6d2ec 100644 --- a/k8chart/ols4/templates/ols4-neo4j-deployment.yaml +++ b/k8chart/ols4/templates/ols4-neo4j-deployment.yaml @@ -30,7 +30,7 @@ spec: claimName: {{ .Release.Name }}-neo4j-pvc containers: - name: neo4j - image: ghcr.io/ebispot/ols4-neo4j:4.4.9-community + image: ghcr.io/ebispot/ols4-neo4j:2025.03.0-community volumeMounts: - mountPath: "/data" name: {{ .Release.Name }}-neo4j-storage From 7a080ce52db7c599033035ee7c1b9f7de523f9cc Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Tue, 8 Apr 2025 17:21:40 +0100 Subject: [PATCH 2/2] use official neo4j image in k8s --- k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml | 2 +- k8chart/ols4/templates/ols4-neo4j-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml b/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml index eab8dcfa4..814245611 100644 --- a/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml +++ b/k8chart-dev/ols4/templates/ols4-neo4j-deployment.yaml @@ -29,7 +29,7 @@ spec: claimName: {{ .Release.Name }}-neo4j-pvc containers: - name: neo4j - image: ghcr.io/ebispot/ols4-neo4j:2025.03.0-community + image: neo4j:2025.03.0-community volumeMounts: - mountPath: "/data" name: {{ .Release.Name }}-neo4j-storage diff --git a/k8chart/ols4/templates/ols4-neo4j-deployment.yaml b/k8chart/ols4/templates/ols4-neo4j-deployment.yaml index b69e6d2ec..fb0025d07 100644 --- a/k8chart/ols4/templates/ols4-neo4j-deployment.yaml +++ b/k8chart/ols4/templates/ols4-neo4j-deployment.yaml @@ -30,7 +30,7 @@ spec: claimName: {{ .Release.Name }}-neo4j-pvc containers: - name: neo4j - image: ghcr.io/ebispot/ols4-neo4j:2025.03.0-community + image: neo4j:2025.03.0-community volumeMounts: - mountPath: "/data" name: {{ .Release.Name }}-neo4j-storage