Merge pull request #18 from devgateway/develop

develop into main

.env

@@ -8,5 +8,6 @@ INFOBIP_KEY=452789534289
 SMSGATEWAY_KEY=98459432897
 SMTP_PORT=25
 SMTP_HOST=host.docker.internal
-RUN_MODE=admin
-SERVER_URL=http://localhost:8090
+SERVER_URL=http://localhost:8090
+SPRING_DEVTOOLS_RESTART_ENABLED=false
+DISABLE_EMAIL_SENDING=false

.github/workflows/docker-image.yml

@@ -0,0 +1,18 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    branches: [ "develop" ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build the Docker image
+      run: docker compose build

Dockerfile

@@ -1,10 +1,12 @@
-FROM maven:3.6-openjdk-8 as compiler
+FROM maven:3.8-openjdk-17 as compiler
 WORKDIR /tmp/build
+RUN microdnf install zip
 COPY forms/pom.xml forms/pom.xml
 COPY persistence/pom.xml persistence/pom.xml
 COPY persistence-mongodb/pom.xml persistence-mongodb/pom.xml
 COPY web/pom.xml web/pom.xml
 COPY pom.xml .
+ENV MAVEN_OPTS="-XX:-TieredCompilation -XX:TieredStopAtLevel=1"
 #we run dependency:go-offline on all pom.xml files copied, to get the dependencies first
 RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
   find . -name pom.xml \
@@ -13,14 +15,14 @@ RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
 COPY . .
 #we compile the code then we explode the fat jar. This is useful to create a reusable layer and save image space/compile time
 RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
-  mvn -T 4 clean package -DskipTests -Dcheckstyle.skip -Dmaven.gitcommitid.skip=true
+  mvn -T 1C clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dmaven.gitcommitid.skip=true
 RUN mkdir -p forms/target/deps \
     && cd forms/target/deps \
     && unzip -qo '../*.jar' || \
     ( e=$? && if [ $e -ne 1 ]; then exit $e; fi ) \
     && rm -f ../*.*
 
-FROM openjdk:8-jdk-slim as prod
+FROM openjdk:17-jdk-slim as prod
 WORKDIR /opt/app
 RUN apt-get update && apt-get install -y fontconfig libfreetype6 && rm -rf /var/lib/apt/lists/*
 #we copy artifacts from exploded jar, one by one, each COPY command will create a separate docker layer
@@ -32,4 +34,19 @@ COPY --chmod=0755 entrypoint.sh .
 EXPOSE 8090
 ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.0/wait /wait
 RUN chmod +x /wait
-CMD /wait && /opt/app/entrypoint.sh
+CMD /wait && /opt/app/entrypoint.sh admin
+
+FROM openjdk:17-jdk-slim as dev
+WORKDIR /opt/app
+RUN apt-get update && apt-get install -y fontconfig libfreetype6 && rm -rf /var/lib/apt/lists/*
+COPY --from=compiler /tmp/build/forms/target/deps/BOOT-INF/lib lib
+RUN rm -f lib/persistence*-SNAPSHOT.jar
+RUN rm -f lib/web*-SNAPSHOT.jar
+COPY --chmod=0755 entrypoint.sh .
+EXPOSE 8090
+EXPOSE 8000
+ENV JAVA_TOOL_OPTIONS -agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n
+#we use the docker-compose-wait script to wait for port 5432. postgres may be busy importing the db if db is not present
+ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.0/wait /wait
+RUN chmod +x /wait
+CMD /wait && /opt/app/entrypoint.sh admin-dev

README.md

@@ -2,6 +2,8 @@
 
 [![Build Status](https://travis-ci.org/devgateway/oc-explorer.svg?branch=develop)](https://travis-ci.org/devgateway/oc-explorer)
 
+[![Docker Image CI](https://github.com/devgateway/ocportal-nandi/actions/workflows/docker-image.yml/badge.svg?branch=develop)](https://github.com/devgateway/ocportal-nandi/actions/workflows/docker-image.yml)
+
 ## Presentation
 
 OCE is a project that allows importing the public procurement data, available in the common MS Excel format, into a native [Open Contracting Data Standard (OCDS)](http://standard.open-contracting.org/) NoSQL storage, and then run visual data analytics (display a *live* dashboard with charts, maps and data tables as well as custom comparison charts). Since the data is natively stored in the OCDS format, it can be readily exported in this format without any transformation required, and with great throughput.

docker-compose-dev.yml

@@ -3,12 +3,12 @@ version: '3.8'
 services:
 
   app:
-    image: ocportal-nandi/admin/prod:${TAG-local}
-    restart: always
+    image: ocportal-nandi/admin/dev:${TAG-local}
+    restart: 'no'
     extra_hosts:
       - "host.docker.internal:host-gateway" #required so docker connects to host interface (this is for sending emails)
     build:
-        target: prod
+        target: dev
     depends_on:
       - db
       - mongo
@@ -17,6 +17,13 @@ services:
       - "8000:8000"
     env_file:
       - .env
+    #entrypoint: /bin/sh -c "while sleep 1000; do :; done"
+    volumes:
+      - ./forms/target/classes:/opt/app/forms/classes
+      - ./persistence/target/classes:/opt/app/persistence/classes
+      - ./persistence-mongodb/target/classes:/opt/app/persistence-mongodb/classes
+      - ./web/target/classes:/opt/app/web/classes
+      - ./ui:/opt/app/ui
 
   db:
     image: postgis/postgis:11-3.3
@@ -39,25 +46,25 @@ services:
     env_file:
       - .env
 
-  pgbackups:
-    image: prodrigestivill/postgres-backup-local
-    restart: 'no'
-    volumes:
-      - /opt/pgbackups:/backups
-    env_file:
-      - .env
-    depends_on:
-      - db
-    environment:
-      - POSTGRES_HOST=db
-      - POSTGRES_DB=ocportal
-      - POSTGRES_USER=$POSTGRES_USER
-      - POSTGRES_PASSWORD=$POSTGRES_PASSWORD
-      - SCHEDULE=@daily
-      - BACKUP_KEEP_DAYS=30
-      - BACKUP_KEEP_WEEKS=12
-      - BACKUP_KEEP_MONTHS=24
-      - HEALTHCHECK_PORT=5432
+#  pgbackups:
+#    image: prodrigestivill/postgres-backup-local
+#    restart: 'no'
+#    volumes:
+#      - /opt/pgbackups:/backups
+#    env_file:
+#      - .env
+#    depends_on:
+#      - db
+#    environment:
+#      - POSTGRES_HOST=db
+#      - POSTGRES_DB=ocportal
+#      - POSTGRES_USER=$POSTGRES_USER
+#      - POSTGRES_PASSWORD=$POSTGRES_PASSWORD
+#      - SCHEDULE=@daily
+#      - BACKUP_KEEP_DAYS=30
+#      - BACKUP_KEEP_WEEKS=12
+#      - BACKUP_KEEP_MONTHS=24
+#      - HEALTHCHECK_PORT=5432
 
 volumes:
     pgdata:

entrypoint.sh

@@ -7,8 +7,20 @@ COMMON_JAVA_ARGS="$(tr '\n' ' ' <<-EOF
   -Dwicket.configuration=deployment
   -Dfile.encoding=UTF-8
   -DserverURL=$SERVER_URL
+  -DdisableEmailSending=$DISABLE_EMAIL_SENDING
+  -Dspring.devtools.restart.enabled=$SPRING_DEVTOOLS_RESTART_ENABLED
   -Xms512m
-  -Xmx4096m
+  -Xmx6144m
+  --add-opens=java.naming/javax.naming=ALL-UNNAMED
+  --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+  --add-opens=java.base/java.lang.ref=ALL-UNNAMED
+  --add-opens=java.base/java.lang=ALL-UNNAMED
+  --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED
+  --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+  --add-opens=java.base/java.io=ALL-UNNAMED
+  --add-opens=java.base/java.security=ALL-UNNAMED
+  --add-opens=java.base/java.util=ALL-UNNAMED
+  --add-opens=java.management/javax.management=ALL-UNNAMED
   -Dspring.mail.port=$SMTP_PORT
   -Dspring.mail.host=$SMTP_HOST
   -XX:MaxMetaspaceSize=512m
@@ -26,7 +38,7 @@ COMMON_JAVA_ARGS="$(tr '\n' ' ' <<-EOF
 EOF
 )"
 
-case "$RUN_MODE" in
+case "$1" in
 admin)
   JAVA_ARGS="${COMMON_JAVA_ARGS} $(tr '\n' ' ' <<-EOF
 			-cp .:lib/*
@@ -35,13 +47,19 @@ admin)
   )"
   exec java $JAVA_ARGS $@
   ;;
-admin-debug)
+admin-dev)
   JAVA_ARGS="${COMMON_JAVA_ARGS} $(tr '\n' ' ' <<-EOF
-			-cp .:lib/*
+			-Dspring.devtools.restart.additional-exclude=logs/**,META-INF/**,ehcache-disk-store/**
+			-Dspring.devtools.restart.poll-interval=3s
+			-Dspring.devtools.restart.quiet-period=2s
+			-XX:+TieredCompilation
+			-XX:TieredStopAtLevel=1
+			-noverify
+			-cp forms/classes:persistence-mongodb/classes:web/classes:persistence/classes:lib/*
 			org.devgateway.toolkit.forms.wicket.FormsWebApplication
 		EOF
   )"
-  exec java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 $JAVA_ARGS $@
+  exec java $JAVA_ARGS $@
   ;;
 *)
   exec "$@"

forms/pom.xml

@@ -28,16 +28,16 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <java.version>1.8</java.version>
-        <wicket.version>8.9.0</wicket.version>
-        <wicketstuff.version>8.9.0</wicketstuff.version>
-        <wicket.bootstrap.version>2.0.12</wicket.bootstrap.version>
+        <java.version>17</java.version>
+        <wicket.version>8.15.0</wicket.version>
+        <wicketstuff.version>8.15.0</wicketstuff.version>
+        <wicket.bootstrap.version>2.0.15</wicket.bootstrap.version>
         <zt.zip.version>1.13</zt.zip.version>
-        <wicket.webjars.version>2.0.18</wicket.webjars.version>
+        <wicket.webjars.version>2.0.20</wicket.webjars.version>
         <closure.compiler.version>v20200406</closure.compiler.version>
-        <cglib.version>3.2.12</cglib.version>
         <yuicompressor.version>2.4.8</yuicompressor.version>
-        <javamelody.version>1.86.0</javamelody.version>
+        <javamelody.version>1.95.0</javamelody.version>
+        <exclude.devtools>false</exclude.devtools>
     </properties>
 
     <dependencies>
@@ -167,12 +167,6 @@
             </exclusions>
         </dependency>
 
-        <dependency>
-            <groupId>cglib</groupId>
-            <artifactId>cglib</artifactId>
-            <version>${cglib.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>com.google.javascript</groupId>
             <artifactId>closure-compiler-unshaded</artifactId>
@@ -360,6 +354,7 @@
                 <artifactId>spring-boot-maven-plugin</artifactId>
                 <configuration>
                     <executable>true</executable>
+                    <excludeDevtools>${exclude.devtools}</excludeDevtools>
                 </configuration>
                 <version>${spring.boot.version}</version>
                 <executions>

forms/src/main/java/org/devgateway/toolkit/forms/service/SessionFinderService.java

@@ -14,6 +14,7 @@
 import nl.dries.wicket.hibernate.dozer.DozerModel;
 import nl.dries.wicket.hibernate.dozer.SessionFinder;
 import org.hibernate.Session;
+import org.hibernate.engine.spi.SessionImplementor;
 import org.springframework.stereotype.Component;
 
 import javax.persistence.EntityManager;
@@ -41,7 +42,7 @@ public class SessionFinderService implements SessionFinder {
      */
     @Override
     public Session getHibernateSession(final Class<?> clazz) {
-        return em.unwrap(Session.class);
+        return em.unwrap(SessionImplementor.class);
     }
 
 }

forms/src/main/resources/argfile.txt

@@ -0,0 +1,10 @@
+--add-opens=java.naming/javax.naming=ALL-UNNAMED
+--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens=java.base/java.lang.ref=ALL-UNNAMED
+--add-opens=java.base/java.lang=ALL-UNNAMED
+--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED
+--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+--add-opens=java.base/java.io=ALL-UNNAMED
+--add-opens=java.base/java.security=ALL-UNNAMED
+--add-opens=java.base/java.util=ALL-UNNAMED
+--add-opens=java.management/javax.management=ALL-UNNAMED

persistence-mongodb/pom.xml

@@ -23,10 +23,9 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <start-class>org.devgateway.toolkit.persistence.spring.PersistenceApplication</start-class>
-        <java.version>1.8</java.version>
+        <java.version>17</java.version>
         <json.patch.version>1.13</json.patch.version>
         <commons-digester3.version>3.2</commons-digester3.version>
-        <reflections.version>0.9.11</reflections.version>
     </properties>
 
     <parent>

persistence-mongodb/src/main/java/org/devgateway/ocds/persistence/mongo/constants/MongoConstants.java

@@ -21,7 +21,7 @@ private MongoConstants() {
 
 
     public static final class OCDSSchemes {
-        public static final String X_KE_INTERNAL_SCHEMA = "X-KE-OCELGEYO-MARAKWET";
+        public static final String X_KE_INTERNAL_SCHEMA = "X-KE-NANDI";
         public static final String X_KE_IFMIS = "x_KE-IFMIS";
         public static final String KE_IFMIS = "KE-IFMIS";
         public static final String UNCEFACT = "UNCEFACT";

persistence-mongodb/src/main/java/org/devgateway/toolkit/persistence/mongo/spring/AbstractMongoDatabaseConfiguration.java

@@ -33,9 +33,8 @@ public void createMandatoryImportIndexes() {
                 new Index().on("roles", Direction.ASC));
         getTemplate().indexOps(Organization.class).ensureIndex(new Index().on("name", Direction.ASC).unique());
         getTemplate().indexOps(DefaultLocation.class).ensureIndex(new Index().on("description", Direction.ASC));
-        getTemplate().indexOps("fs.files").ensureIndex(new Index().on("md5", Direction.ASC));
         getTemplate().indexOps("fs.files").ensureIndex(new CompoundIndexDefinition(
-                new Document("filename", 1).append("md5", 1)).unique());
+                new Document("filename", 1).append("metadata.md5", 1)).unique());
         getLogger().info("Added mandatory Mongo indexes");
     }
 

persistence-mongodb/src/main/java/org/devgateway/toolkit/persistence/mongo/spring/MongoPersistenceApplication.java

@@ -22,7 +22,7 @@
 import org.springframework.core.convert.converter.Converter;
 import org.springframework.data.convert.ReadingConverter;
 import org.springframework.data.convert.WritingConverter;
-import org.springframework.data.mongodb.core.convert.CustomConversions;
+import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
 import org.springframework.data.mongodb.core.geo.GeoJsonPoint;
 import org.springframework.util.Assert;
 import org.springframework.util.ObjectUtils;
@@ -46,7 +46,7 @@
  */
 @SpringBootApplication
 @ComponentScan(value = "org.devgateway", excludeFilters = @ComponentScan.Filter(
-        type = FilterType.ASPECTJ, pattern = "org.devgateway.jocds.ValidatorConfiguration" ))
+        type = FilterType.ASPECTJ, pattern = "org.devgateway.jocds.ValidatorConfiguration"))
 @PropertySource("classpath:/org/devgateway/toolkit/persistence/mongo/application.properties")
 @EnableCaching
 public class MongoPersistenceApplication {
@@ -56,12 +56,12 @@ public static void main(final String[] args) {
     }
 
     @Bean
-    public CustomConversions customConversions() {
-        return new CustomConversions(Arrays
-                .asList(new Object[]{BigDecimalToDoubleConverter.INSTANCE, DoubleToBigDecimalConverter.INSTANCE,
+    public MongoCustomConversions customConversions() {
+        return new MongoCustomConversions(Arrays
+                .asList(BigDecimalToDoubleConverter.INSTANCE, DoubleToBigDecimalConverter.INSTANCE,
                         DbObjectToGeoJsonPointConverter.INSTANCE,
                         ZonedDateTimeReadConverter.INSTANCE, ZonedDateTimeWriteConverter.INSTANCE,
-                        URIToStringConverter.INSTANCE}));
+                        URIToStringConverter.INSTANCE));
     }
 
     public enum URIToStringConverter implements Converter<URI, String> {

persistence-mongodb/src/main/java/org/devgateway/toolkit/persistence/mongo/spring/MongoTemplateConfig.java

@@ -1,16 +1,15 @@
 package org.devgateway.toolkit.persistence.mongo.spring;
 
-import com.mongodb.MongoClientURI;
 import org.springframework.beans.factory.annotation.Autowire;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.mongo.MongoProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
-import org.springframework.data.mongodb.core.convert.CustomConversions;
+import org.springframework.data.mongodb.core.SimpleMongoClientDbFactory;
 import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
 
 /**
  * Created by mpostelnicu on 6/12/17.
@@ -25,11 +24,11 @@ public class MongoTemplateConfig {
     private MongoProperties properties;
 
     @Autowired
-    private CustomConversions customConversions;
+    private MongoCustomConversions customConversions;
 
     @Bean(autowire = Autowire.BY_NAME, name = "mongoTemplate")
     public MongoTemplate mongoTemplate() throws Exception {
-        MongoTemplate template = new MongoTemplate(new SimpleMongoDbFactory(new MongoClientURI(properties.getUri())));
+        MongoTemplate template = new MongoTemplate(new SimpleMongoClientDbFactory(properties.getUri()));
         ((MappingMongoConverter) template.getConverter()).setCustomConversions(customConversions);
 
         // CALL THIS MANULLY, so that all the default converters will be registered!