Merge pull request #103 from ProvideQ/release/0.4.0

ProvideQ Release 0.4.0

.github/workflows/ci-cd-develop.yml

@@ -28,12 +28,12 @@ jobs:
           echo "$GAMS_PATH" >> "$GITHUB_PATH"
           echo "GMSPYTHONLIB=/opt/conda/envs/gams/lib/libpython3.10.so" >> "$GITHUB_ENV"
       - name: Install solver dependencies
-        run: ./scripts/install-solver-dependencies.sh
+        run: ./scripts/ci-setup-solvers.sh
       - name: Run checks
         shell: bash # required for the source command
         run: |
           source /opt/conda/bin/activate gams
-          ./gradlew check
+          ./gradlew check --info
       - name: Archive test reports
         if: always()
         uses: actions/upload-artifact@v3

.gitignore

@@ -1,9 +1,12 @@
 HELP.md
+*.DS_Store
 .gradle
 build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
+### Solution Files from Vehicle Routing Solver:
+*.sol
 
 ### STS ###
 .apt_generated
@@ -13,7 +16,8 @@ build/
 .settings
 .springBeans
 .sts4-cache
-bin/
+bin/main
+bin/test
 !**/src/main/**/bin/
 !**/src/test/**/bin/
 
@@ -37,6 +41,9 @@ out/
 .vscode/
 
 
+### virtual envs and python ###
+venv/
+__pycache__/
 
 ### ProvideQ toolbox-server ###
 # input, output and temporary files for solving problems with our solvers
@@ -48,5 +55,5 @@ gamslice.txt
 # Listing files compiled from our GAMS scripts
 *.lst
 *.op2
-gams/**/225a
+solvers/gams/**/225a
 *.out

Dockerfile

@@ -48,10 +48,9 @@ COPY --from=builder /app/build/jre /opt/java
 ENV PATH="${PATH}:/opt/java/bin"
 
 # Install the toolbox server and its solver scripts
-COPY gams gams
-COPY qiskit qiskit
-COPY cirq cirq
-RUN scripts/install-solver-dependencies.sh
+COPY solvers solvers
+COPY demonstrators demonstrators
+RUN scripts/ci-setup-solvers.sh
 COPY --from=builder /app/build/libs/toolbox-server-*.jar toolbox-server.jar
 
 # Run the toolbox server on dokku's default port

README.md

@@ -7,36 +7,51 @@ A web-based user interface can be found in our
 [ProvideQ/ProvideQ repository](https://github.com/ProvideQ/ProvideQ).
 
 ## Development setup
-1. Install Java 17 (check with `java -version`)
+1. Install Java 17 or newer (check with `java -version`)
 2. Clone this repository
-3. Install a python env that works with GAMS (skip this step if you don't need GAMS)
-   1. Install GAMS.
-   2. Install miniconda (or anaconda, if you prefer that):
-      https://conda.io/projects/conda/en/stable/user-guide/install/index.html
-   3. Create a GAMS conda environment: `conda create --name gams python=3.10 --yes`
-   4. Activate your conda environment: `conda activate gams`.
-   5. Make GAMS use that python environment by setting the `GMSPYTHONLIB=<path-to-conda>/envs/gams/lib/libpython3.10.so`
-      environment variable.
-   6. Install GAMS packages to the GAMS conda env:
-      `pip install gams[core,connect] --find-links <path-to-gams>/api/python/bdist`
-      * If you get an error building `psycopg2`, try to install these postgres packages:
-        `sudo apt-get install postgresql libpq-dev` and run the `pip install ...` command again
-   7. Install the python dependencies we use in our python packages: `pip install -r gams/requirements.txt`
-4. Install solver dependencies (skip this step if you don't want to use the solvers):
-   * Note that these dependencies must be installed to the gams conda env if you want to use GAMS and other solvers from
-     the same toolbox installation!
-   1. Install GAMS solver dependencies: `pip install -r gams/requirements.txt`
-   2. Install Qiskit solver dependencies: `pip install -r qiskit/requirements.txt`
-   3. Install Cirq solver dependencies: `pip install -r cirq/requirements.txt`
-5. Run the server using `./gradlew bootRun`
+3. [Optional, Solver Installation - install the Solvers that you want/need]
+   1. Python-based Solvers (Qiskit, Cirq, Dwave, Qrisp)
+      1. Install Python
+      2. Install Python dependencies by running `python scripts/install-python-dependencies.py` <br>
+         Alternatively, run `pip install -r requirements.txt` on all requirement.txt files in the /solvers directory.
+   2. Compiled Solvers (e.g. used for VRP and TSP)
+      1. Solvers implemented in compiled languages must be executed via binaries that are compiled for your operating system. For those types of solvers we usually include pre-compiled binaries for windows, mac (only arm), and unix.
+      * General Note: Solvers might be programmed in different languages. E.g., LKH-3 is implemented in C. Make sure that the solver-specific language is installed on your system.
+      2. In case the pre-compiled versions do not work on your machine: re-compile them:
+      * LKH-3:
+        1. Build LKH-3 using the offical guide: http://webhotel4.ruc.dk/~keld/research/LKH-3/
+        2. Put the build binary in `solvers/lkh/bin`, replace the binary that matches your OS.
+      * VRP-Pipeline (used for K-means, Two Phase Clustering, VRP to QUBO convertion):
+        1. Install Rust: https://www.rust-lang.org/tools/install
+        2. Install a specific Rust nightly build (needed cause the solver uses experimental features): `rustup install nightly-2023-07-01`
+        3. Check how the nightly build is called on your machine (this is shown when running the install command, on Mac it is called *nightly-2023-07-01-aarch64-apple-darwin*)
+        4. Set the nightly build as default: `rustup default nightly-2023-07-01(... specific version name on machine)`
+        5. Download source code of the VRP-Pipeline: https://github.com/ProvideQ/hybrid-vrp-solver
+        6. build the source code using `cargo build`
+        7. Put the build binary in `solvers/berger-vrp/bin`, replace the binary that matches your OS.
+   3. GAMS (multiple solvers are build on this):
+      1. Install a python env that works with GAMS (skip this step if you don't need GAMS)
+      2. Install GAMS. (https://www.gams.com/download/)
+      3. Install miniconda (or anaconda, if you prefer that):
+         https://conda.io/projects/conda/en/stable/user-guide/install/index.html
+      4. Create a GAMS conda environment: `conda create --name gams python=3.10 --yes`
+      5. Activate your conda environment: `conda activate gams`.
+      6. Make GAMS use that python environment by setting the `GMSPYTHONLIB=<path-to-conda>/envs/gams/lib/libpython3.10.so`
+            environment variable.
+      7. Install GAMS packages to the GAMS conda env:
+            `pip install gams[core,connect] --find-links <path-to-gams>/api/python/bdist`
+            * If you get an error building `psycopg2`, try to install these postgres packages:
+              `sudo apt-get install postgresql libpq-dev` and run the `pip install ...` command again
+      8. Install the python dependencies we use in our python packages: `pip install -r gams/requirements.txt`
+4. Run the server using `./gradlew bootRun`
 
 ## Deployment
 This repository is designed to be deployed with [Dokku](https://dokku.com/), but you can also run 
 the Java application directly or inside a docker container (`Dockerfile` is included!).
 The docker container can be built and run as follows:
 ```shell
 # we assume that you have a gamslice.txt file in this directory containing a valid GAMS license (typically 6 lines)
-docker build --tag provideq-toolbox-backend --build-arg GAMS_LICENSE=$(base64 -w 0 ./gamslice.txt) .
+docker build --tag provideq-toolbox-backend --build-arg GAMS_LICENSE=$(base64 --wrap=0 ./gamslice.txt) .
 docker run --publish 8080:5000 provideq-toolbox-backend
 ```
 
@@ -54,7 +69,33 @@ docker run --publish 8080:5000 provideq-toolbox-backend
    merge it into the develop branch (`git checkout develop && git pull && git merge main`)
    and push it (`git push`).
 
+## CI / CD
+We're using GitHub Actions to automate the execution of our validation tools and the deployment of our software.
+To use this, enable GitHub Actions and configure the following secrets in the GitHub repository settings:
+
+* `GAMS_LICENSE`:
+  1. Get a license for GAMS and put it in a `gamslice.txt` file.
+     A free [demo or community license](https://www.gams.com/try_gams/) should be sufficient.
+  2. Convert the license text to base64 using this command:
+     ```shell
+     base64 --wrap=0 ./gamslice.txt
+     ```
+  3. Copy the printed base64 string to the `GAMS_LICENSE` secret in your GitHub repository.
+* `DOKKU_SERVER_ADDRESS`:
+  1. Install [Dokku](https://dokku.com/) on your deployment server.
+     Make sure it can be reached from GitHub Actions!
+  2. Set the `DOKKU_SERVER_ADDRESS` secret to the address of your server.
+
+     *Tip: If you use an IP address, the IP address will be obfuscated in the Actions logs.
+     Counterintuitively, using a domain name as an address might leak your IP address due to SSH address resolution
+     logs.*
+* `DOKKU_DEPLOYMENT_KEY`:
+  1. Generate an ssh key and register it as a dokku user.
+  2. Make sure this user has push access the `toolbox-frontend`, `toolbox-backend`, `toolbox-frontend-staging` and
+     `toolbox-backend-staging` apps.
+  3. Set `DOKKU_DEPLOYMENT_KEY` to the private key generated in step i.
+
 ## License
 Copyright (c) 2022 - 2023 ProvideQ
 
-This project is available under the [MIT License](./LICENSE).
+This project is available under the [MIT License](./LICENSE).

build.gradle

@@ -6,7 +6,7 @@ plugins {
 }
 
 group = 'edu.kit.provideq'
-version = '0.3.0'
+version = '0.4.0'
 sourceCompatibility = '17'
 
 repositories {
@@ -20,18 +20,29 @@ dependencies {
     implementation 'com.bpodgursky:jbool_expressions:1.24'
     implementation files('lib/de.ovgu.featureide.lib.fm-v3.9.1.jar', 'lib/uvl-parser.jar')
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
+    testImplementation 'org.mockito:mockito-core:5.+'
 }
 
 jar {
     // avoid creation of plain jar, spring boot will create a fat jar anyways
     enabled = false
 }
 
+// inject JUnit into test task
 tasks.named('test') {
     useJUnitPlatform()
 }
+
+// set Java language level
 targetCompatibility = JavaVersion.VERSION_17
 
+tasks.withType(JavaCompile).configureEach {
+    // treat "unchecked" warnings as errors
+    options.compilerArgs << '-Xlint:unchecked'
+    options.compilerArgs << '-Xlint:deprecation'
+    options.compilerArgs << '-Werror'
+}
+
 checkstyle {
     toolVersion = '10.12.0'
 }

demonstrators/cplex/mip-solver/mip-solver.py

@@ -0,0 +1,82 @@
+import cplex
+import random
+import time
+import matplotlib.pyplot as plt
+import numpy as np
+import sys
+from io import StringIO
+
+if len(sys.argv) != 4:
+    raise TypeError('This script expects exactly 3 arguments: max number of variables, step size, and number of repetitons')
+
+maxNbVars = int(sys.argv[1])
+stepSize = int(sys.argv[2])
+repetitions = int(sys.argv[3])
+
+def generatePlot(table,):
+    """ generates plot with table elements (x,y)
+    table: elements are tuples first tuple is title of axes
+    """
+    xvalues = [ a for (a,_) in table]
+    yvalues = [ a for (_,a) in table]
+    xlabel = xvalues.pop(0)
+    ylabel = yvalues.pop(0)    
+
+    regression = np.poly1d(np.polyfit(xvalues, yvalues, 3))
+    plt.plot(xvalues, regression(xvalues))
+
+    plt.plot(xvalues, yvalues, 'o')
+    plt.ylabel(ylabel=ylabel)
+    plt.xlabel(xlabel=xlabel)
+
+    plt.title(ylabel + " for " + xlabel)
+
+    stringIo = StringIO()
+    plt.savefig(stringIo, format='svg')
+    plt.close()
+
+    print(stringIo.getvalue())
+
+def generateProblem(numberVars):
+    """ returns random MIP Problem
+    numberVars: the number if variables and constraints
+    """
+    pb = cplex.Cplex()
+    pb.set_problem_type(pb.problem_type.MILP)
+
+    pb.objective.set_sense(pb.objective.sense.maximize)
+
+    obj = [1 for _ in range(0,numberVars)]
+    ub = [random.randint(1,100) for _ in range(0,numberVars)]
+    lb = [ -i for i in ub]
+    pb.variables.add(obj=obj, lb=lb, ub=ub)
+
+    senses=['L' for _ in range(0,numberVars)]
+    rhs = [random.randint(1,10) for _ in range(0,numberVars)]
+
+    lin_expr = [cplex.SparsePair(ind = [i for i in range(0,numberVars)], val = [random.choice([-1,1]) * round(random.randint(1, 10)) for _ in range(0,numberVars)]) for _ in range(0,numberVars)]
+    pb.linear_constraints.add(senses=senses, rhs=rhs, lin_expr=lin_expr)
+
+    #print(pb.write_as_string())
+    return pb
+
+# The following experiment generates MIP problems and solves them with CPLEX.
+# It calculates the time taken by the solver for rising numbers of variables and shows results as plot.
+
+results = [("number of variables","time (sec)")]
+
+for i in [j*stepSize for j in range(0, maxNbVars // stepSize)]:
+    totalTime = 0
+    for _ in (0,repetitions):
+        problem = generateProblem(i)
+        problem.set_results_stream(None)
+        problem.set_log_stream(None)
+
+        start = time.perf_counter()
+        problem.solve()
+        end = time.perf_counter()
+        totalTime += end - start
+
+        results.append((i,totalTime))
+
+generatePlot(results)

demonstrators/cplex/requirements.txt

@@ -0,0 +1,3 @@
+cplex
+matplotlib
+numpy

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

scripts/ci-setup-solvers.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# this script is made for the CI-pipeline, do not use this to install dependencies on your private machine!
+
+# exit on error
+set -e
+
+# get base directory of the repository
+REPO_DIR=$(dirname "$(dirname "$(readlink -f "$0")")")
+
+# make sure to install to the gams conda env
+source /opt/conda/bin/activate gams
+
+# install solver dependencies / quantum frameworks and python wrappers
+python3 "$REPO_DIR/scripts/install-python-dependencies.py"
+

scripts/install-python-dependencies.py

@@ -0,0 +1,19 @@
+import os
+import subprocess
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+base_dirs = [
+    os.path.join(script_dir, '..', 'solvers'),
+    os.path.join(script_dir, '..', 'demonstrators')
+]
+
+for base_dir in base_dirs:
+    for root, dirs, files in os.walk(base_dir):
+        # Check if 'requirements.txt' is in the current directory
+        req_file = os.path.join(root, 'requirements.txt')
+        if os.path.exists(req_file):
+            print(f"Found requirements.txt in {root}, installing dependencies...")
+            try:
+                subprocess.run(['pip', 'install', '-r', req_file], check=True) # single thread to avoid dep. errors
+            except subprocess.CalledProcessError as e:
+                print(f"Error installing requirements for {root}: {e}")

solvers/cirq/requirements.txt

@@ -0,0 +1,3 @@
+# Cirq MaxCut
+matplotlib
+cirq==1.5.0.dev20241009204224