diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a19ae5c7..b3b68a62 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,16 +11,16 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Install metafacture-core-5.4.1-rc3 + - name: Install metafacture-core-5.5.1-rc1 run: | git clone https://github.com/metafacture/metafacture-core.git cd metafacture-core - git checkout metafacture-core-5.4.1-rc3 + git checkout metafacture-core-5.5.1-rc1 ./gradlew publishToMavenLocal cd .. - - name: Install metafacture-fix 0.4.0 + - name: Install metafacture-fix 0.5.1 run: | - git clone https://github.com/metafacture/metafacture-fix.git -b 0.4.0 + git clone https://github.com/metafacture/metafacture-fix.git -b 0.5.1 cd metafacture-fix ./gradlew publishToMavenLocal cd .. diff --git a/README.md b/README.md index 8a9254b8..a603bc6d 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,9 @@ This repo contains the RPB data transformation and the RPB web application. ## Setup -### Install metafix dependency +### Install metafacture dependencies -```bash -git clone https://github.com/metafacture/metafacture-fix.git -cd metafacture-fix -./gradlew publishToMavenLocal -cd .. -``` +See `.github/workflows/build.yml`. ### Clone RPB project repo @@ -42,6 +37,18 @@ sbt "runMain rpb.ETL conf/rpb-test-titel-to-strapi.flux" This writes a single `.json` files to `output/` (it's actually JSON lines, but the suffix makes it work with JSON tools, e.g. for syntax coloring and formatting). +### Import strapi data + +```bash +sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','u') PATH=articles" +sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','s') PATH=books" +sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','sbd') PATH=books" +sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=exists('f36t') PATH=multi-volume-books" +sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','sm') PATH=periodicals" +``` + +This attempts to import all data selected with the `PICK` variable to the API endpoint in `PATH`, and prints the server response. + ### Run transformation to lobid data ```bash diff --git a/build.sbt b/build.sbt index 3e33d7be..bf1c624b 100644 --- a/build.sbt +++ b/build.sbt @@ -10,14 +10,14 @@ libraryDependencies ++= Seq( cache, javaWs, "com.typesafe.play" % "play-test_2.11" % "2.4.11", - "org.metafacture" % "metafacture-elasticsearch" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-io" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-strings" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-json" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-flux" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-triples" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafacture-formatting" % "metafacture-core-5.4.1-rc3", - "org.metafacture" % "metafix" % "0.4.0", + "org.metafacture" % "metafacture-elasticsearch" % "5.5.1-rc1", + "org.metafacture" % "metafacture-io" % "5.5.1-rc1", + "org.metafacture" % "metafacture-strings" % "5.5.1-rc1", + "org.metafacture" % "metafacture-json" % "5.5.1-rc1", + "org.metafacture" % "metafacture-flux" % "5.5.1-rc1", + "org.metafacture" % "metafacture-triples" % "5.5.1-rc1", + "org.metafacture" % "metafacture-formatting" % "5.5.1-rc1", + "org.metafacture" % "metafix" % "0.5.1", "org.elasticsearch" % "elasticsearch" % "1.7.5" withSources(), "com.github.jsonld-java" % "jsonld-java" % "0.5.0", "org.apache.commons" % "commons-rdf-jena" % "0.5.0", diff --git a/conf/rpb-test-titel-import.flux b/conf/rpb-test-titel-import.flux new file mode 100644 index 00000000..274de027 --- /dev/null +++ b/conf/rpb-test-titel-import.flux @@ -0,0 +1,20 @@ +API_URL = "http://test-metadaten-nrw.hbz-nrw.de:1339/api/" + PATH; +FIX = " +unless " + PICK + " + reject() +end +"; + +FLUX_DIR + "output/test-output-strapi.json" +| open-file +| as-lines +| decode-json +| fix(FIX) +| encode-json +| regex-decode("(?.+)") +| stream-to-triples +| template("{\"${p}\":${o}}") // wrap into 'data' object for strapi +| open-http(url=API_URL, method="POST", contentType="application/json") +| as-lines +| print +;