Add RPPD export comparison workflow, tweak transformation (RPB-59)

hbz · Mar 12, 2024 · e29df3b · e29df3b
1 parent 963f8bc
commit e29df3b
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 5 deletions.
diff --git a/conf/rppd-to-lobid.fix b/conf/rppd-to-lobid.fix
@@ -60,7 +60,7 @@ do once("map")
 end
 
 # Kommentar Doku: #14 "x" eintragen, falls der Datensatz nicht im Webopac erscheinen soll, z.B. noch nicht aufgearbeitete ps-Sätze
-if exists("doNotIndex")
+if all_equal("doNotIndex", "true")
   reject()
 end
 
@@ -318,10 +318,12 @@ move_field("_temp", "gndSubjectCategory[]")
 #1ny	(Datum der letzten inhaltlichen Änderung) ->	describedBy.dateModified	
 # Kommentar Doku:  JJJJMMTT, z.B. 20120928 für 28.09.2012
 
-if any_match("dateModified", "(\\d{4})(\\d{2})(\\d{2})")
-  replace_all("dateModified", "(\\d{4})(\\d{2})(\\d{2})", "$1-$2-$3")
-  copy_field("dateModified", "describedBy.dateModified")
-end
+# Strapi: 2024-03-12T10:05:45.841Z -> 2024-03-12
+substring("updatedAt", "0", "10")
+move_field("updatedAt", "dateModified")
+# Allegro: "20240312" -> 2024-03-12
+replace_all("dateModified", "(\\d{4})(\\d{2})(\\d{2})", "$1-$2-$3")
+copy_field("dateModified", "describedBy.dateModified")
 
 # -------
 #1z1	(1. biogr. Anmerkung) ->	biographicalOrHistoricalInformation

diff --git a/conf/test-export-compare-rppd.flux b/conf/test-export-compare-rppd.flux
@@ -0,0 +1,32 @@
+// Get test data for the specified type; for each record,
+// fetch the entry from Strapi, convert that to lobid, write.
+
+// sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio_Test.txt OUT_FILE=test-output-rppd.json"
+// sbt -mem 2048 "runMain rpb.ETL conf/test-export-compare-rppd.flux"
+FLUX_DIR + "output/test-output-rppd.json"
+| open-file
+| as-lines
+| decode-json
+| fix("
+prepend(rppdId, 'https://rpb-cms-test.lobid.org/api/persons?populate=*&filters[rppdId][$eq]=')
+retain(rppdId)
+")
+| literal-to-object
+| log-object("Strapi URL: ")
+| open-http
+| as-records
+| decode-json(recordPath="data.[*].attributes")
+| fix(FLUX_DIR + "rppd-to-lobid.fix")
+| encode-json
+| write(FLUX_DIR + "output/test-rppd-output-from-strapi.json")
+;
+
+// To compare, convert test data directly to lobid, write.
+FLUX_DIR + "output/test-output-rppd.json"
+| open-file
+| as-lines
+| decode-json
+| fix(FLUX_DIR + "rppd-to-lobid.fix")
+| encode-json
+| write(FLUX_DIR + "output/test-rppd-output-from-file.json")
+;