#288 full rematch and log rematching progress

AtlasOfLivingAustralia · Oct 14, 2023 · eb53e3a · eb53e3a
1 parent 3f59093
commit eb53e3a
Show file tree

Hide file tree

Showing 12 changed files with 250 additions and 146 deletions.
diff --git a/build.gradle b/build.gradle
@@ -16,7 +16,7 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "4.4.0-SNAPSHOT"
+version "4.5.0-SNAPSHOT"
 group "au.org.ala"
 
 apply plugin:"eclipse"

diff --git a/grails-app/controllers/au/org/ala/specieslist/AdminController.groovy b/grails-app/controllers/au/org/ala/specieslist/AdminController.groovy
@@ -57,11 +57,14 @@ class AdminController {
             // retrieve qualified SpeciesListItems for performance reason
             def itemsIds = queryService.getFilterSpeciesListItemsIds(params)
             def lists = queryService.getFilterListResult(params, false, itemsIds)
+            def rematchLogs = helperService.queryRematchingProcess()
             def model = [lists:lists,
                          total:lists.totalCount,
                          typeFacets: (params.listType) ? null : queryService.getTypeFacetCounts(params, false, itemsIds),
                          tagFacets: queryService.getTagFacetCounts(params, itemsIds),
-                         selectedFacets:queryService.getSelectedFacets(params)]
+                         selectedFacets:queryService.getSelectedFacets(params),
+                         rematchLogs: rematchLogs
+            ]
             if (searchTerm) {
                 params.q = searchTerm
                 model.errors = "Error: Search terms must contain at least 3 characters"

diff --git a/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy b/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy
@@ -507,13 +507,12 @@ class SpeciesListController {
     /**
      * Rematches the scientific names in the supplied list
      */
-    @Transactional
     def rematch() {
         if (params.id && !params.id.startsWith("dr")) {
             params.id = SpeciesList.get(params.id)?.dataResourceUid
             log.info("Rematching for " + params.id)
         } else {
-            log.error("Rematching for ALL")
+            log.warn("Rematching for ALL")
         }
         Integer totalRows, offset = 0;
         String id = params.id
@@ -523,66 +522,7 @@ class SpeciesListController {
             return
         }
 
-        if (id) {
-            totalRows = SpeciesListItem.countByDataResourceUid(id)
-        } else {
-            totalRows = SpeciesListItem.count();
-        }
-
-        while (offset < totalRows) {
-            List items
-            List guidBatch = [], sliBatch = []
-            Map<SpeciesList, List<SpeciesListItem>> batches = new HashMap<>()
-            List<SpeciesListItem> searchBatch = new ArrayList<SpeciesListItem>()
-            if (id) {
-                items = SpeciesListItem.findAllByDataResourceUid(id, [max: BATCH_SIZE, offset: offset])
-            } else {
-                items = SpeciesListItem.list(max: BATCH_SIZE, offset: offset)
-            }
-
-            SpeciesListItem.withSession { session ->
-                items.eachWithIndex { SpeciesListItem item, Integer i ->
-                    SpeciesList speciesList = item.mylist
-                    List<SpeciesListItem> batch = batches.get(speciesList)
-                    if (batch == null) {
-                        batch = new ArrayList<>();
-                        batches.put(speciesList, batch)
-                    }
-                    String rawName = item.rawScientificName
-                    log.debug i + ". Rematching: " + rawName + "/" + speciesList.dataResourceUid
-                    if (rawName && rawName.length() > 0) {
-                        batch.add(item)
-                    } else {
-                        item.guid = null
-                        if (!item.save(flush: true)) {
-                            log.error "Error saving item (" + rawName + "): " + item.errors()
-                        }
-                    }
-                }
-                batches.each { list, batch ->
-                    helperService.matchAll(batch, list)
-                    batch.each {SpeciesListItem item ->
-                        if (item.guid) {
-                            guidBatch.push(item.guid)
-                            sliBatch.push(item)
-                        }
-                    }
-                }
-
-                if (!guidBatch.isEmpty()) {
-                    helperService.getCommonNamesAndUpdateRecords(sliBatch, guidBatch)
-                }
-
-                session.flush()
-                session.clear()
-            }
-
-            offset += BATCH_SIZE;
-            log.info("Rematched ${offset} of ${totalRows} - ${Math.round(offset * 100 / totalRows)}% complete")
-            if (offset > totalRows) {
-                log.error("Rematched ${offset} of ${totalRows} - ${Math.round(offset * 100 / totalRows)}% complete")
-            }
-        }
+        helperService.rematch(id)
 
         render(text: "${message(code: 'admin.lists.page.button.rematch.messages', default: 'Rematch complete')}")
     }

diff --git a/grails-app/controllers/au/org/ala/specieslist/UrlMappings.groovy b/grails-app/controllers/au/org/ala/specieslist/UrlMappings.groovy
@@ -70,8 +70,8 @@ class UrlMappings {
         "/ws/speciesListItemKvp/$druid" (controller: 'webService'){
             action = [GET: 'getSpeciesListItemKvp']
         }
-        "/ws/update_4.4" (controller: "webService", action: "updateMatchedSpecies_v4")
-
+        "/ws/rematchSpecies" (controller: "webService", action: "rematchSpecies")
+        "/ws/rematchStatus" (controller: "webService", action: "rematchStatus")
 		"/"(controller: 'public' ,action:  'index')
 		"500"(view:'/error')
         "404"(view:'/404')

diff --git a/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy b/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy
@@ -1457,16 +1457,19 @@ class WebServiceController {
         }
     }
 
+
     /**
-     * Only used for 4.4.0 release
-     * Update existing SpeciesListItem, add a linked matched species
+     * rematch existing SpeciesListItem
      */
-    def updateMatchedSpecies_v4(int max) {
-        if (!max || max < 0) {
-            max = 1000
-        }
-        def result = helperService.updateMatchedSpecies_v4(max)
-        def resp =  [remaining: "${result.total - result.updated}" , newMatchedSpecies: "${result.newAddedSpecies}", eclipsed: "${result.time}"]
+    def rematchSpecies() {
+        def result = helperService.rematchSpecies("developer", params.matchAll)
+        def resp = result.toMap()
+        render resp as JSON
+    }
+
+    def rematchStatus() {
+        def result = helperService.queryRematchingProcess()
+        def resp = result
         render resp as JSON
     }
 

diff --git a/grails-app/domain/au/org/ala/specieslist/MatchedSpecies.groovy b/grails-app/domain/au/org/ala/specieslist/MatchedSpecies.groovy
@@ -18,7 +18,6 @@ package au.org.ala.specieslist
 //import au.org.ala.names.ws.api.SearchStyle
 
 class MatchedSpecies {
-    String id
     String taxonConceptID
     String scientificName
     String scientificNameAuthorship
@@ -30,6 +29,7 @@ class MatchedSpecies {
     String taxonRank
     String family
     String genus
+    Date lastUpdated
 
     static constraints = {
         vernacularName(nullable:  true)
@@ -43,6 +43,7 @@ class MatchedSpecies {
         taxonRank(nullable: true)
         phylum(nullable: true)
         genus(nullable: true)
+        lastUpdated(nullable: true)
     }
 
 
@@ -61,4 +62,18 @@ class MatchedSpecies {
                 collectEntries { [it.name, this[it.name]] }
     }
 
+    def isSame(def target){
+        return this.taxonConceptID?.equalsIgnoreCase(target.taxonConceptID) &&
+                this.scientificNameAuthorship?.equalsIgnoreCase(target.scientificNameAuthorship) &&
+                this.vernacularName?.equalsIgnoreCase(target.vernacularName) &&
+                this.kingdom?.equalsIgnoreCase(target.kingdom) &&
+                this.phylum?.equalsIgnoreCase(target.phylum) &&
+                this.taxonClass?.equalsIgnoreCase(target.classs) &&
+                this.taxonOrder?.equalsIgnoreCase(target.order) &&
+                this.family?.equalsIgnoreCase(target.family) &&
+                this.genus?.equalsIgnoreCase(target.genus) &&
+                this.taxonRank?.equalsIgnoreCase(target.rank)
+
+    }
+
 }
diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties
@@ -215,6 +215,8 @@ specieslist.metadata.label=Metadata link
 specieslist.url.label=URL
 specieslist.wkt.label=WKT (GIS feature)
 
+speciesListItem.author=Author(s)
+
 view.lists.header=Species List
 view.lists.dataresource.tooltip=view Data Resource page
 view.lists.listinfo.edit.button.label=Edit

diff --git a/grails-app/services/au/org/ala/specieslist/ColumnMatchingService.groovy b/grails-app/services/au/org/ala/specieslist/ColumnMatchingService.groovy
@@ -119,8 +119,6 @@ class ColumnMatchingService implements GrailsConfigurationAware {
             }
         }
 
-        //headerResponse = parseHeadersCamelCase(headerResponse)
-
         if (hasName)
             [header: headerResponse, nameFound: hasName]
         else