Fetching collection fields for Solr

Now able to use what Solr has defined on startup to validate and determine create/update/remove of schema scaffold.

src/main/java/edu/tamu/scholars/middleware/config/model/IndexConfig.java

@@ -7,6 +7,8 @@
 @ConfigurationProperties(prefix = "middleware.index")
 public class IndexConfig {
 
+    private String name = "scholars-discovery";
+
     private String cron = "0 0 0 * * SUN";
 
     private String zone = "America/Chicago";
@@ -21,6 +23,14 @@ public class IndexConfig {
 
     private int batchSize = 10000;
 
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
     public String getCron() {
         return cron;
     }

src/main/java/edu/tamu/scholars/middleware/discovery/DiscoveryConstants.java

@@ -29,7 +29,8 @@ public class DiscoveryConstants {
     public static final String DISCOVERY_MODEL_PACKAGE = "edu.tamu.scholars.middleware.discovery.model";
 
     public static final String PARENTHESES_TEMPLATE = "(%s)";
-
+
+     // TODO: update CollectionTarget use and get collection from index.getName()
     public static final String COLLECTION = "scholars-discovery";
 
 }

src/main/java/edu/tamu/scholars/middleware/discovery/indicator/IndexHealthIndicator.java

@@ -16,6 +16,7 @@
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Component;
 
+import edu.tamu.scholars.middleware.config.model.IndexConfig;
 import edu.tamu.scholars.middleware.discovery.model.repo.IndividualRepo;
 import edu.tamu.scholars.middleware.discovery.service.IndexService;
 
@@ -32,14 +33,18 @@ public class IndexHealthIndicator implements HealthIndicator {
     @Autowired
     private IndexService indexService;
 
+    @Autowired
+    private IndexConfig index;
+
     @Override
     public Health health() {
         Health.Builder status = Health.down();
 
         Map<String, Object> details = new HashMap<String, Object>();
 
         try {
-            SolrPingResponse response = solrClient.ping("scholars-discovery");
+            SolrPingResponse response = solrClient.ping(index.getName());
+
             String message = (String) response.getResponse().get("status");
 
             // NOTE: not a REST response status code
@@ -58,6 +63,7 @@ public Health health() {
                 details.put("initializing", indexService.isSchematizing());
                 details.put("indexing", indexService.isIndexing());
                 details.put("ready", !indexService.isSchematizing() && !indexService.isIndexing());
+                details.put("scaffold", indexService.getScaffold());
                 details.put("schema", indexService.getSchema());
 
             } else {

src/main/java/edu/tamu/scholars/middleware/discovery/service/IndexService.java

@@ -1,27 +1,39 @@
 package edu.tamu.scholars.middleware.discovery.service;
 
+import java.io.IOException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import javax.annotation.PostConstruct;
 
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldsResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
 import org.springframework.stereotype.Service;
 
 import edu.tamu.scholars.middleware.config.model.IndexConfig;
+import edu.tamu.scholars.middleware.discovery.DiscoveryConstants;
+import edu.tamu.scholars.middleware.discovery.model.repo.IndividualRepo;
 import edu.tamu.scholars.middleware.discovery.service.component.AddOnlyAtomicHashSet;
 import edu.tamu.scholars.middleware.discovery.service.component.Harvester;
 import edu.tamu.scholars.middleware.discovery.service.component.Indexer;
+import edu.tamu.scholars.middleware.discovery.service.component.NamedTypedField;
 import edu.tamu.scholars.middleware.service.Triplestore;
 
 @Service
@@ -35,22 +47,29 @@ public class IndexService {
 
     public final static Set<String> CREATED_FIELDS = AddOnlyAtomicHashSet.forCreatedFields();
 
-    public final static Map<String, List<String>> SCHEMA = new ConcurrentHashMap<>(7);
+    public final static Map<String, List<NamedTypedField>> SCAFFOLD = new ConcurrentHashMap<>(7);
+
+    @Autowired
+    private ThreadPoolTaskScheduler threadPoolTaskScheduler;
 
     @Autowired
     private IndexConfig index;
 
     @Autowired
-    private List<Harvester> harvesters;
+    private SolrClient solrClient;
 
+    @Lazy
     @Autowired
-    private List<Indexer> indexers;
+    private IndividualRepo individualRepo;
 
     @Autowired
     private Triplestore triplestore;
 
     @Autowired
-    private ThreadPoolTaskScheduler threadPoolTaskScheduler;
+    private List<Harvester> harvesters;
+
+    @Autowired
+    private List<Indexer> indexers;
 
     public Boolean isSchematizing() {
         return schematizing.get();
@@ -60,21 +79,81 @@ public Boolean isIndexing() {
         return indexing.get();
     }
 
-    public Map<String, List<String>> getSchema() {
-        return SCHEMA;
+    public Map<String, List<NamedTypedField>> getScaffold() {
+        return SCAFFOLD;
+    }
+
+    public List<Map<String, Object>> getSchema() {
+        Optional<FieldsResponse> fieldsRes = Optional.empty();
+
+        Optional<Object[]> response = Optional.ofNullable(this.ping());
+
+        if (response.isPresent() && (Integer) response.get()[0] == 0) {
+
+            SchemaRequest.Fields fieldsReq = new SchemaRequest.Fields();
+            try {
+                fieldsRes = Optional.ofNullable(fieldsReq.process(solrClient, index.getName()));
+            } catch (SolrServerException e) {
+                logger.error("Unable to get fields from collection ... continue", e);
+            } catch (IOException e) {
+                logger.error("Unable to get fields from collection ... continue", e);
+            }
+
+        } else {
+            throw new RuntimeException("ping request failed");
+        }
+
+        if (fieldsRes.isPresent()) {
+            return fieldsRes.map(fr -> fr.getFields())
+                .get();
+        }
+
+        throw new RuntimeException("fields request failed");
     }
 
     @PostConstruct
     public void startup() {
+        logger.info("Scaffolding index fields...");
+        indexers.stream().forEach(indexer -> {
+            logger.info("Scaffolding {} fields.", indexer.name());
+            indexer.scaffold();
+        });
+
         if (index.isInitOnStartup()) {
-            schematize();
-        } else {
-            logger.info("Scaffolding index fields...");
-            indexers.stream().forEach(indexer -> {
-                logger.info(String.format("Scaffolding %s fields.", indexer.name()));
-                indexer.scaffold();
-            });
+            if (schematizing.compareAndSet(false, true)) {
+
+                Optional<Object[]> response = Optional.ofNullable(this.ping());
+
+                int status = -1;
+
+                if (response.isPresent()) {
+                    Object[] obj = response.get();
+                    status = (Integer) obj[0];
+                }
+
+                // success
+                if (status == 0) {
+                    logger.info("Initializing index fields for {}", index.getName());
+
+                    Map<String, Object> details = new HashMap<String, Object>();
+
+                    details.put("schema", getSchema());
+
+                    // suspecting some issues without shallow clone of response from Solr
+
+                    indexers.stream().forEach(indexer -> {
+                        logger.info("Initializing fields for {}", indexer.name());
+                        indexer.init((List<Map<String, Object>>) details.get("schema"));
+                    });
+                } else {
+                    logger.warn("Unable to connect to Solr collection {}", index.getName());
+
+                }
+
+                schematizing.set(false);
+            }
         }
+
         if (index.isOnStartup()) {
             threadPoolTaskScheduler.schedule(new Runnable() {
 
@@ -87,17 +166,6 @@ public void run() {
         }
     }
 
-    public void schematize() {
-        if (schematizing.compareAndSet(false, true)) {
-            logger.info("Initializing index fields...");
-            indexers.stream().forEach(indexer -> {
-                logger.info(String.format("Initializing %s fields.", indexer.name()));
-                indexer.init();
-            });
-            schematizing.set(false);
-        }
-    }
-
     @Scheduled(cron = "${middleware.index.cron}", zone = "${middleware.index.zone}")
     public void index() {
         if (indexing.compareAndSet(false, true)) {
@@ -129,4 +197,32 @@ public void index() {
         }
     }
 
+    private Object[] ping() {
+        Optional<SolrPingResponse> response = Optional.empty();
+
+        // assume nothing and return failed to connect
+        Integer status = -1;
+        Long count = 0L;
+        String message = "";
+
+        try {
+            response = Optional.ofNullable(solrClient.ping(index.getName()));
+        } catch (SolrServerException e) {
+            logger.error("Unable to connect to Solr ... continue", e);
+        } catch (IOException e) {
+            logger.error("Unable to connect to Solr ... continue", e);
+        }
+
+        if (response.isPresent()) {
+            status = response.get().getStatus();
+            message = (String) response.get().getResponse().get("status");
+
+            if (message.equals("OK")) {
+                count = individualRepo.count(DiscoveryConstants.DEFAULT_QUERY, List.of());
+            }
+        }
+
+        return new Object[] { status, count, message };
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/discovery/service/component/Indexer.java

@@ -1,29 +1,57 @@
 package edu.tamu.scholars.middleware.discovery.service.component;
 
 import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 
 import edu.tamu.scholars.middleware.discovery.model.AbstractIndexDocument;
 
+/**
+ * Indexer interface so far. See solr.SolrIndexer ^^^ and ..IndexService <<<.
+ */
 public interface Indexer {
 
     /**
-     * Called to initialize index.
+     * Scaffold in memory fields expected from concrete discovery.model.
      */
-    public void init();
+    public void scaffold();
 
     /**
-     * To be called when not needing to initialize index.
+     * Everything the application needs the solr collection to have specified.
      */
-    public void scaffold();
+    public void init(List<Map<String, Object>> schema);
 
+    /**
+     * Index a batch of abstract index documents.
+     * 
+     * @param documents batch to index
+     */
     public void index(Collection<AbstractIndexDocument> documents);
 
+    /**
+     * Index an abstract index documents.
+     *
+     * @param document individual
+     */
     public void index(AbstractIndexDocument document);
 
+    /**
+     * Used to flush commits.
+     */
     public void optimize();
 
+    /**
+     * Reflected type.
+     * 
+     * @return the typed class for abstract index document
+     */
     public Class<AbstractIndexDocument> type();
 
+    /**
+     * The concrete name provided by the implementation. Please place in discovery level constants class as final static.
+     * 
+     * @return name for the implementation
+     */
     public String name();
 
 }

src/main/java/edu/tamu/scholars/middleware/discovery/service/component/NamedTypedField.java

@@ -0,0 +1,14 @@
+package edu.tamu.scholars.middleware.discovery.service.component;
+
+import java.lang.reflect.Field;
+
+import edu.tamu.scholars.middleware.discovery.annotation.FieldType;
+
+// TODO: make class members private and add getters and setters
+// use public static method and private constructor
+// check if any class members can be final
+public class NamedTypedField {
+    public String name;
+    public FieldType fieldType;
+    public Field field;
+}

src/main/java/edu/tamu/scholars/middleware/discovery/service/component/solr/SolrCollectionAdminUtility.java

@@ -0,0 +1,14 @@
+package edu.tamu.scholars.middleware.discovery.service.component.solr;
+
+public class SolrCollectionAdminUtility {
+
+    public final static String SOLR_CONSTANT_PING_RESPONSE_STATUS = "status";
+    public final static String SOLR_CONSTANT_PING_RESPONSE_MESSAGE = "message";
+    public final static String SOLR_CONSTANT_PING_RESPONSE_OK = "OK";
+
+    private SolrCollectionAdminUtility() {
+
+    }
+
+
+}