Skip to content

Commit

Permalink
SeaseLtd#128: Implement collection overrides for external Solr:
Browse files Browse the repository at this point in the history 
- Add collectionName to configuration;
- Implement for checkCollection and executeQuery methods;
- Add integration tests.
  • Loading branch information
@mattflax
mattflax committed Nov 23, 2020
1 parent 02b4275 commit 18bef73
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 11 deletions.
22 changes: 19 additions & 3 deletions ...orm-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand All @@ -60,6 +61,7 @@ public class ExternalApacheSolr implements SearchPlatform {
static final String SETTINGS_FILE = "solr-settings.json";

private final SolrClientManager clientManager = new SolrClientManager();
private final Map<String, SolrSettings> settingsMap = new HashMap<>();

@Override
public void beforeStart(Map<String, Object> configuration) {
Expand All @@ -85,6 +87,7 @@ public void load(File dataToBeIndexed, File settingsFile, String collection, Str
try {
// Load the index settings for this version of the search platform
SolrSettings settings = mapper.readValue(settingsFile, SolrSettings.class);
settingsMap.put(getFullyQualifiedDomainName(collection, version), settings);

if (clientManager.getSolrClient(version) == null) {
clientManager.buildSolrClient(version, settings);
Expand Down Expand Up @@ -123,11 +126,11 @@ public QueryOrSearchResponse executeQuery(String collection, String version, Str
}

return of(clientManager.getSolrClient(version)
.query(collection, query, SolrRequest.METHOD.POST))
.query(resolveCollectionName(collection, version), query, SolrRequest.METHOD.POST))
.map(response ->
new QueryOrSearchResponse(
response.getResults().getNumFound(),
new ArrayList<Map<String, Object>>(response.getResults())))
new ArrayList<>(response.getResults())))
.get();
} catch (SolrException e) {
LOGGER.error("Caught Solr exception :: " + e.getMessage());
Expand Down Expand Up @@ -167,7 +170,7 @@ public boolean checkCollection(String collection, String version) {
try {
SolrClient client = clientManager.getSolrClient(version);
if (client != null) {
SolrPingResponse response = client.ping(collection);
SolrPingResponse response = client.ping(resolveCollectionName(collection, version));
return response.getStatus() == 0;
}
} catch (SolrException e) {
Expand All @@ -181,10 +184,17 @@ public boolean checkCollection(String collection, String version) {
return false;
}

private String resolveCollectionName(String defaultCollection, String version) {
return Optional.ofNullable(settingsMap.get(getFullyQualifiedDomainName(defaultCollection, version)).getCollectionName())
.orElse(defaultCollection);
}

public static class SolrSettings {

@JsonProperty("baseUrls")
private final List<String> baseUrls;
@JsonProperty("collectionName")
private final String collectionName;
@JsonProperty("zkHosts")
private final List<String> zkHosts;
@JsonProperty("zkChroot")
Expand All @@ -195,11 +205,13 @@ public static class SolrSettings {
private final Integer socketTimeout;

public SolrSettings(@JsonProperty("baseUrls") List<String> baseUrls,
@JsonProperty("collectionName") String collectionName,
@JsonProperty("zkHosts") List<String> zkHosts,
@JsonProperty("zkChroot") String zkChroot,
@JsonProperty("connectionTimeoutMillis") Integer connectionTimeout,
@JsonProperty("socketTimeoutMillis") Integer socketTimeout) throws IllegalArgumentException {
this.baseUrls = baseUrls;
this.collectionName = collectionName;
this.zkHosts = zkHosts;
this.zkChroot = (zkChroot != null && zkChroot.length() > 0 ? zkChroot : null);
this.connectionTimeout = connectionTimeout;
Expand All @@ -219,6 +231,10 @@ public List<String> getBaseUrls() {
return baseUrls;
}

public String getCollectionName() {
return collectionName;
}

public List<String> getZkHosts() {
return zkHosts;
}
Expand Down
92 changes: 90 additions & 2 deletions ...m-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/ExternalApacheSolrIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@
*/
package io.sease.rre.search.api.impl;

import io.sease.rre.search.api.QueryOrSearchResponse;
import io.sease.rre.search.api.SearchPlatform;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -26,7 +30,11 @@

import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

Expand All @@ -43,8 +51,8 @@
*/
public class ExternalApacheSolrIT {

private static String SOLR_CONTAINER_BASE = "solr";
private static String DEFAULT_SOLR_VERSION = "8.6";
private static final String SOLR_CONTAINER_BASE = "solr";
private static final String DEFAULT_SOLR_VERSION = "8.6";

private static final String INDEX_NAME = "test";
private static final String INDEX_VERSION = "v1.0";
Expand Down Expand Up @@ -92,4 +100,84 @@ public void checkCollectionReturnsTrue_whenCollectionExists() throws Exception {
assertTrue(platform.checkCollection(INDEX_NAME, INDEX_VERSION));
solrContainer.close();
}

@Test
public void checkCollection_allowsCollectionOverride() throws Exception {
final String overrideCollection = "override";

final SolrContainer solrContainer = new SolrContainer(DOCKER_IMAGE).withCollection(overrideCollection);
solrContainer.start();

final File settingsFile = tempFolder.newFile("ccaco_settings.json");
FileWriter fw = new FileWriter(settingsFile);
fw.write("{ \"baseUrls\": [ \"http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr\" ], \"collectionName\": \"" + overrideCollection + "\" }");
fw.close();

platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION);

assertTrue(platform.checkCollection(INDEX_NAME, INDEX_VERSION));
solrContainer.close();
}


@Test
public void executeQuery_returnsResults() throws Exception {
final int numDocs = new Random().nextInt(10);

final SolrContainer solrContainer = new SolrContainer(DOCKER_IMAGE).withCollection(INDEX_NAME);
solrContainer.start();

// Index some documents
createDocuments(solrContainer.getHost(), solrContainer.getSolrPort(), INDEX_NAME, numDocs);

final File settingsFile = tempFolder.newFile("eq_settings.json");
FileWriter fw = new FileWriter(settingsFile);
fw.write("{ \"baseUrls\": [ \"http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr\" ]}");
fw.close();

platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION);

QueryOrSearchResponse response = platform.executeQuery(INDEX_NAME, INDEX_VERSION, "{ \"q\": \"*:*\" }", new String[]{ "id", "title_s" }, 10);

assertEquals(numDocs, response.totalHits());
solrContainer.close();
}

@Test
public void executeQuery_allowsOverride() throws Exception {
final int numDocs = new Random().nextInt(10);
final String overrideCollection = "override";

final SolrContainer solrContainer = new SolrContainer(DOCKER_IMAGE).withCollection(overrideCollection);
solrContainer.start();

// Index some documents
createDocuments(solrContainer.getHost(), solrContainer.getSolrPort(), overrideCollection, numDocs);

final File settingsFile = tempFolder.newFile("eqao_settings.json");
FileWriter fw = new FileWriter(settingsFile);
fw.write("{ \"baseUrls\": [ \"http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr\" ], \"collectionName\": \"" + overrideCollection + "\" }");
fw.close();

platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION);

QueryOrSearchResponse response = platform.executeQuery(INDEX_NAME, INDEX_VERSION, "{ \"q\": \"*:*\" }", new String[]{ "id", "title_s" }, 10);

assertEquals(numDocs, response.totalHits());
solrContainer.close();
}

private void createDocuments(String solrHost, int solrPort, String collection, int numDocs) throws Exception {
List<SolrInputDocument> docs = new ArrayList<>(numDocs);
for (int i = 0; i < numDocs; i ++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "" + i);
doc.addField("title_s", "Test");
docs.add(doc);
}

final SolrClient client = new HttpSolrClient.Builder("http://" + solrHost + ":" + solrPort + "/solr").build();
client.add(collection, docs);
client.commit(collection);
}
}
6 changes: 3 additions & 3 deletions ...-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/SolrClientManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public void shutdownClientManager() {
@Test
public void buildsHttpSolrClientForSingleHost() {
ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(
Collections.singletonList("http://localhost:8983/solr"), null, null, null, null);
Collections.singletonList("http://localhost:8983/solr"), null, null, null, null, null);

clientManager.buildSolrClient(TARGET_INDEX, settings);

Expand All @@ -83,7 +83,7 @@ public void buildsCloudSolrClientForMultipleHosts() throws Exception {
.map(jsr -> "http://localhost:" + jsr.getLocalPort() + "/solr")
.collect(Collectors.toList());

ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(baseUrls, null, null, null, null);
ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(baseUrls, null, null, null, null, null);

clientManager.buildSolrClient(TARGET_INDEX, settings);

Expand All @@ -96,7 +96,7 @@ public void buildsCloudSolrClientForMultipleHosts() throws Exception {
@Test
public void buildsCloudSolrClientForZkHosts() {
ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(
null, asList("localhost:2181", "localhost:2182"), null, null, null);
null, null, asList("localhost:2181", "localhost:2182"), null, null, null);

clientManager.buildSolrClient(TARGET_INDEX, settings);

Expand Down
8 changes: 5 additions & 3 deletions ...tform-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/SolrSettingsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import java.io.InputStream;
import java.util.Collections;

import static org.junit.Assert.*;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

/**
* Unit tests for the Solr settings.
Expand Down Expand Up @@ -69,12 +71,12 @@ public void canDeserializeZookeeperSettings() throws Exception {

@Test(expected=java.lang.IllegalArgumentException.class)
public void constructorThrowsException_whenNoURLsSet() {
new ExternalApacheSolr.SolrSettings(null, null, null, null, null);
new ExternalApacheSolr.SolrSettings(null, null, null, null, null, null);
}

@Test
public void canConstructWithZkHostsOnly() {
ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(null, Collections.singletonList("localhost:2181"), null, null, null);
ExternalApacheSolr.SolrSettings settings = new ExternalApacheSolr.SolrSettings(null, null, Collections.singletonList("localhost:2181"), null, null, null);
assertNotNull(settings);
assertTrue(settings.hasZookeeperSettings());
}
Expand Down

0 comments on commit 18bef73

Please sign in to comment.