Skip to content

Commit 09aeeb6

Browse files
jdamerowjdamerow
authored
Merge pull request #193 from diging/develop
prepare release
2 parents bdbf8db + 71b6f70 commit 09aeeb6

File tree

9 files changed

+88
-45
lines changed

9 files changed

+88
-45
lines changed

‎Conceptpower+Spring/pom.xml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<version>2.0-BUILD-SNAPSHOT</version>
1010
<properties>
1111
<java-version>1.8</java-version>
12-
<org.springframework-version>4.3.7.RELEASE</org.springframework-version>
12+
<org.springframework-version>4.3.18.RELEASE</org.springframework-version>
1313
<org.aspectj-version>1.7.3</org.aspectj-version>
1414
<org.slf4j-version>1.7.5</org.slf4j-version>
1515
<tomcat.deploy.path>http://localhost:8080/manager/text</tomcat.deploy.path>
@@ -19,21 +19,21 @@
1919
<db.files.path>/Users/jdamerow/Software/Conceptpower/db_files</db.files.path>
2020
<email.username></email.username>
2121
<email.password></email.password>
22-
<email.server.port></email.server.port>
22+
<email.server.port>8081</email.server.port>
2323
<email.server.host></email.server.host>
2424
<email.from></email.from>
2525
<admin.username>admin</admin.username>
2626
<admin.password>admin</admin.password>
2727
<wordnet.path>/usr/local/WordNet-3.0</wordnet.path>
2828
<spring-security-version>3.2.8.RELEASE</spring-security-version>
29-
<lucene.index>/Users/karthikeyanmohan/Software/Conceptpower/indexFiles</lucene.index>
29+
<lucene.index></lucene.index>
3030
<pullrequest></pullrequest>
3131
<debug_level>debug</debug_level>
3232
<skip.integration.tests>false</skip.integration.tests>
3333
<skip.unit.tests>false</skip.unit.tests>
34-
<concept.list.url>https://www.dropbox.com/s/48gftjus29qyp2q/conceptLists.db?raw=1</concept.list.url>
35-
<concept.types.url>https://www.dropbox.com/s/mla61f63fgfil3y/conceptTypes.db?raw=1</concept.types.url>
36-
<users.url>https://www.dropbox.com/s/n0vgpc5ow1xko4v/users.db?raw=1</users.url>
34+
<concept.list.url>https://www.dropbox.com/s/r82nymnt2247zz4/conceptLists.db?raw=1</concept.list.url>
35+
<concept.types.url>https://www.dropbox.com/s/nwn15s84fzz37s5/conceptTypes.db?raw=1</concept.types.url>
36+
<users.url>https://www.dropbox.com/s/p6h3974hmlouq0i/users.db?raw=1</users.url>
3737
</properties>
3838

3939
<repositories>

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/core/impl/ConceptManager.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import edu.asu.conceptpower.app.wordnet.WordNetManager;
3232
import edu.asu.conceptpower.core.ConceptEntry;
3333
import edu.asu.conceptpower.core.ConceptList;
34+
import edu.asu.conceptpower.rest.SearchParamters;
3435
import edu.asu.conceptpower.servlet.core.ChangeEvent;
3536
import edu.asu.conceptpower.servlet.core.ChangeEvent.ChangeEventTypes;
3637

@@ -72,7 +73,14 @@ public class ConceptManager implements IConceptManager {
7273
@Override
7374
public ConceptEntry getConceptEntry(String id) {
7475

75-
ConceptEntry[] entries = client.getEntriesByFieldContains(SearchFieldNames.MERGED_IDS, id);
76+
Map<String, String> fieldMap = new HashMap<>();
77+
fieldMap.put(SearchFieldNames.MERGED_IDS, id);
78+
ConceptEntry[] entries;
79+
try {
80+
entries = indexService.searchForConcepts(fieldMap, SearchParamters.OP_OR);
81+
} catch (IllegalAccessException | LuceneException | IndexerRunningException e) {
82+
return null;
83+
}
7684
if (entries != null && entries.length > 0) {
7785
fillConceptEntry(entries[0]);
7886
alternativeIdService.addAlternativeIds(id, entries[0]);

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/lucene/impl/LuceneUtility.java

Lines changed: 40 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.io.File;
44
import java.io.IOException;
5+
import java.io.StringReader;
56
import java.net.MalformedURLException;
67
import java.net.URL;
78
import java.nio.file.FileSystems;
@@ -22,10 +23,19 @@
2223
import javax.annotation.PreDestroy;
2324

2425
import org.apache.lucene.analysis.Analyzer;
26+
import org.apache.lucene.analysis.TokenStream;
27+
import org.apache.lucene.analysis.Tokenizer;
2528
import org.apache.lucene.analysis.core.KeywordAnalyzer;
29+
import org.apache.lucene.analysis.core.LowerCaseFilter;
2630
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
31+
import org.apache.lucene.analysis.custom.CustomAnalyzer;
32+
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
2733
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
2834
import org.apache.lucene.analysis.standard.StandardAnalyzer;
35+
import org.apache.lucene.analysis.standard.StandardFilter;
36+
import org.apache.lucene.analysis.standard.StandardTokenizer;
37+
import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
38+
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
2939
import org.apache.lucene.document.Document;
3040
import org.apache.lucene.document.Field;
3141
import org.apache.lucene.document.SortedDocValuesField;
@@ -36,6 +46,7 @@
3646
import org.apache.lucene.index.IndexWriterConfig;
3747
import org.apache.lucene.index.IndexableField;
3848
import org.apache.lucene.index.Term;
49+
import org.apache.lucene.queryparser.classic.QueryParser;
3950
import org.apache.lucene.search.BooleanClause;
4051
import org.apache.lucene.search.BooleanClause.Occur;
4152
import org.apache.lucene.search.BooleanQuery;
@@ -53,7 +64,9 @@
5364
import org.apache.lucene.store.Directory;
5465
import org.apache.lucene.store.FSDirectory;
5566
import org.apache.lucene.util.BytesRef;
67+
import org.apache.lucene.util.PagedBytes.Reader;
5668
import org.apache.lucene.util.QueryBuilder;
69+
import org.apache.lucene.util.Version;
5770
import org.jsoup.Jsoup;
5871
import org.slf4j.Logger;
5972
import org.slf4j.LoggerFactory;
@@ -62,7 +75,6 @@
6275
import org.springframework.context.annotation.PropertySource;
6376
import org.springframework.core.env.Environment;
6477
import org.springframework.stereotype.Component;
65-
6678
import edu.asu.conceptpower.app.constants.LuceneFieldNames;
6779
import edu.asu.conceptpower.app.constants.SearchFieldNames;
6880
import edu.asu.conceptpower.app.db4o.IConceptDBManager;
@@ -124,6 +136,7 @@ public class LuceneUtility implements ILuceneUtility {
124136

125137
private String lucenePath;
126138

139+
127140
private int numberOfResults;
128141

129142
private IndexWriter writer = null;
@@ -132,19 +145,23 @@ public class LuceneUtility implements ILuceneUtility {
132145
private Directory index;
133146
private Path relativePath = null;
134147
private IndexSearcher searcher = null;
135-
148+
private Analyzer customAnalyzer = null;
149+
136150
/**
137151
*
138152
* @throws LuceneException
139153
*/
140154
@PostConstruct
141-
public void init() throws LuceneException {
155+
public void init() throws LuceneException, IOException {
156+
customAnalyzer = CustomAnalyzer.builder().withTokenizer("keyword").addTokenFilter("asciifolding").addTokenFilter("worddelimiter").
157+
addTokenFilter("lowercase").build();
142158
lucenePath = env.getProperty("lucenePath");
143159
numberOfResults = Integer.parseInt(env.getProperty("numberOfLuceneResults"));
144160
try {
145161
relativePath = FileSystems.getDefault().getPath(lucenePath, "index");
162+
146163
index = FSDirectory.open(relativePath);
147-
configWhiteSpace = new IndexWriterConfig(standardAnalyzer);
164+
configWhiteSpace = new IndexWriterConfig(customAnalyzer);
148165
writer = new IndexWriter(index, configWhiteSpace);
149166
reader = DirectoryReader.open(writer, true);
150167
searcher = new IndexSearcher(reader);
@@ -256,7 +273,7 @@ private ConceptEntry getConceptFromDocument(Document d) throws IllegalAccessExce
256273
LuceneField luceneFieldAnnotation = field.getAnnotation(LuceneField.class);
257274
field.setAccessible(true);
258275
if (luceneFieldAnnotation != null && d.get(luceneFieldAnnotation.lucenefieldName()) != null)
259-
if (!luceneFieldAnnotation.isMultiple()) {
276+
if (luceneFieldAnnotation.isMultiple()) {
260277
IndexableField[] indexableFields = d.getFields(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.NOT_LOWERCASED);
261278
if (indexableFields == null || indexableFields.length == 0) {
262279
indexableFields = d.getFields(luceneFieldAnnotation.lucenefieldName());
@@ -497,7 +514,7 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
497514
if (operator == null || operator.equalsIgnoreCase(SearchParamters.OP_AND)) {
498515
occur = BooleanClause.Occur.MUST;
499516
}
500-
517+
501518
java.lang.reflect.Field[] fields = ConceptEntry.class.getDeclaredFields();
502519

503520
for (java.lang.reflect.Field field : fields) {
@@ -517,9 +534,8 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
517534

518535
}
519536

520-
PerFieldAnalyzerWrapper perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(standardAnalyzer,
521-
analyzerPerField);
522-
537+
538+
PerFieldAnalyzerWrapper perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(customAnalyzer, analyzerPerField);
523539
QueryBuilder qBuild = new QueryBuilder(perFieldAnalyzerWrapper);
524540
BooleanQuery.Builder builder = new BooleanQuery.Builder();
525541

@@ -579,32 +595,25 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
579595
ConceptEntry entry = getConceptFromDocument(d);
580596
concepts.add(entry);
581597
}
598+
return concepts.toArray(new ConceptEntry[concepts.size()]);
582599
}
583600

584601
catch (IOException ex) {
585602
throw new LuceneException("Issues in querying lucene index. Please retry", ex);
586603
}
587-
logger.debug("Number of concepts retrieved from lucene = " + concepts.size());
588-
return concepts.toArray(new ConceptEntry[concepts.size()]);
589-
590604
}
591605

592-
private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFieldAnalyzerWrapper,
593-
QueryBuilder qBuild, BooleanQuery.Builder builder, LuceneField luceneFieldAnnotation, String searchString) {
606+
private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFieldAnalyzerWrapper,QueryBuilder qBuild, BooleanQuery.Builder builder, LuceneField luceneFieldAnnotation, String searchString) {
594607
if (luceneFieldAnnotation.isTokenized()) {
595608
BooleanQuery.Builder tokenizedQueryBuilder = new BooleanQuery.Builder();
596-
buildTokenizedOrWildCardQuery(luceneFieldAnnotation, searchString, tokenizedQueryBuilder);
609+
buildTokenizedOrWildCardQuery(luceneFieldAnnotation, searchString, qBuild, tokenizedQueryBuilder);
597610

598611
if (luceneFieldAnnotation.isShortPhraseSearchable()) {
599612
BooleanQuery.Builder rootQueryBuilder = new BooleanQuery.Builder();
600613
rootQueryBuilder.add(tokenizedQueryBuilder.build(), Occur.SHOULD);
601614
// Short word searching
602615
BooleanQuery.Builder shortWordSearchQueryBuilder = new BooleanQuery.Builder();
603-
shortWordSearchQueryBuilder.add(
604-
new PhraseQuery(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.UNTOKENIZED_SUFFIX,
605-
searchString),
606-
Occur.SHOULD);
607-
616+
shortWordSearchQueryBuilder.add(new PhraseQuery(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.UNTOKENIZED_SUFFIX, searchString), Occur.SHOULD);
608617
rootQueryBuilder.add(shortWordSearchQueryBuilder.build(), Occur.SHOULD);
609618
tokenizedQueryBuilder = rootQueryBuilder;
610619
}
@@ -613,24 +622,25 @@ private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFi
613622
} else {
614623
if (luceneFieldAnnotation.isWildCardSearchEnabled()) {
615624
createWildCardSearchQuery(luceneFieldAnnotation, searchString, builder, occur);
616-
} else {
617-
builder.add(new BooleanClause(
618-
new TermQuery(new Term(luceneFieldAnnotation.lucenefieldName(), searchString)), occur));
619625
}
620-
}
626+
builder.add(new BooleanClause((qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString)), occur));
627+
}
621628
}
622629

623-
private void buildTokenizedOrWildCardQuery(LuceneField luceneFieldAnnotation, String searchString,
624-
BooleanQuery.Builder tokenizedQueryBuilder) {
625-
for (String searchValue : searchString.split(" ")) {
630+
private void buildTokenizedOrWildCardQuery(LuceneField luceneFieldAnnotation, String searchString, QueryBuilder qBuild,
631+
BooleanQuery.Builder tokenizedQueryBuilder) {
626632
if (luceneFieldAnnotation.isWildCardSearchEnabled()) {
627-
createWildCardSearchQuery(luceneFieldAnnotation, searchValue, tokenizedQueryBuilder, Occur.MUST);
633+
BooleanQuery.Builder analyzedBuilder = new BooleanQuery.Builder();
634+
createWildCardSearchQuery(luceneFieldAnnotation, searchString, analyzedBuilder, Occur.SHOULD);
635+
analyzedBuilder.add(new BooleanClause(
636+
(qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString)), Occur.SHOULD));
637+
tokenizedQueryBuilder.add(analyzedBuilder.build(), Occur.MUST);
628638
} else {
629-
tokenizedQueryBuilder.add(new PhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchValue),
639+
tokenizedQueryBuilder.add(qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString),
630640
Occur.MUST);
631641
}
632642
}
633-
}
643+
634644

635645
/**
636646
* This method adds the wild card query to the query builder when the

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/wordnet/WordNetManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public ConceptEntry getConcept(String id) {
6868
try {
6969
wordId = WordID.parseWordID(id);
7070
} catch (IllegalArgumentException e) {
71-
logger.error("Could not find id '" + id + "' in WordNet.", e);
71+
logger.warn("Could not find id '" + id + "' in WordNet.", e);
7272
return null;
7373
}
7474

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/rest/Concepts.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.util.List;
88
import java.util.ListIterator;
99

10+
import org.apache.commons.lang3.StringEscapeUtils;
1011
import org.json.simple.JSONArray;
1112
import org.json.simple.JSONObject;
1213
import org.json.simple.parser.JSONParser;
@@ -167,7 +168,7 @@ public class Concepts {
167168
HttpHeaders responseHeaders = new HttpHeaders();
168169
responseHeaders.add("Content-Type", MediaType.APPLICATION_JSON_VALUE + "; charset=utf-8");
169170

170-
return new ResponseEntity<String>(jsonObject.toJSONString(), responseHeaders,
171+
return new ResponseEntity<String>(StringEscapeUtils.unescapeJson(jsonObject.toJSONString()), responseHeaders,
171172
HttpStatus.OK);
172173
}
173174

@@ -242,7 +243,7 @@ public ResponseEntity<String> addConcepts(@RequestBody String body, Principal pr
242243
HttpHeaders responseHeaders = new HttpHeaders();
243244
responseHeaders.add("Content-Type", MediaType.APPLICATION_JSON_VALUE + "; charset=utf-8");
244245

245-
return new ResponseEntity<String>(responseArray.toJSONString(), responseHeaders,
246+
return new ResponseEntity<String>(StringEscapeUtils.unescapeJson(responseArray.toJSONString()), responseHeaders,
246247
HttpStatus.OK);
247248
}
248249

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/ConceptSearchController.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ public String search(HttpServletRequest req, ModelMap model, @RequestParam(defau
8787
if (results.hasErrors()) {
8888
return "conceptsearch";
8989
}
90+
conceptSearchBean.setWord(conceptSearchBean.getWord().trim());
9091

9192
if (conceptIdsToMerge != null) {
9293
model.addAttribute("conceptIdsToMerge", conceptIdsToMerge);

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/ConceptWrapperAddController.java

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import edu.asu.conceptpower.app.core.IConceptManager;
2929
import edu.asu.conceptpower.app.core.IIndexService;
30+
import edu.asu.conceptpower.app.core.impl.ConceptManager;
3031
import edu.asu.conceptpower.app.exceptions.DictionaryDoesNotExistException;
3132
import edu.asu.conceptpower.app.exceptions.DictionaryModifyException;
3233
import edu.asu.conceptpower.app.exceptions.IndexerRunningException;
@@ -82,10 +83,23 @@ private void initBinder(WebDataBinder binder) {
8283
* page
8384
*/
8485
@RequestMapping(value = "auth/conceptlist/addconceptwrapper")
85-
public String prepareConceptWrapperAdd(@ModelAttribute ConceptWrapperAddBean conceptWrapperAddBean,
86-
HttpServletRequest req, ModelMap model) {
86+
public String prepareConceptWrapperAdd(@RequestParam(value = "wrapperId", required = false) String wrapperId,
87+
HttpServletRequest req, ModelMap model){
8788
model.addAttribute("types", conceptWrapperService.fetchAllConceptTypes());
8889
model.addAttribute("lists", conceptWrapperService.fetchAllConceptLists());
90+
ConceptEntry entry = null;
91+
try {
92+
entry = conceptManager.getWordnetConceptEntry(wrapperId);
93+
} catch (LuceneException ex) {
94+
logger.error("Error while fetching concepts based on concept id", ex);
95+
return "/auth/conceptlist/addconceptwrapper";
96+
}
97+
ConceptWrapperAddBean conceptWrapperAddBean = new ConceptWrapperAddBean();
98+
conceptWrapperAddBean.setDescription(entry.getDescription());
99+
conceptWrapperAddBean.setWord(entry.getWord());
100+
conceptWrapperAddBean.setSelectedConceptList(entry.getConceptList());
101+
conceptWrapperAddBean.setPos(entry.getPos());
102+
conceptWrapperAddBean.setWrapperids(wrapperId);
89103
model.addAttribute("conceptWrapperAddBean", conceptWrapperAddBean);
90104
return "/auth/conceptlist/addconceptwrapper";
91105
}

‎Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/backing/ConceptWrapperAddBean.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public class ConceptWrapperAddBean {
1010
private String equals;
1111
private String similar;
1212
private String wrapperids;
13+
private String pos;
1314

1415
public String getWord() {
1516
return word;
@@ -75,4 +76,12 @@ public void setSynonymids(String synonymids) {
7576
this.synonymids = synonymids;
7677
}
7778

79+
public String getPos() {
80+
return pos;
81+
}
82+
83+
public void setPos(String pos) {
84+
this.pos = pos;
85+
}
86+
7887
}

‎Conceptpower+Spring/src/main/webapp/WEB-INF/views/home.jsp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -205,8 +205,8 @@ function prepareMergeConcept(conceptId) {
205205
}
206206
}
207207
208-
var createWrapper = function(word, pos, conceptList, description, conceptType, wrapperId) {
209-
window.location = '${pageContext.servletContext.contextPath}/auth/conceptlist/addconceptwrapper?word=' + word + '&selectedConceptList=' + conceptList + '&description=' + description + '&selectedType=' + conceptType + '&wrapperids=' + wrapperId;
208+
function createWrapper(wrapperId) {
209+
window.location = '${pageContext.servletContext.contextPath}/auth/conceptlist/addconceptwrapper?wrapperId=' + wrapperId;
210210
}
211211
212212
</script>
@@ -324,7 +324,7 @@ var createWrapper = function(word, pos, conceptList, description, conceptType, w
324324
</a>
325325
</c:when>
326326
<c:otherwise>
327-
<a href="#" onclick="createWrapper('${concept.entry.word}', '${concept.entry.pos}', '${concept.entry.conceptList}', '${concept.entry.description}', '${concept.entry.typeId}', '${concept.entry.id}');">
327+
<a href="#" onclick="createWrapper('${concept.entry.id}');">
328328
<i title='Create wrapper' class="fa fa-pencil-square-o"></i>
329329
</a>
330330
</c:otherwise>

0 commit comments

Comments
 (0)