Skip to content

Commit 9d3f26a

Browse files
marevolmarevol
committed
#2857 replace with fess-crawler-opensearch
1 parent 8ca1138 commit 9d3f26a

File tree

26 files changed

+106
-105
lines changed

26 files changed

+106
-105
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1441,7 +1441,7 @@
14411441
</dependency>
14421442
<dependency>
14431443
<groupId>org.codelibs.fess</groupId>
1444-
<artifactId>fess-crawler-es</artifactId>
1444+
<artifactId>fess-crawler-opensearch</artifactId>
14451445
<version>${crawler.version}</version>
14461446
</dependency>
14471447
<dependency>

‎src/main/java/org/codelibs/fess/app/web/admin/AdminAction.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.codelibs.fess.app.web.admin.dict.AdminDictAction;
2828
import org.codelibs.fess.app.web.admin.duplicatehost.AdminDuplicatehostAction;
2929
import org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction;
30-
import org.codelibs.fess.app.web.admin.esreq.AdminEsreqAction;
3130
import org.codelibs.fess.app.web.admin.failureurl.AdminFailureurlAction;
3231
import org.codelibs.fess.app.web.admin.fileauth.AdminFileauthAction;
3332
import org.codelibs.fess.app.web.admin.fileconfig.AdminFileconfigAction;
@@ -46,6 +45,7 @@
4645
import org.codelibs.fess.app.web.admin.role.AdminRoleAction;
4746
import org.codelibs.fess.app.web.admin.scheduler.AdminSchedulerAction;
4847
import org.codelibs.fess.app.web.admin.searchlog.AdminSearchlogAction;
48+
import org.codelibs.fess.app.web.admin.sereq.AdminSereqAction;
4949
import org.codelibs.fess.app.web.admin.storage.AdminStorageAction;
5050
import org.codelibs.fess.app.web.admin.suggest.AdminSuggestAction;
5151
import org.codelibs.fess.app.web.admin.systeminfo.AdminSysteminfoAction;
@@ -142,8 +142,8 @@ protected String getActionRole() {
142142
AdminBackupAction.ROLE + VIEW, //
143143
AdminMaintenanceAction.ROLE, //
144144
AdminMaintenanceAction.ROLE + VIEW, //
145-
AdminEsreqAction.ROLE, //
146-
AdminEsreqAction.ROLE + VIEW, //
145+
AdminSereqAction.ROLE, //
146+
AdminSereqAction.ROLE + VIEW, //
147147
AdminDashboardAction.ROLE, //
148148
AdminDashboardAction.ROLE + VIEW, //
149149
AdminWizardAction.ROLE, //
@@ -216,8 +216,8 @@ protected String getActionRole() {
216216
AdminBackupAction.ROLE + VIEW, //
217217
AdminMaintenanceAction.ROLE, //
218218
AdminMaintenanceAction.ROLE + VIEW, //
219-
AdminEsreqAction.ROLE, //
220-
AdminEsreqAction.ROLE + VIEW, //
219+
AdminSereqAction.ROLE, //
220+
AdminSereqAction.ROLE + VIEW, //
221221
AdminDashboardAction.ROLE, //
222222
AdminDashboardAction.ROLE + VIEW, //
223223
AdminWizardAction.ROLE, //
@@ -290,8 +290,8 @@ protected String getActionRole() {
290290
AdminBackupAction.ROLE + VIEW, //
291291
AdminMaintenanceAction.ROLE, //
292292
AdminMaintenanceAction.ROLE + VIEW, //
293-
AdminEsreqAction.ROLE, //
294-
AdminEsreqAction.ROLE + VIEW,//
293+
AdminSereqAction.ROLE, //
294+
AdminSereqAction.ROLE + VIEW,//
295295

296296
})
297297
public HtmlResponse index() {
@@ -413,8 +413,8 @@ public static Class<? extends FessAdminAction> getAdminActionClass(final FessUse
413413
if (user.hasRoles(getActionRoles(AdminMaintenanceAction.ROLE))) {
414414
return AdminMaintenanceAction.class;
415415
}
416-
if (user.hasRoles(getActionRoles(AdminEsreqAction.ROLE))) {
417-
return AdminEsreqAction.class;
416+
if (user.hasRoles(getActionRoles(AdminSereqAction.ROLE))) {
417+
return AdminSereqAction.class;
418418
}
419419
return null;
420420
}
@@ -423,4 +423,4 @@ private static String[] getActionRoles(final String role) {
423423
return new String[] { role, role + VIEW };
424424
}
425425

426-
}
426+
}

‎src/main/java/org/codelibs/fess/app/web/admin/esreq/AdminEsreqAction.java renamed to ‎src/main/java/org/codelibs/fess/app/web/admin/sereq/AdminSereqAction.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* either express or implied. See the License for the specific language
1414
* governing permissions and limitations under the License.
1515
*/
16-
package org.codelibs.fess.app.web.admin.esreq;
16+
package org.codelibs.fess.app.web.admin.sereq;
1717

1818
import java.io.BufferedReader;
1919
import java.io.File;
@@ -42,16 +42,16 @@
4242
/**
4343
* @author shinsuke
4444
*/
45-
public class AdminEsreqAction extends FessAdminAction {
45+
public class AdminSereqAction extends FessAdminAction {
4646

47-
public static final String ROLE = "admin-esreq";
47+
public static final String ROLE = "admin-sereq";
4848

49-
private static final Logger logger = LogManager.getLogger(AdminEsreqAction.class);
49+
private static final Logger logger = LogManager.getLogger(AdminSereqAction.class);
5050

5151
@Override
5252
protected void setupHtmlData(final ActionRuntime runtime) {
5353
super.setupHtmlData(runtime);
54-
runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameEsreq()));
54+
runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameSereq()));
5555
}
5656

5757
@Override
@@ -95,7 +95,7 @@ public ActionResponse upload(final UploadForm form) {
9595
throwValidationError(messages -> messages.addErrorsInvalidHeaderForRequestFile(GLOBAL, msg), () -> asListHtml(this::saveToken));
9696
} else {
9797
try (final CurlResponse response = curlRequest.body(buf.toString()).execute()) {
98-
final File tempFile = ComponentUtil.getSystemHelper().createTempFile("esreq_", ".json");
98+
final File tempFile = ComponentUtil.getSystemHelper().createTempFile("sereq_", ".json");
9999
try (final InputStream in = response.getContentAsStream()) {
100100
CopyUtil.copy(in, tempFile);
101101
} catch (final Exception e1) {
@@ -160,7 +160,7 @@ private HtmlResponse asListHtml(final Runnable runnable) {
160160
if (runnable != null) {
161161
runnable.run();
162162
}
163-
return asHtml(path_AdminEsreq_AdminEsreqJsp).useForm(UploadForm.class);
163+
return asHtml(path_AdminSereq_AdminSereqJsp).useForm(UploadForm.class);
164164
}
165165

166166
}

‎src/main/java/org/codelibs/fess/app/web/admin/esreq/UploadForm.java renamed to ‎src/main/java/org/codelibs/fess/app/web/admin/sereq/UploadForm.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* either express or implied. See the License for the specific language
1414
* governing permissions and limitations under the License.
1515
*/
16-
package org.codelibs.fess.app.web.admin.esreq;
16+
package org.codelibs.fess.app.web.admin.sereq;
1717

1818
import org.lastaflute.web.ruts.multipart.MultipartFormFile;
1919
import org.lastaflute.web.validation.Required;

‎src/main/java/org/codelibs/fess/crawler/service/FessUrlQueueService.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020

2121
import org.apache.logging.log4j.LogManager;
2222
import org.apache.logging.log4j.Logger;
23-
import org.codelibs.fess.crawler.entity.EsUrlQueue;
24-
import org.codelibs.fess.crawler.service.impl.EsUrlQueueService;
25-
import org.codelibs.fess.crawler.util.EsCrawlerConfig;
23+
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
24+
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
25+
import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;
2626
import org.codelibs.fess.helper.CrawlingConfigHelper;
2727
import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
2828
import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.ConfigName;
@@ -33,21 +33,21 @@
3333
import org.opensearch.search.sort.SortBuilders;
3434
import org.opensearch.search.sort.SortOrder;
3535

36-
public class FessUrlQueueService extends EsUrlQueueService {
36+
public class FessUrlQueueService extends OpenSearchUrlQueueService {
3737
private static final Logger logger = LogManager.getLogger(FessUrlQueueService.class);
3838

39-
public FessUrlQueueService(final EsCrawlerConfig crawlerConfig) {
39+
public FessUrlQueueService(final OpenSearchCrawlerConfig crawlerConfig) {
4040
super(crawlerConfig);
4141
}
4242

4343
@Override
44-
protected List<EsUrlQueue> fetchUrlQueueList(final String sessionId) {
44+
protected List<OpenSearchUrlQueue> fetchUrlQueueList(final String sessionId) {
4545
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
4646
final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(sessionId);
4747
final Map<String, String> configParams = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG);
4848
final String crawlOrder = configParams.getOrDefault(CrawlingConfig.Param.Config.CRAWL_ORDER, "sequential");
4949
if ("random".equals(crawlOrder)) {
50-
return getList(EsUrlQueue.class, sessionId,
50+
return getList(OpenSearchUrlQueue.class, sessionId,
5151
QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(),
5252
new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { new FunctionScoreQueryBuilder.FilterFunctionBuilder(
5353
new RandomScoreFunctionBuilder().seed(sessionId.hashCode())) }),

‎src/main/java/org/codelibs/fess/crawler/util/FessCrawlerConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
import org.codelibs.fess.util.ComponentUtil;
1919

20-
public class FessCrawlerConfig extends EsCrawlerConfig {
20+
public class FessCrawlerConfig extends OpenSearchCrawlerConfig {
2121

2222
@Override
2323
public String getQueueIndex() {

‎src/main/java/org/codelibs/fess/helper/PluginHelper.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ protected boolean isExcludedName(final ArtifactType artifactType, final String n
173173
|| "fess-crawler-db-h2".equals(name)//
174174
|| "fess-crawler-db-mysql".equals(name)//
175175
|| "fess-crawler-es".equals(name)//
176+
|| "fess-crawler-opensearch".equals(name)//
176177
|| "fess-crawler-lasta".equals(name)//
177178
|| "fess-crawler-parent".equals(name)//
178179
|| "fess-crawler-playwright".equals(name)//

‎src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@
3333
import org.codelibs.fess.crawler.CrawlerContext;
3434
import org.codelibs.fess.crawler.CrawlerStatus;
3535
import org.codelibs.fess.crawler.interval.FessIntervalController;
36-
import org.codelibs.fess.crawler.service.impl.EsDataService;
37-
import org.codelibs.fess.crawler.service.impl.EsUrlFilterService;
38-
import org.codelibs.fess.crawler.service.impl.EsUrlQueueService;
36+
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
37+
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
38+
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
3939
import org.codelibs.fess.indexer.IndexUpdater;
4040
import org.codelibs.fess.opensearch.config.exbhv.BoostDocumentRuleBhv;
4141
import org.codelibs.fess.opensearch.config.exentity.BoostDocumentRule;
@@ -142,7 +142,7 @@ protected void doCrawl(final String sessionId, final List<WebConfig> webConfigLi
142142
if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
143143
deleteCrawlData(sid);
144144
} else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
145-
final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
145+
final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
146146
try {
147147
urlFilterService.delete(sid);
148148
} catch (final Exception e) {
@@ -268,7 +268,7 @@ protected void doCrawl(final String sessionId, final List<WebConfig> webConfigLi
268268
if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
269269
deleteCrawlData(sid);
270270
} else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
271-
final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
271+
final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
272272
try {
273273
urlFilterService.delete(sid);
274274
} catch (final Exception e) {
@@ -479,9 +479,9 @@ protected List<BoostDocumentRule> getAvailableBoostDocumentRuleList() {
479479
}
480480

481481
protected void deleteCrawlData(final String sid) {
482-
final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
483-
final EsUrlQueueService urlQueueService = ComponentUtil.getComponent(EsUrlQueueService.class);
484-
final EsDataService dataService = ComponentUtil.getComponent(EsDataService.class);
482+
final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
483+
final OpenSearchUrlQueueService urlQueueService = ComponentUtil.getComponent(OpenSearchUrlQueueService.class);
484+
final OpenSearchDataService dataService = ComponentUtil.getComponent(OpenSearchDataService.class);
485485

486486
try {
487487
// clear url filter

‎src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@
2828
import org.codelibs.fess.crawler.Crawler;
2929
import org.codelibs.fess.crawler.entity.AccessResult;
3030
import org.codelibs.fess.crawler.entity.AccessResultData;
31-
import org.codelibs.fess.crawler.entity.EsAccessResult;
32-
import org.codelibs.fess.crawler.entity.EsUrlQueue;
31+
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
32+
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
3333
import org.codelibs.fess.crawler.service.DataService;
3434
import org.codelibs.fess.crawler.service.UrlFilterService;
3535
import org.codelibs.fess.crawler.service.UrlQueueService;
36-
import org.codelibs.fess.crawler.service.impl.EsDataService;
36+
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
3737
import org.codelibs.fess.crawler.transformer.Transformer;
38-
import org.codelibs.fess.crawler.util.EsResultList;
38+
import org.codelibs.fess.crawler.util.OpenSearchResultList;
3939
import org.codelibs.fess.exception.ContainerNotAvailableException;
4040
import org.codelibs.fess.exception.FessSystemException;
4141
import org.codelibs.fess.helper.IndexingHelper;
@@ -70,10 +70,10 @@ public class IndexUpdater extends Thread {
7070
protected SearchEngineClient searchEngineClient;
7171

7272
@Resource
73-
protected DataService<EsAccessResult> dataService;
73+
protected DataService<OpenSearchAccessResult> dataService;
7474

7575
@Resource
76-
protected UrlQueueService<EsUrlQueue> urlQueueService;
76+
protected UrlQueueService<OpenSearchUrlQueue> urlQueueService;
7777

7878
@Resource
7979
protected UrlFilterService urlFilterService;
@@ -176,17 +176,17 @@ public void run() {
176176
try {
177177
final Consumer<SearchRequestBuilder> cb = builder -> {
178178
final QueryBuilder queryBuilder =
179-
QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(EsAccessResult.SESSION_ID, sessionIdList))
180-
.filter(QueryBuilders.termQuery(EsAccessResult.STATUS, org.codelibs.fess.crawler.Constants.OK_STATUS));
179+
QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(OpenSearchAccessResult.SESSION_ID, sessionIdList)).filter(
180+
QueryBuilders.termQuery(OpenSearchAccessResult.STATUS, org.codelibs.fess.crawler.Constants.OK_STATUS));
181181
builder.setQuery(queryBuilder);
182182
builder.setFrom(0);
183183
final int maxDocumentCacheSize = fessConfig.getIndexerWebfsMaxDocumentCacheSizeAsInteger();
184184
builder.setSize(maxDocumentCacheSize <= 0 ? 1 : maxDocumentCacheSize);
185-
builder.addSort(EsAccessResult.CREATE_TIME, SortOrder.ASC);
185+
builder.addSort(OpenSearchAccessResult.CREATE_TIME, SortOrder.ASC);
186186
};
187187

188188
final DocList docList = new DocList();
189-
final List<EsAccessResult> accessResultList = new ArrayList<>();
189+
final List<OpenSearchAccessResult> accessResultList = new ArrayList<>();
190190

191191
long updateTime = systemHelper.getCurrentTimeAsLong();
192192
int errorCount = 0;
@@ -219,13 +219,13 @@ public void run() {
219219

220220
updateTime = systemHelper.getCurrentTimeAsLong();
221221

222-
List<EsAccessResult> arList = getAccessResultList(cb, cleanupTime);
222+
List<OpenSearchAccessResult> arList = getAccessResultList(cb, cleanupTime);
223223
if (arList.isEmpty()) {
224224
emptyListCount++;
225225
} else {
226226
emptyListCount = 0; // reset
227227
}
228-
long hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
228+
long hitCount = ((OpenSearchResultList<OpenSearchAccessResult>) arList).getTotalHits();
229229
while (hitCount > 0) {
230230
if (arList.isEmpty()) {
231231
ThreadUtil.sleep(fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue());
@@ -235,7 +235,7 @@ public void run() {
235235
cleanupTime = cleanupAccessResults(accessResultList);
236236
}
237237
arList = getAccessResultList(cb, cleanupTime);
238-
hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
238+
hitCount = ((OpenSearchResultList<OpenSearchAccessResult>) arList).getTotalHits();
239239
}
240240
if (!docList.isEmpty()) {
241241
indexingHelper.sendDocuments(searchEngineClient, docList);
@@ -320,11 +320,11 @@ public void run() {
320320

321321
}
322322

323-
private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList,
324-
final List<EsAccessResult> arList) {
323+
private void processAccessResults(final DocList docList, final List<OpenSearchAccessResult> accessResultList,
324+
final List<OpenSearchAccessResult> arList) {
325325
final FessConfig fessConfig = ComponentUtil.getFessConfig();
326326
final long maxDocumentRequestSize = Long.parseLong(fessConfig.getIndexerWebfsMaxDocumentRequestSize());
327-
for (final EsAccessResult accessResult : arList) {
327+
for (final OpenSearchAccessResult accessResult : arList) {
328328
if (logger.isDebugEnabled()) {
329329
logger.debug("Indexing {}", accessResult.getUrl());
330330
}
@@ -395,7 +395,7 @@ private void processAccessResults(final DocList docList, final List<EsAccessResu
395395
}
396396
}
397397

398-
private AccessResultData<?> getAccessResultData(final EsAccessResult accessResult) {
398+
private AccessResultData<?> getAccessResultData(final OpenSearchAccessResult accessResult) {
399399
try {
400400
return accessResult.getAccessResultData();
401401
} catch (final Exception e) {
@@ -483,7 +483,7 @@ protected void addFavoriteCountField(final Map<String, Object> map) {
483483
}
484484
}
485485

486-
private long cleanupAccessResults(final List<EsAccessResult> accessResultList) {
486+
private long cleanupAccessResults(final List<OpenSearchAccessResult> accessResultList) {
487487
if (!accessResultList.isEmpty()) {
488488
final long execTime = systemHelper.getCurrentTimeAsLong();
489489
final int size = accessResultList.size();
@@ -498,12 +498,12 @@ private long cleanupAccessResults(final List<EsAccessResult> accessResultList) {
498498
return -1;
499499
}
500500

501-
private List<EsAccessResult> getAccessResultList(final Consumer<SearchRequestBuilder> cb, final long cleanupTime) {
501+
private List<OpenSearchAccessResult> getAccessResultList(final Consumer<SearchRequestBuilder> cb, final long cleanupTime) {
502502
if (logger.isDebugEnabled()) {
503503
logger.debug("Getting documents in IndexUpdater queue.");
504504
}
505505
final long execTime = systemHelper.getCurrentTimeAsLong();
506-
final List<EsAccessResult> arList = ((EsDataService) dataService).getAccessResultList(cb);
506+
final List<OpenSearchAccessResult> arList = ((OpenSearchDataService) dataService).getAccessResultList(cb);
507507
final FessConfig fessConfig = ComponentUtil.getFessConfig();
508508
if (!arList.isEmpty()) {
509509
final long commitMarginTime = fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue();
@@ -513,7 +513,7 @@ private List<EsAccessResult> getAccessResultList(final Consumer<SearchRequestBui
513513
}
514514
}
515515
}
516-
final long totalHits = ((EsResultList<EsAccessResult>) arList).getTotalHits();
516+
final long totalHits = ((OpenSearchResultList<OpenSearchAccessResult>) arList).getTotalHits();
517517
if (logger.isInfoEnabled()) {
518518
final StringBuilder buf = new StringBuilder(100);
519519
buf.append("Processing ");

0 commit comments

Comments
 (0)