From d1fbaaaab1f8cc4f086a312a2e2875620fa39d89 Mon Sep 17 00:00:00 2001 From: William Welling Date: Tue, 27 Aug 2024 13:40:25 -0500 Subject: [PATCH 1/4] Add logging to search export --- .../discovery/argument/BoostArg.java | 5 +++++ .../discovery/argument/FilterArg.java | 5 +++++ .../discovery/argument/QueryArg.java | 7 +++++++ .../discovery/model/repo/IndividualRepo.java | 19 +++++++++++++++++-- .../middleware/export/argument/ExportArg.java | 5 +++++ .../IndividualSearchExportController.java | 6 ++++++ 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/BoostArg.java b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/BoostArg.java index 3adff4f16..958b8a75d 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/BoostArg.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/BoostArg.java @@ -29,4 +29,9 @@ public static BoostArg of(String parameter) { return new BoostArg(parts[0], parts.length > 1 ? Float.valueOf(parts[1]) : 1.0f); } + @Override + public String toString() { + return "BoostArg [field=" + field + ", value=" + value + "]"; + } + } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/FilterArg.java b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/FilterArg.java index bb56a01c1..e3a37073b 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/FilterArg.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/FilterArg.java @@ -54,4 +54,9 @@ public static FilterArg of(String field, Optional value, Optional export(QueryArg query, List filters, List { try { solrClient.queryAndStreamResponse(collectionName, builder.query(), new StreamingResponseCallback() { @@ -217,16 +220,21 @@ public Flux export(QueryArg query, List filters, List 0) { remaining.set(numFound); } else { + logger.info("{}: COMPLETE", builder.getId()); emitter.complete(); } } @@ -443,7 +451,7 @@ private Page findAllQuery(SolrQuery query, Pageable pageable) { private List getValues(SolrDocument document, List dataFields) { return dataFields.stream() - .filter(v -> document.containsKey(v)) + .filter(document::containsKey) .flatMap(v -> document.getFieldValues(v).stream()) .map(v -> (String) v) .collect(Collectors.toList()); @@ -451,6 +459,8 @@ private List getValues(SolrDocument document, List dataFields) { private class SolrQueryBuilder { + private final UUID id; + private final SolrQuery query; private final List filters; @@ -460,6 +470,7 @@ private SolrQueryBuilder() { } private SolrQueryBuilder(String query) { + this.id = UUID.randomUUID(); this.query = new SolrQuery() .setParam("defType", defType) .setParam("q.op", defaultOperator) @@ -467,6 +478,10 @@ private SolrQueryBuilder(String query) { this.filters = new ArrayList<>(); } + public String getId() { + return id.toString(); + } + public SolrQueryBuilder withQuery(QueryArg query) { if (StringUtils.isNotEmpty(query.getDefaultField())) { diff --git a/src/main/java/edu/tamu/scholars/middleware/export/argument/ExportArg.java b/src/main/java/edu/tamu/scholars/middleware/export/argument/ExportArg.java index d99ebae29..0a574a1b9 100644 --- a/src/main/java/edu/tamu/scholars/middleware/export/argument/ExportArg.java +++ b/src/main/java/edu/tamu/scholars/middleware/export/argument/ExportArg.java @@ -39,4 +39,9 @@ public static ExportArg of(String parameter) { return new ExportArg(parts[0], parts.length > 1 ? parts[1] : parts[0]); } + @Override + public String toString() { + return "ExportArg [field=" + field + ", label=" + label + "]"; + } + } diff --git a/src/main/java/edu/tamu/scholars/middleware/export/controller/IndividualSearchExportController.java b/src/main/java/edu/tamu/scholars/middleware/export/controller/IndividualSearchExportController.java index 2e618ce01..bc7c1729e 100644 --- a/src/main/java/edu/tamu/scholars/middleware/export/controller/IndividualSearchExportController.java +++ b/src/main/java/edu/tamu/scholars/middleware/export/controller/IndividualSearchExportController.java @@ -11,6 +11,8 @@ import java.util.Optional; import java.util.concurrent.ExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Sort; @@ -40,6 +42,8 @@ @RestController public class IndividualSearchExportController implements RepresentationModelProcessor { + private static final Logger logger = LoggerFactory.getLogger(IndividualSearchExportController.class); + @Lazy @Autowired private IndividualRepo repo; @@ -58,7 +62,9 @@ public ResponseEntity export( List boosts, List export ) throws UnknownExporterTypeException, InterruptedException, ExecutionException { + logger.info("/individual/search/export {} {} {} {} {} {} {}", view, type, query, sort, filters, boosts, export); Exporter exporter = exporterRegistry.getExporter(type); + return ResponseEntity.ok() .header(CONTENT_DISPOSITION, exporter.contentDisposition(FilenameUtility.normalizeExportFilename(view))) .header(CONTENT_TYPE, exporter.contentType()) From 5a4a06d880fc57a0a785591516878e197fa63833 Mon Sep 17 00:00:00 2001 From: William Welling Date: Tue, 27 Aug 2024 14:16:51 -0500 Subject: [PATCH 2/4] Add additional checks to ensure complete is not called early --- .../discovery/model/Individual.java | 5 ++++ .../discovery/model/repo/IndividualRepo.java | 24 ++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Individual.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Individual.java index 650e6cd56..b0f98aaac 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Individual.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Individual.java @@ -118,4 +118,9 @@ public static Individual from(Map content) { return new Individual(content); } + @Override + public String toString() { + return "Individual [content=" + content + "]"; + } + } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java index da4214fb6..26d6ecccb 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java @@ -17,6 +17,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -215,26 +216,31 @@ public Flux export(QueryArg query, List filters, List 0) { - remaining.set(numFound); - } else { - logger.info("{}: COMPLETE", builder.getId()); + logger.info("{}: streamDocListInfo {} {} {}", builder.getId(), numFound, start, maxScore); + + remaining.set(numFound); + docListInfoReceived.set(true); + + if (numFound == 0) { + logger.info("{}: streamDocListInfo COMPLETE", builder.getId()); emitter.complete(); } } From 3550b555ab252ee0d152348584b44f961a07c088 Mon Sep 17 00:00:00 2001 From: William Welling Date: Tue, 27 Aug 2024 14:42:59 -0500 Subject: [PATCH 3/4] Handle Date conversion during export --- .../discovery/model/repo/IndividualRepo.java | 6 +-- .../export/service/CsvExporter.java | 26 ++++++---- .../middleware/utility/DateFormatUtility.java | 50 +++++++++++++------ 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java index 26d6ecccb..53411c34a 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java @@ -221,11 +221,11 @@ public Flux export(QueryArg query, List filters, List individuals, Lis .map(e -> e.getField()) .collect(Collectors.toList()); try (CSVPrinter printer = new CSVPrinter(outputStreamWriter, format)) { - individuals.subscribe( + individuals.doOnComplete(() -> { + try { + printer.flush(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Failed to flush CSV printer", e); + } + }).subscribe( individual -> { try { List row = getRow(individual, properties); @@ -78,16 +87,11 @@ public StreamingResponseBody streamIndividuals(Flux individuals, Lis | IOException e ) { e.printStackTrace(); + throw new RuntimeException("Failed mapping and printing individuals", e); } - }, error -> { - throw new RuntimeException("Failed attempting to stream individuals", error); - }, - () -> { - try { - printer.flush(); - } catch (IOException e) { - e.printStackTrace(); - } + }, e -> { + e.printStackTrace(); + throw new RuntimeException("Failed attempting to stream individuals", e); } ); } catch (IllegalArgumentException e) { @@ -136,6 +140,8 @@ private List getRow( .collect(Collectors.toList())); } + } else if (Date.class.isAssignableFrom(value.getClass())) { + data = DateFormatUtility.format((Date) value); } else { data = (String) value; } diff --git a/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java b/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java index 961e53599..a7857b73b 100644 --- a/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java +++ b/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java @@ -1,9 +1,12 @@ package edu.tamu.scholars.middleware.utility; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.Instant; +import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.Locale; @@ -15,35 +18,50 @@ */ public class DateFormatUtility { - private static final String[] datePatterns = { - "yyyy", - "E MMM dd HH:mm:ss z yyyy", - "yyyy-MM-dd'T'HH:mm:ss'Z'", - "yyyy-MM-dd'T'HH:mm:ss", - "dd-MM-yy", - "MM-dd-yyyy", - "yyyy-MM-dd HH:mm:ss", - "EEEEE MMMMM yyyy HH:mm:ss.SSSZ" + private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT); + + private static final String[] DATE_PATTERNS = { + DATE_FORMAT, + "yyyy", + "E MMM dd HH:mm:ss z yyyy", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ss", + "dd-MM-yy", + "MM-dd-yyyy", + "EEEEE MMMMM yyyy HH:mm:ss.SSSZ" }; - public static String parseYear(String value) throws IllegalArgumentException, ParseException { + private DateFormatUtility() { + + } + + public static String parseYear(String value) throws ParseException { return String.valueOf(parseZonedDateTime(value).getYear()); } - public static ZonedDateTime parseZonedDateTime(String value) throws IllegalArgumentException, ParseException { + public static String format(Date date) { + return date.toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDateTime() + .format(DATE_FORMATTER); + } + + public static ZonedDateTime parseZonedDateTime(String value) throws ParseException { return parseDate(value) - .toInstant() - .atZone(ZoneId.systemDefault()); + .toInstant() + .atZone(ZoneId.systemDefault()); } - public static Date parseDate(String value) throws IllegalArgumentException, ParseException { + public static Date parseDate(String value) throws ParseException { Locale locale = LocaleContextHolder.getLocale(); - return DateUtils.parseDate(value, locale, datePatterns); + return DateUtils.parseDate(value, locale, DATE_PATTERNS); } public static int getYear() { return ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault()) - .getYear(); + .getYear(); } } From cf4b0397449cd1d8e3f0ff71680b948c2fb75047 Mon Sep 17 00:00:00 2001 From: William Welling Date: Tue, 27 Aug 2024 15:13:03 -0500 Subject: [PATCH 4/4] Remove unused imports --- .../edu/tamu/scholars/middleware/utility/DateFormatUtility.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java b/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java index a7857b73b..f9fb3e878 100644 --- a/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java +++ b/src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java @@ -1,9 +1,7 @@ package edu.tamu.scholars.middleware.utility; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter;