diff --git a/.gitignore b/.gitignore index aec25a57b..96e7e975c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ Thumbs.db .settings/* # IntelliJ stuff -/.idea/* +/.idea /*.iml /*.ipr /*.iws diff --git a/CHANGES.md b/CHANGES.md index 5496b8600..e3ac2baaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,39 @@ # Change Log +## 6.0 + +### Endpoints + +* Image and information responses include a `Last-Modified` header when + possible. +* The health endpoint is enabled via `endpoint.health.enabled` rather than + `endpoint.api.enabled`. +* Added an HTTP API method to purge all infos from the derivative cache. +* Added a configuration option to automatically purge source-cached images + whose format cannot be inferred. + +### Sources + +* HttpSource supports a client HTTP proxy. (Thanks to @mightymax and + @mlindeman) +* HttpSource can be configured to send a ranged GET request instead of a HEAD + request, enabling it to work with pre-signed URLs that do not allow HEAD + requests. +* S3Source supports multiple endpoints when using ScriptLookupStrategy. + +### Caches + +* S3Cache uses multipart uploads, which reduces memory usage when caching + derivatives larger than 5 MB. + +### Delegate Script + +* The delegate script pathname can be set using the + `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the + `delegate_script.pathname` configuration key. +* The delegate script's `metadata` context key contains a new field, + `xmp_elements`, that provides a high-level key-value view of the XMP data. + ## 5.0.6 * IIIF information endpoints always return JSON in HTTP 4xx responses. diff --git a/UPGRADING.md b/UPGRADING.md index db1e9a7e3..3e38801c7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -3,6 +3,20 @@ If you are skipping versions, work through these sections backwards from your current version. +## 5.0 → 6.0 + +1. Add the following keys from the sample configuration: + * `endpoint.health.enabled` + * `HttpSource.proxy.http.host` + * `HttpSource.proxy.http.port` + * `HttpSource.BasicLookupStrategy.send_head_requests` + * `processor.purge_incompatible_from_source_cache` +2. Add the following methods from the sample delegate script: + * `jdbcsource_last_modified()` +3. If you are using a Java delegate, add the following method to your delegate + class: + * `getJDBCSourceLastModified()` + ## 4.1.x → 5.0 1. Note that the application is now packaged as a JAR file which can no longer diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index 6de429746..e205ad760 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -88,6 +88,8 @@ delegate_script.enabled = false # !! This can be an absolute path, or a filename; if only a filename is # specified, it will be searched for in the same folder as this file, and # then the current working directory. +# The delegate script pathname can also be set using the +# -Dcantaloupe.delegate_script VM argument, which overrides this value. delegate_script.pathname = delegates.rb ########################################################################### @@ -126,6 +128,9 @@ endpoint.api.enabled = false endpoint.api.username = endpoint.api.secret = +# Enables the health check endpoint, at /health. +endpoint.health.enabled = true + # If true, sources and caches will be checked, resulting in a more robust # but slower health check. Set this to false if these services already have # their own health checks. @@ -170,6 +175,10 @@ HttpSource.allow_insecure = false # Request timeout in seconds. HttpSource.request_timeout = +# !! Client HTTP proxy. +HttpSource.proxy.http.host = +HttpSource.proxy.http.port = + # Tells HttpSource how to look up resources. Allowed values are # `BasicLookupStrategy` and `ScriptLookupStrategy`. ScriptLookupStrategy # uses a delegate method for dynamic lookups; see the user manual. @@ -187,6 +196,12 @@ HttpSource.BasicLookupStrategy.url_suffix = HttpSource.BasicLookupStrategy.auth.basic.username = HttpSource.BasicLookupStrategy.auth.basic.secret = +# Before an image is retrieved, a preliminary request is sent to check +# various characteristics. Typically this is a HEAD request, but some +# resources, such as those using pre-signed URLs, may not support HEAD +# requests. This key enables a ranged GET request to be sent instead. +HttpSource.BasicLookupStrategy.send_head_requests = true + # Read data in chunks when it may be more efficient. (This also may end up # being less efficient, depending on many variables; see the user manual.) HttpSource.chunking.enabled = true @@ -359,6 +374,10 @@ processor.stream_retrieval_strategy = StreamStrategy # * `AbortStrategy` causes the request to fail. processor.fallback_retrieval_strategy = DownloadStrategy +# If true, images stored in the source cache for which no format can be +# inferred will be purged. +processor.purge_incompatible_from_source_cache = false + # Resolution of vector rasterization (of e.g. PDFs) at a scale of 1. processor.dpi = 150 diff --git a/delegates.rb.sample b/delegates.rb.sample index 6d39f0b56..d56cf1e9b 100644 --- a/delegates.rb.sample +++ b/delegates.rb.sample @@ -221,10 +221,17 @@ class CustomDelegate # # 1. String URI # 2. Hash with the following keys: - # * `uri` [String] (required) - # * `username` [String] For HTTP Basic authentication (optional). - # * `secret` [String] For HTTP Basic authentication (optional). - # * `headers` [Hash] Hash of request headers (optional). + # * `uri` [String] (required) + # * `username` [String] For HTTP Basic authentication + # (optional). + # * `secret` [String] For HTTP Basic authentication + # (optional). + # * `headers` [Hash] Hash of request headers + # (optional). + # * `send_head_request` [Boolean] Optional; defaults to `true`. See the + # documentation of the + # `HttpSource.BasicLookupStrategy.send_head_requests` + # configuration key. # 3. nil if not found. # # N.B.: this method should not try to perform authorization. `authorize()` @@ -241,18 +248,33 @@ class CustomDelegate # should be used instead. # # @param options [Hash] Empty hash. - # @return [String] Identifier of the image corresponding to the given - # identifier in the database. + # @return [String, nil] Database identifier of the image corresponding to the + # identifier in the context, or nil if not found. # def jdbcsource_database_identifier(options = {}) end + ## + # Returns either the last-modified timestamp of an image in ISO 8601 format, + # or an SQL statement that can be used to retrieve it from a `TIMESTAMP`-type + # column in the database. In the latter case, the "SELECT" and "FROM" clauses + # should be in uppercase in order to be autodetected. + # + # Implementing this method is optional, but may be necessary for certain + # features (like `Last-Modified` response headers) to work. + # + # @param options [Hash] Empty hash. + # @return [String, nil] + # + def jdbcsource_last_modified(options = {}) + end + ## # Returns either the media (MIME) type of an image, or an SQL statement that - # can be used to retrieve it, if it is stored in the database. In the latter - # case, the "SELECT" and "FROM" clauses should be in uppercase in order to - # be autodetected. If nil is returned, the media type will be inferred some - # other way, such as by identifier extension or magic bytes. + # can be used to retrieve it from a `CHAR`-type column in the database. In + # the latter case, the "SELECT" and "FROM" clauses should be in uppercase in + # order to be autodetected. If nil is returned, the media type will be + # inferred some other way, such as by identifier extension or magic bytes. # # @param options [Hash] Empty hash. # @return [String, nil] @@ -273,8 +295,11 @@ class CustomDelegate # should be used instead. # # @param options [Hash] Empty hash. - # @return [Hash,nil] Hash containing `bucket` and `key` keys; - # or nil if not found. + # @return [Hash,nil] Hash containing `bucket` and `key` keys. + # It may also contain an `endpoint` key, indicating that the endpoint + # is different from the one set in the configuration. In that case, + # it may also contain `region`, `access_key_id`, and/or + # `secret_access_key` keys. # def s3source_object_info(options = {}) end @@ -356,7 +381,14 @@ class CustomDelegate # "Field2Name": value # ], # "xmp_string": "...", - # "xmp_model": https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html + # "xmp_model": See https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html, + # "xmp_elements": { + # "Field1Name": "value", + # "Field2Name": [ + # "value1", + # "value2" + # ] + # }, # "native": { # # structure varies # } @@ -366,10 +398,13 @@ class CustomDelegate # * The `exif` key refers to embedded EXIF data. This also includes IFD0 # metadata from source TIFFs, whether or not an EXIF IFD is present. # * The `iptc` key refers to embedded IPTC IIM data. - # * The `xmp_string` key refers to raw embedded XMP data, which may or may - # not contain EXIF and/or IPTC information. + # * The `xmp_string` key refers to raw embedded XMP data. # * The `xmp_model` key contains a Jena Model object pre-loaded with the # contents of `xmp_string`. + # * The `xmp_elements` key contains a view of the embedded XMP data as key- + # value pairs. This is convenient to use, but may not work correctly with + # all XMP fields--in particular, those that cannot be expressed as + # key-value pairs. # * The `native` key refers to format-specific metadata. # # Any combination of the above keys may be present or missing depending on diff --git a/pom.xml b/pom.xml index 28b9dc726..94f60f209 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ edu.illinois.library.cantaloupe cantaloupe jar - 5.0.5 + 6.0-SNAPSHOT Cantaloupe https://cantaloupe-project.github.io/ 2015 @@ -17,6 +17,7 @@ UTF-8 2.21.4 2.15.2 + 9.4.53.v20231009 9.4.3.0 3.0.0-M5 @@ -423,8 +424,9 @@ random false - - --add-opens java.desktop/sun.awt.image=ALL-UNNAMED + + + --add-opens java.desktop/sun.awt.image=ALL-UNNAMED --illegal-access=permit diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java index 9db7b83e9..0071991a2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java @@ -13,9 +13,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.script.ScriptEngineManager; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; import java.util.stream.Collectors; /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java index a048842a0..a35c07c7f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.resource.FileServlet; import edu.illinois.library.cantaloupe.resource.HandlerServlet; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -265,7 +266,8 @@ public void start() throws Exception { // HTTP/2. if (isHTTPEnabled()) { HttpConfiguration config = new HttpConfiguration(); - HttpConnectionFactory http1 = new HttpConnectionFactory(); + config.setUriCompliance(UriCompliance.LEGACY); + HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2 = new HTTP2CServerConnectionFactory(config); @@ -280,11 +282,12 @@ public void start() throws Exception { // Initialize the HTTPS server. if (isHTTPSEnabled()) { HttpConfiguration config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.setSecurePort(getHTTPSPort()); config.addCustomizer(new SecureRequestCustomizer()); - final SslContextFactory contextFactory = new SslContextFactory.Server(); + final SslContextFactory.Server contextFactory = new SslContextFactory.Server(); contextFactory.setKeyStorePath(getHTTPSKeyStorePath()); if (getHTTPSKeyStorePassword() != null) { contextFactory.setKeyStorePassword(getHTTPSKeyStorePassword()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index 22bd99001..2d212dd6b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -67,7 +67,7 @@ private static class CustomBlobOutputStream /** * Constructor for an instance that writes into the given temporary - * blob. Upon closure, if the stream is {@link #isCompletelyWritten() + * blob. Upon closure, if the stream is {@link #isComplete() * completely written}, the temporary blob is copied into place and * deleted. Otherwise, the temporary blob is deleted. * @@ -96,7 +96,7 @@ public void close() throws IOException { blobOutputStream.flush(); blobOutputStream.close(); if (container != null) { - if (isCompletelyWritten()) { + if (isComplete()) { // Copy the temporary blob into place. CloudBlockBlob destBlob = container.getBlockBlobReference(blobKey); @@ -144,6 +144,8 @@ public void write(byte[] b, int off, int len) throws IOException { private static final Logger LOGGER = LoggerFactory.getLogger(AzureStorageCache.class); + private static final String INFO_EXTENSION = ".json"; + private static CloudBlobClient client; /** @@ -197,7 +199,7 @@ private static Instant getEarliestValidInstant() { @Override public Optional getInfo(Identifier identifier) throws IOException { - final String containerName = getContainerName(); + final String containerName = getContainerName(); final CloudBlobClient client = getClientInstance(); try { @@ -211,6 +213,12 @@ public Optional getInfo(Identifier identifier) throws IOException { if (isValid(blob)) { try (InputStream is = blob.openInputStream()) { Info info = Info.fromJSON(is); + // Populate the serialization timestamp if it is not + // already, as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp( + blob.getProperties().getLastModified().toInstant()); + } LOGGER.debug("getInfo(): read {} from container {} in {}", objectKey, containerName, watch); return Optional.of(info); @@ -290,7 +298,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) */ String getObjectKey(Identifier identifier) { return getObjectKeyPrefix() + "info/" + - StringUtils.md5(identifier.toString()) + ".json"; + StringUtils.md5(identifier.toString()) + INFO_EXTENSION; } /** @@ -390,6 +398,32 @@ private void purgeAsync(CloudBlob blob) { }); } + @Override + public void purgeInfos() throws IOException { + final String containerName = getContainerName(); + final CloudBlobClient client = getClientInstance(); + try { + final CloudBlobContainer container = + client.getContainerReference(containerName); + int count = 0, deletedCount = 0; + for (ListBlobItem item : container.listBlobs(getObjectKeyPrefix(), true)) { + if (item instanceof CloudBlob) { + CloudBlob blob = (CloudBlob) item; + count++; + if (blob.getName().endsWith(INFO_EXTENSION)) { + if (blob.deleteIfExists()) { + deletedCount++; + } + } + } + } + LOGGER.debug("purgeInfos(): deleted {} of {} items", + deletedCount, count); + } catch (URISyntaxException | StorageException e) { + throw new IOException(e.getMessage(), e); + } + } + @Override public void purgeInvalid() throws IOException { final String containerName = getContainerName(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java index 78ff159e5..9bdbfcf92 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java @@ -186,6 +186,17 @@ public void purge(OperationList opList) throws IOException { } } + /** + * @see DerivativeCache#purgeInfos() + * @since 6.0 + */ + public void purgeInfos() throws IOException { + Optional optCache = getDerivativeCache(); + if (optCache.isPresent()) { + optCache.get().purgeInfos(); + } + } + /** * @see Cache#purgeInvalid */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java index a0d9b3f98..1f2a0e70a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java @@ -10,14 +10,14 @@ */ public abstract class CompletableOutputStream extends OutputStream { - private boolean isCompletelyWritten; + private boolean isComplete; - public boolean isCompletelyWritten() { - return isCompletelyWritten; + public boolean isComplete() { + return isComplete; } - public void setCompletelyWritten(boolean isCompletelyWritten) { - this.isCompletelyWritten = isCompletelyWritten; + public void setComplete(boolean isCompletelyWritten) { + this.isComplete = isCompletelyWritten; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index 364baaf71..f7cdd0032 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -20,6 +20,13 @@ public interface DerivativeCache extends Cache { *

Reads the cached image information corresponding to the given * identifier.

* + *

If the returned instance's {@link Info#getSerializationTimestamp() + * serialization timestamp} is {@code null} (which it will be for {@link + * Info.Serialization serialization versions} earlier than {@link + * Info.Serialization#VERSION_5}), implementations should try to + * populate it with the last-modified time of the cached resource, if + * possible.

+ * *

If invalid image information exists in the cache, implementations * should delete it—ideally asynchronously.

* @@ -57,7 +64,7 @@ InputStream newDerivativeImageInputStream(OperationList opList) * *

The {@link CompletableOutputStream#close()} method of the returned * instance must check the return value of {@link - * CompletableOutputStream#isCompletelyWritten()} before committing data + * CompletableOutputStream#isComplete()} before committing data * to the cache. If it returns {@code false}, any written data should be * discarded.

* @@ -80,6 +87,16 @@ CompletableOutputStream newDerivativeImageOutputStream(OperationList opList) */ void purge(OperationList opList) throws IOException; + /** + * Deletes all cached infos. + * + * @throws IOException upon fatal error. Implementations should do the + * best they can to complete the operation and swallow and log + * non-fatal errors. + * @since 5.0 + */ + void purgeInfos() throws IOException; + /** *

Synchronously adds image information to the cache.

* diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index 9b7bc92e5..ab0dc383a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -17,7 +17,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileVisitOption; import java.nio.file.Files; @@ -126,8 +126,8 @@ private static class ConcurrentFileOutputStream private boolean isClosed = false; private final Object lock; private final Path tempFile; - private T toRemove; - private OutputStream wrappedOutputStream; + private final T toRemove; + private final OutputStream wrappedOutputStream; /** * @param tempFile Pathname of the temp file to write to. @@ -180,7 +180,7 @@ public void close() { // If the written file is complete, move it into place. // Otherwise, delete it. - if (isCompletelyWritten()) { + if (isComplete()) { CFOS_LOGGER.debug("close(): moving {} to {}", tempFile, destinationFile); Files.move(tempFile, destinationFile, @@ -237,7 +237,7 @@ public void write(byte[] b, int off, int len) throws IOException { private static final String INFO_FOLDER = "info"; private static final String SOURCE_IMAGE_FOLDER = "source"; - private static final String INFO_EXTENSION = ".json"; + static final String INFO_EXTENSION = ".json"; private static final String TEMP_EXTENSION = ".tmp"; /** @@ -316,7 +316,7 @@ static String hashedPathFragment(String uniqueString) { try { final MessageDigest digest = MessageDigest.getInstance(HASH_ALGORITHM); - digest.update(uniqueString.getBytes(Charset.forName("UTF8"))); + digest.update(uniqueString.getBytes(StandardCharsets.UTF_8)); final String sum = Hex.encodeHexString(digest.digest()); final Configuration config = Configuration.getInstance(); @@ -526,7 +526,14 @@ public Optional getInfo(Identifier identifier) throws IOException { final Path cacheFile = infoFile(identifier); if (!isExpired(cacheFile)) { LOGGER.debug("getInfo(): hit: {}", cacheFile); - return Optional.of(Info.fromJSON(cacheFile)); + Info info = Info.fromJSON(cacheFile); + // Populate the serialization timestamp if it is not + // already, as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp( + Files.getLastModifiedTime(cacheFile).toInstant()); + } + return Optional.of(info); } else { purgeAsync(cacheFile); } @@ -627,7 +634,7 @@ identifier, sourceImageTempFile(identifier), // work with newDerivativeImageOutputStream(). But this method does not // need that extra functionality, so setting it as completely written // here makes it behave like an ordinary OutputStream. - os.setCompletelyWritten(true); + os.setComplete(true); return os; } @@ -839,6 +846,44 @@ private void purgeAsync(final Path path) { }); } + @Override + public void purgeInfos() throws IOException { + if (isGlobalPurgeInProgress.get()) { + LOGGER.debug("purgeInfos() called with a purge in progress. Aborting."); + return; + } + synchronized (infoPurgeLock) { + while (!infosBeingPurged.isEmpty()) { + try { + LOGGER.debug("purgeInfos(): waiting..."); + infoPurgeLock.wait(); + } catch (InterruptedException e) { + break; + } + } + } + + try { + isGlobalPurgeInProgress.set(true); + + final InfoFileVisitor visitor = new InfoFileVisitor(); + + LOGGER.debug("purgeInfos(): starting..."); + Files.walkFileTree(rootPath(), + EnumSet.of(FileVisitOption.FOLLOW_LINKS), + Integer.MAX_VALUE, + visitor); + LOGGER.debug("purgeInfos(): purged {} info(s) totaling {} bytes", + visitor.getDeletedFileCount(), + visitor.getDeletedFileSize()); + } finally { + isGlobalPurgeInProgress.set(false); + synchronized (infoPurgeLock) { + infoPurgeLock.notifyAll(); + } + } + } + /** *

Crawls the image directory, deleting all expired files within it * (temporary or not), and then does the same in the info directory.

diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java index 414986b70..f870565e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java @@ -184,7 +184,7 @@ private class HeapCacheOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { LOGGER.debug("Closing stream for {}", opList); - if (isCompletelyWritten()) { + if (isComplete()) { Key key = itemKey(opList); Item item = new Item(wrappedStream.toByteArray()); cache.put(key, item); @@ -233,7 +233,7 @@ public void run() { if (workerShouldWork.get()) { try { purgeExcess(); - logger.debug("Cache size: {} items ({} bytes)", + logger.trace("Cache size: {} items ({} bytes)", size(), getByteSize()); Thread.sleep(INTERVAL_SECONDS * 1000); } catch (ConfigurationException e) { @@ -590,6 +590,11 @@ void purgeExcess() throws ConfigurationException { } } + @Override + public void purgeInfos() { + cache.entrySet().removeIf(entry -> entry.getKey().opList == null); + } + /** * Does nothing, as items in this cache never expire. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java b/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java new file mode 100644 index 000000000..d8d91f80f --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java @@ -0,0 +1,55 @@ +package edu.illinois.library.cantaloupe.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * Used by {@link Files#walkFileTree} to delete all info files within a + * directory. + */ +class InfoFileVisitor extends SimpleFileVisitor { + + private static final Logger LOGGER = + LoggerFactory.getLogger(InfoFileVisitor.class); + + private long deletedFileCount = 0; + private long deletedFileSize = 0; + + long getDeletedFileCount() { + return deletedFileCount; + } + + long getDeletedFileSize() { + return deletedFileSize; + } + + @Override + public FileVisitResult visitFile(Path path, + BasicFileAttributes attrs) { + try { + if (path.toString().endsWith(FilesystemCache.INFO_EXTENSION)) { + long size = Files.size(path); + Files.delete(path); + deletedFileCount++; + deletedFileSize += size; + } + } catch (IOException e) { + LOGGER.warn(e.getMessage(), e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + LOGGER.warn("visitFileFailed(): {}", e.getMessage()); + return FileVisitResult.CONTINUE; + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 41ff213c2..ee09c2997 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -48,7 +48,7 @@ class JdbcCache implements DerivativeCache { /** * Wraps a {@link Blob} OutputStream, for writing an image to a BLOB. * The constructor creates a transaction that is committed on close if the - * stream is {@link CompletableOutputStream#isCompletelyWritten() + * stream is {@link CompletableOutputStream#isComplete() * completely written}. */ private class ImageBlobOutputStream extends CompletableOutputStream { @@ -71,7 +71,16 @@ private class ImageBlobOutputStream extends CompletableOutputStream { connection.setAutoCommit(false); - blob = connection.createBlob(); + final Configuration config = Configuration.getInstance(); + final String sql = String.format( + "INSERT INTO %s (%s, %s, %s) VALUES (?, ?, ?)", + config.getString(Key.JDBCCACHE_DERIVATIVE_IMAGE_TABLE), + DERIVATIVE_IMAGE_TABLE_OPERATIONS_COLUMN, + DERIVATIVE_IMAGE_TABLE_IMAGE_COLUMN, + DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); + LOGGER.trace(sql); + + final Blob blob = connection.createBlob(); blobOutputStream = blob.setBinaryStream(1); } @@ -80,7 +89,7 @@ public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); PreparedStatement statement = null; try { - if (isCompletelyWritten()) { + if (isComplete()) { blobOutputStream.close(); final Configuration config = Configuration.getInstance(); final String sql = String.format( @@ -253,7 +262,7 @@ private void accessDerivativeImage(OperationList opList, statement.setTimestamp(1, now()); statement.setString(2, opList.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); statement.executeUpdate(); } } @@ -288,7 +297,7 @@ private void accessInfo(Identifier identifier, Connection connection) statement.setTimestamp(1, now()); statement.setString(2, identifier.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); statement.executeUpdate(); } } @@ -332,7 +341,7 @@ public Optional getInfo(Identifier identifier) throws IOException { statement.setString(1, identifier.toString()); statement.setTimestamp(2, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { accessInfoAsync(identifier); @@ -368,7 +377,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) statement.setString(1, opList.toString()); statement.setTimestamp(2, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { LOGGER.debug("Hit for image: {}", opList); @@ -431,6 +440,23 @@ public void purge(OperationList ops) throws IOException { } } + @Override + public void purgeInfos() throws IOException { + try (Connection connection = getConnection()) { + connection.setAutoCommit(false); + int numDeleted; + final String sql = "DELETE FROM " + getInfoTableName(); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + LOGGER.trace(sql); + numDeleted = statement.executeUpdate(); + } + connection.commit(); + LOGGER.debug("purgeInfos(): purged {} info(s)", numDeleted); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + } + @Override public void purgeInvalid() throws IOException { try (Connection connection = getConnection()) { @@ -457,7 +483,7 @@ private int purgeExpiredDerivativeImages(Connection conn) DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setTimestamp(1, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -472,7 +498,7 @@ private int purgeExpiredInfos(Connection conn) getInfoTableName(), INFO_TABLE_LAST_ACCESSED_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setTimestamp(1, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -489,7 +515,7 @@ private int purgeDerivativeImage(OperationList ops, Connection conn) DERIVATIVE_IMAGE_TABLE_OPERATIONS_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, ops.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -516,7 +542,7 @@ private void purgeDerivativeImageAsync(OperationList ops) { private int purgeDerivativeImages(Connection conn) throws SQLException { final String sql = "DELETE FROM " + getDerivativeImageTableName(); try (PreparedStatement statement = conn.prepareStatement(sql)) { - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -536,7 +562,7 @@ private int purgeDerivativeImages(Identifier identifier, Connection conn) " LIKE ?"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, identifier.toString() + "%"); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -570,7 +596,7 @@ private int purgeInfo(Identifier identifier, Connection conn) getInfoTableName(), INFO_TABLE_IDENTIFIER_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, identifier.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -592,7 +618,7 @@ private void purgeInfoAsync(Identifier identifier) { private int purgeInfos(Connection conn) throws SQLException { final String sql = "DELETE FROM " + getInfoTableName(); try (PreparedStatement statement = conn.prepareStatement(sql)) { - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -600,7 +626,7 @@ private int purgeInfos(Connection conn) throws SQLException { @Override public void put(Identifier identifier, Info info) throws IOException { if (!info.isPersistable()) { - LOGGER.debug("put(): info for {} is incomplete; ignoring", + LOGGER.trace("put(): info for {} is incomplete; ignoring", identifier); return; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java index 4d828c9ab..6996fde1b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java @@ -31,14 +31,16 @@ * *

Content is structured as follows:

* - * 
{
+ * {@code {
  *     #{@link #IMAGE_HASH_KEY}: {
  *         "operation list string representation": image byte array
  *     },
  *     #{@link #INFO_HASH_KEY}: {
  *         "identifier": "UTF-8 JSON string"
  *     }
- * }
 + * }} + * + * @since 3.4 */ class RedisCache implements DerivativeCache { @@ -154,7 +156,7 @@ private static class RedisOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { try { - if (isCompletelyWritten()) { + if (isComplete()) { connection.async().hset(hashKey, valueKey, bufferStream.toByteArray()); } @@ -202,7 +204,7 @@ private static synchronized StatefulRedisConnection getConnectio RedisURI.Builder.redis(config.getString(Key.REDISCACHE_HOST)). withPort(config.getInt(Key.REDISCACHE_PORT, 6379)). withSsl(config.getBoolean(Key.REDISCACHE_SSL, false)). - withPassword(config.getString(Key.REDISCACHE_PASSWORD, "")). + withPassword(config.getString(Key.REDISCACHE_PASSWORD, "").toCharArray()). withDatabase(config.getInt(Key.REDISCACHE_DATABASE, 0)). build(); RedisClient client = RedisClient.create(redisUri); @@ -249,13 +251,8 @@ public InputStream newDerivativeImageInputStream(OperationList opList) { @Override public void purge() { - // Purge infos - LOGGER.debug("purge(): purging {}...", INFO_HASH_KEY); - getConnection().sync().del(INFO_HASH_KEY); - - // Purge images - LOGGER.debug("purge(): purging {}...", IMAGE_HASH_KEY); - getConnection().sync().del(IMAGE_HASH_KEY); + purgeInfos(); + purgeImages(); } @Override @@ -271,11 +268,22 @@ public void purge(Identifier identifier) { MapScanCursor cursor = getConnection().sync(). hscan(IMAGE_HASH_KEY, imagePattern); - for (Object key : cursor.getMap().keySet()) { - getConnection().sync().hdel(IMAGE_HASH_KEY, (String) key); + for (String key : cursor.getMap().keySet()) { + getConnection().sync().hdel(IMAGE_HASH_KEY, key); } } + private void purgeImages() { + LOGGER.debug("purgeImages(): purging {}...", IMAGE_HASH_KEY); + getConnection().sync().del(IMAGE_HASH_KEY); + } + + @Override + public void purgeInfos() { + LOGGER.debug("purgeInfos(): purging {}...", INFO_HASH_KEY); + getConnection().sync().del(INFO_HASH_KEY); + } + /** * No-op. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index 8630ccac5..d01117de1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -30,10 +30,8 @@ import software.amazon.awssdk.services.s3.model.S3Object; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -64,159 +62,13 @@ */ class S3Cache implements DerivativeCache { - /** - *

Wraps a {@link ByteArrayOutputStream} for upload to S3.

- * - *

N.B.: S3 does not allow uploads without a {@code Content-Length} - * header, which cannot be provided when streaming an unknown amount of - * data (which this class is going to be doing all the time). From the - * documentation of {@link PutObjectRequest}:

- * - *
"When uploading directly from an input stream, content - * length must be specified before data can be uploaded to Amazon S3. If - * not provided, the library will have to buffer the contents of the input - * stream in order to calculate it. Amazon S3 explicitly requires that the - * content length be sent in the request headers before any of the data is - * sent."
- * - *

Since it's not possible to write an {@link OutputStream} of unknown - * length to the S3 client as the {@link Cache} interface requires, this - * class buffers written data in a byte array before uploading it to S3 - * upon closure. (The upload is submitted to the - * {@link ThreadPool#getInstance() application thread pool} in order for - * {@link #close()} to be able to return immediately.)

- */ - private static class S3OutputStream extends CompletableOutputStream { - - private final ByteArrayOutputStream bufferStream = - new ByteArrayOutputStream(); - private final S3Client client; - private final String bucketName; - private final String objectKey; - private final String contentType; - - /** - * @param client S3 client. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - */ - S3OutputStream(final S3Client client, - final String bucketName, - final String objectKey, - final String contentType) { - this.client = client; - this.bucketName = bucketName; - this.objectKey = objectKey; - this.contentType = contentType; - } - - @Override - public void close() throws IOException { - try { - bufferStream.close(); - byte[] data = bufferStream.toByteArray(); - if (isCompletelyWritten()) { - // At this point, the client has received all image data, - // but it is still waiting for the connection to close. - // Uploading in a separate thread will allow this to happen - // immediately. - ThreadPool.getInstance().submit(new S3Upload( - client, data, bucketName, objectKey, - contentType, null)); - } - } finally { - super.close(); - } - } - - @Override - public void flush() throws IOException { - bufferStream.flush(); - } - - @Override - public void write(int b) { - bufferStream.write(b); - } - - @Override - public void write(byte[] b) throws IOException { - bufferStream.write(b); - } - - @Override - public void write(byte[] b, int off, int len) { - bufferStream.write(b, off, len); - } - - } - - private static class S3Upload implements Runnable { - - private static final Logger UPLOAD_LOGGER = - LoggerFactory.getLogger(S3Upload.class); - - private final String bucketName, contentEncoding, contentType, objectKey; - private final byte[] data; - private final S3Client client; - - /** - * @param client S3 client. - * @param data Data to upload. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - * @param contentEncoding Content encoding. May be {@code null}. - */ - S3Upload(S3Client client, - byte[] data, - String bucketName, - String objectKey, - String contentType, - String contentEncoding) { - this.client = client; - this.bucketName = bucketName; - this.data = data; - this.contentType = contentType; - this.contentEncoding = contentEncoding; - this.objectKey = objectKey; - } - - @Override - public void run() { - if (data.length > 0) { - PutObjectRequest request = PutObjectRequest.builder() - .bucket(bucketName) - .key(objectKey) - .contentType(contentType) - .contentEncoding(contentEncoding) - .build(); - final Stopwatch watch = new Stopwatch(); - - UPLOAD_LOGGER.debug("Uploading {} bytes to {} in bucket {}", - data.length, request.key(), request.bucket()); - - try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { - client.putObject(request, - RequestBody.fromInputStream(is, data.length)); - } catch (IOException e) { - UPLOAD_LOGGER.warn(e.getMessage(), e); - } - - UPLOAD_LOGGER.trace("Wrote {} bytes to {} in bucket {} in {}", - data.length, request.key(), request.bucket(), - watch); - } else { - UPLOAD_LOGGER.trace("No data to upload; returning"); - } - } - - } - private static final Logger LOGGER = LoggerFactory.getLogger(S3Cache.class); + private static final String IMAGE_KEY_PREFIX = "image/"; + private static final String INFO_EXTENSION = ".json"; + private static final String INFO_KEY_PREFIX = "info/"; + /** * Lazy-initialized by {@link #getClientInstance}. */ @@ -283,6 +135,11 @@ public Optional getInfo(Identifier identifier) throws IOException { // This extra validity check may be needed with minio server if (is != null && is.response().lastModified().isAfter(earliestValidInstant())) { final Info info = Info.fromJSON(is); + // Populate the serialization timestamp if it is not already, + // as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp(is.response().lastModified()); + } LOGGER.debug("getInfo(): read {} from bucket {} in {}", objectKey, bucketName, watch); touchAsync(objectKey); @@ -341,10 +198,10 @@ public InputStream newDerivativeImageInputStream(OperationList opList) @Override public CompletableOutputStream newDerivativeImageOutputStream(OperationList opList) { - final String objectKey = getObjectKey(opList); - final String bucketName = getBucketName(); - final S3Client client = getClientInstance(); - return new S3OutputStream(client, bucketName, objectKey, + final String objectKey = getObjectKey(opList); + final String bucketName = getBucketName(); + final S3Client client = getClientInstance(); + return new S3MultipartAsyncOutputStream(client, bucketName, objectKey, opList.getOutputFormat().getPreferredMediaType().toString()); } @@ -353,8 +210,8 @@ public InputStream newDerivativeImageInputStream(OperationList opList) * given identifier. */ String getObjectKey(Identifier identifier) { - return getObjectKeyPrefix() + "info/" + - StringUtils.md5(identifier.toString()) + ".json"; + return getObjectKeyPrefix() + INFO_KEY_PREFIX + + StringUtils.md5(identifier.toString()) + INFO_EXTENSION; } /** @@ -370,8 +227,8 @@ String getObjectKey(OperationList opList) { if (encode != null) { extension = "." + encode.getFormat().getPreferredExtension(); } - return String.format("%simage/%s/%s%s", - getObjectKeyPrefix(), idHash, opsHash, extension); + return getObjectKeyPrefix() + IMAGE_KEY_PREFIX + idHash + "/" + + opsHash + extension; } /** @@ -415,7 +272,7 @@ public void purge(final Identifier identifier) { // purge images final S3Client client = getClientInstance(); final String bucketName = getBucketName(); - final String prefix = getObjectKeyPrefix() + "image/" + + final String prefix = getObjectKeyPrefix() + IMAGE_KEY_PREFIX + StringUtils.md5(identifier.toString()); final AtomicInteger counter = new AtomicInteger(); @@ -456,6 +313,24 @@ private void purgeAsync(final String bucketName, final String key) { }); } + @Override + public void purgeInfos() { + final S3Client client = getClientInstance(); + final String bucketName = getBucketName(); + final String prefix = getObjectKeyPrefix() + INFO_KEY_PREFIX; + final AtomicInteger counter = new AtomicInteger(); + + S3Utils.walkObjects(client, bucketName, prefix, (object) -> { + LOGGER.trace("purgeInfos(): deleting {}", object.key()); + client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(object.key()) + .build()); + counter.incrementAndGet(); + }); + LOGGER.debug("purgeInfos(): deleted {} items", counter.get()); + } + @Override public void purgeInvalid() { final S3Client client = getClientInstance(); @@ -506,13 +381,28 @@ public void put(Identifier identifier, Info info) throws IOException { @Override public void put(Identifier identifier, String info) throws IOException { LOGGER.debug("put(): caching info for {}", identifier); - final S3Client client = getClientInstance(); - final String objectKey = getObjectKey(identifier); - final String bucketName = getBucketName(); + final Stopwatch watch = new Stopwatch(); + + PutObjectRequest request = PutObjectRequest.builder() + .bucket(getBucketName()) + .key(getObjectKey(identifier)) + .contentType(MediaType.APPLICATION_JSON.toString()) + .contentEncoding("UTF-8") + .build(); + byte[] data = info.getBytes(StandardCharsets.UTF_8); + LOGGER.trace("put(): uploading {} bytes to {} in bucket {}", + data.length, request.key(), request.bucket()); + + try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { + getClientInstance().putObject(request, + RequestBody.fromInputStream(is, data.length)); + } catch (IOException e) { + LOGGER.warn(e.getMessage(), e); + } - new S3Upload(client, info.getBytes(StandardCharsets.UTF_8), - bucketName, objectKey, MediaType.APPLICATION_JSON.toString(), - "UTF-8").run(); + LOGGER.trace("put(): wrote {} bytes to {} in bucket {} in {}", + data.length, request.key(), request.bucket(), + watch); } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java new file mode 100644 index 000000000..56990bf74 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java @@ -0,0 +1,322 @@ +package edu.illinois.library.cantaloupe.cache; + +import edu.illinois.library.cantaloupe.async.ThreadPool; +import org.apache.commons.compress.utils.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + *

Uploads written data to S3 in parts without blocking on uploads.

+ * + *

The multi-part upload process involves three types of operations: + * creating the upload, uploading the parts, and completing the upload. Each of + * these are encapsulated in {@link Runnable runnable} inner classes. The + * {@link #write} methods add appropriate instances of these to a queue which + * is consumed by a worker running in the {@link ThreadPool#getInstance() + * application thread pool}.

+ * + *

Clients will notice that calls to {@link #write} and {@link #close()} + * (that would otherwise block on communication with S3) return immediately. + * After {@link #close()} returns, the resulting object will take a little bit + * of time to appear in the bucket.

+ * + *

Note that because this is a {@link CompletableOutputStream}, if the + * instance is not {@link #setComplete(boolean) marked as complete} before + * closure, the upload will be aborted.

+ * + *

Multi-part uploads can reduce memory usage when uploading objects larger + * than the part length, as that is roughly the maximum amount that has to be + * buffered in memory (provided that the length of the byte array passed to + * either of the {@link #write} methods is not greater than the part + * length).

+ * + *

N.B.: Incomplete uploads should be aborted automatically, but when using + * Amazon S3, it may be helpful to enable the {@literal + * AbortIncompleteMultipartUpload} lifecycle rule as a fallback.

+ * + * @author Alex Dolski UIUC + * @since 6.0 + */ +public class S3MultipartAsyncOutputStream extends CompletableOutputStream { + + private interface TerminalTask {} + + private static class Worker implements Runnable { + private final BlockingQueue workQueue = + new LinkedBlockingQueue<>(); + private boolean isDone, isStopped; + + void add(Runnable task) { + workQueue.add(task); + } + + @Override + public void run() { + while (!isDone && !isStopped) { + try { + Runnable task = workQueue.take(); + task.run(); + if (task instanceof TerminalTask) { + isDone = true; + } + } catch (InterruptedException e) { + isStopped = true; + } + } + } + } + + private class RequestCreator implements Runnable { + private final Logger logger = + LoggerFactory.getLogger(RequestCreator.class); + + @Override + public void run() { + logger.trace("Creating request [bucket: {}] [key: {}]", + bucket, key); + CreateMultipartUploadRequest createMultipartUploadRequest = + CreateMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .contentType(contentType) + .contentEncoding("UTF-8") + .build(); + CreateMultipartUploadResponse response = + client.createMultipartUpload(createMultipartUploadRequest); + uploadID = response.uploadId(); + } + } + + private class PartUploader implements Runnable { + private final Logger logger = + LoggerFactory.getLogger(PartUploader.class); + + private final ByteArrayOutputStream part; + + PartUploader(ByteArrayOutputStream part) { + this.part = part; + } + + @Override + public void run() { + try { + final int partNumber = partIndex + 1; + + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .partNumber(partNumber) + .build(); + + // There is a small chance that the last part will be empty. + if (part.size() == 0) { + logger.trace("Skipping empty part {} [upload ID: {}]", + partNumber, uploadID); + return; + } + + byte[] bytes = part.toByteArray(); + + logger.trace("Uploading part {} ({} bytes) [upload ID: {}]", + uploadPartRequest.partNumber(), bytes.length, uploadID); + + String etag = client.uploadPart( + uploadPartRequest, + RequestBody.fromBytes(bytes)).eTag(); + CompletedPart completedPart = CompletedPart.builder() + .partNumber(uploadPartRequest.partNumber()) + .eTag(etag) + .build(); + completedParts.add(completedPart); + } finally { + IOUtils.closeQuietly(part); + } + } + } + + private class RequestCompleter implements Runnable, TerminalTask { + private final Logger logger = + LoggerFactory.getLogger(RequestCompleter.class); + + @Override + public void run() { + try { + logger.trace("Completing {}-part request [upload ID: {}]", + completedParts.size(), uploadID); + + CompletedMultipartUpload completedMultipartUpload = + CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + CompleteMultipartUploadRequest completeMultipartUploadRequest = + CompleteMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .multipartUpload(completedMultipartUpload) + .build(); + client.completeMultipartUpload(completeMultipartUploadRequest); + setComplete(true); // CompletableOutputStream method + } catch (S3Exception e) { + logger.warn(e.getMessage()); + } finally { + if (observer != null) { + synchronized (instance) { + instance.notifyAll(); + } + } + } + } + } + + private class RequestAborter implements Runnable, TerminalTask { + private final Logger logger = + LoggerFactory.getLogger(RequestAborter.class); + + @Override + public void run() { + try { + logger.trace("Aborting multipart request [upload ID: {}]", + uploadID); + + AbortMultipartUploadRequest abortMultipartUploadRequest = + AbortMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .build(); + client.abortMultipartUpload(abortMultipartUploadRequest); + setComplete(false); + } catch (S3Exception e) { + logger.warn(e.getMessage()); + } finally { + if (observer != null) { + synchronized (instance) { + instance.notifyAll(); + } + } + } + } + } + + /** 5 MB is the minimum allowed by S3 for all but the last part. */ + public static final int MINIMUM_PART_LENGTH = 1024 * 1024 * 5; + + private final S3Client client; + private final String bucket, key, contentType; + + private ByteArrayOutputStream currentPart; + private final List completedParts = new ArrayList<>(); + private final Worker worker = new Worker(); + private boolean requestCreated; + + private String uploadID; + private int partIndex; + private long indexWithinPart; + + /** For an instance to wait for an upload notification during testing. */ + Object observer; + + /** Helps notify {@link #observer} of a completed upload during testing. */ + private final S3MultipartAsyncOutputStream instance; + + /** + * @param client Client. + * @param bucket Target bucket. + * @param key Target key. + * @param contentType Content type of the created object. + */ + public S3MultipartAsyncOutputStream(S3Client client, + String bucket, + String key, + String contentType) { + this.client = client; + this.bucket = bucket; + this.key = key; + this.contentType = contentType; + this.instance = this; + ThreadPool.getInstance().submit(worker); + } + + @Override + public void close() throws IOException { + if (isComplete()) { + worker.add(new PartUploader(getCurrentPart())); + // The worker will exit after running this. + worker.add(new RequestCompleter()); + } else { + // The worker will exit after running this. + worker.add(new RequestAborter()); + } + } + + @Override + public void write(int b) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b); + indexWithinPart++; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + @Override + public void write(byte[] b) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b); + indexWithinPart += b.length; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b, off, len); + indexWithinPart += len; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + private ByteArrayOutputStream getCurrentPart() { + if (currentPart == null) { + currentPart = new ByteArrayOutputStream(); + } + return currentPart; + } + + private void createRequestIfNecessary() { + if (!requestCreated) { + worker.add(new RequestCreator()); + requestCreated = true; + } + } + + private void uploadPartIfNecessary() { + if (indexWithinPart >= MINIMUM_PART_LENGTH) { + worker.add(new PartUploader(currentPart)); + IOUtils.closeQuietly(currentPart); + currentPart = null; + indexWithinPart = 0; + partIndex++; + } + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 6ecab5cf9..2ac19ce09 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -91,6 +91,7 @@ public enum Key { FILESYSTEMSOURCE_PATH_PREFIX("FilesystemSource.BasicLookupStrategy.path_prefix"), FILESYSTEMSOURCE_PATH_SUFFIX("FilesystemSource.BasicLookupStrategy.path_suffix"), HEALTH_DEPENDENCY_CHECK("endpoint.health.dependency_check"), + HEALTH_ENDPOINT_ENABLED("endpoint.health.enabled"), HEAPCACHE_PATHNAME("HeapCache.persist.filesystem.pathname"), HEAPCACHE_PERSIST("HeapCache.persist"), HEAPCACHE_TARGET_SIZE("HeapCache.target_size"), @@ -107,8 +108,11 @@ public enum Key { HTTPSOURCE_CHUNK_SIZE("HttpSource.chunking.chunk_size"), HTTPSOURCE_CHUNK_CACHE_ENABLED("HttpSource.chunking.cache.enabled"), HTTPSOURCE_CHUNK_CACHE_MAX_SIZE("HttpSource.chunking.cache.max_size"), + HTTPSOURCE_HTTP_PROXY_HOST("HttpSource.proxy.http.host"), + HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), HTTPSOURCE_LOOKUP_STRATEGY("HttpSource.lookup_strategy"), HTTPSOURCE_REQUEST_TIMEOUT("HttpSource.request_timeout"), + HTTPSOURCE_SEND_HEAD_REQUESTS("HttpSource.BasicLookupStrategy.send_head_requests"), HTTPSOURCE_URL_PREFIX("HttpSource.BasicLookupStrategy.url_prefix"), HTTPSOURCE_URL_SUFFIX("HttpSource.BasicLookupStrategy.url_suffix"), HTTPS_ENABLED("https.enabled"), @@ -170,6 +174,7 @@ public enum Key { PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY("processor.fallback_retrieval_strategy"), PROCESSOR_JPG_PROGRESSIVE("processor.jpg.progressive"), PROCESSOR_JPG_QUALITY("processor.jpg.quality"), + PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE("processor.purge_incompatible_from_source_cache"), PROCESSOR_SELECTION_STRATEGY("processor.selection_strategy"), PROCESSOR_SHARPEN("processor.sharpen"), PROCESSOR_STREAM_RETRIEVAL_STRATEGY("processor.stream_retrieval_strategy"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java index fd9adb6db..845ec7d67 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java @@ -54,6 +54,13 @@ public enum DelegateMethod { */ JDBCSOURCE_MEDIA_TYPE("jdbcsource_media_type"), + /** + * Called by {@link DelegateProxy#getJdbcSourceLastModified()}. + * + * @since 6.0 + */ + JDBCSOURCE_LAST_MODIFIED("jdbcsource_last_modified"), + /** * Called by {@link DelegateProxy#getJdbcSourceLookupSQL()}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java index 01730cb5d..1fee9ef7a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java @@ -87,6 +87,13 @@ Map getExtraIIIF3InformationResponseKeys() */ String getJdbcSourceDatabaseIdentifier() throws ScriptException; + /** + * @return Return value of {@link DelegateMethod#JDBCSOURCE_LAST_MODIFIED}. + * May be {@code null}. + * @since 6.0 + */ + String getJdbcSourceLastModified() throws ScriptException; + /** * @return Return value of {@link DelegateMethod#JDBCSOURCE_MEDIA_TYPE}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java index 90ed0ee46..6c9c91c14 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java @@ -26,6 +26,9 @@ public final class DelegateProxyService { private static final Logger LOGGER = LoggerFactory.getLogger(DelegateProxyService.class); + static final String DELEGATE_SCRIPT_VM_ARGUMENT = + "cantaloupe.delegate_script"; + private static DelegateProxyService instance; private static boolean isScriptCodeLoaded; @@ -95,23 +98,29 @@ public static synchronized DelegateProxyService getInstance() { } /** - * @return Absolute path representing the delegate script, regardless of - * whether the delegate script system is {@link #isScriptEnabled() - * enabled}; or {@code null} if {@link - * Key#DELEGATE_SCRIPT_PATHNAME} is not set. + *

Returns the absolute path to the delegate script, regardless of + * whether the delegate script system is {@link #isScriptEnabled() + * enabled}. The path is obtained from the {@link + * #DELEGATE_SCRIPT_VM_ARGUMENT delegate script VM argument}, if set, or + * the {@link Key#DELEGATE_SCRIPT_PATHNAME configuration} otherwise. If + * neither are set, set, {@code null} is returned.

+ * + *

The contents of the script are not validated.

+ * * @throws NoSuchFileException If the script specified in {@link * Key#DELEGATE_SCRIPT_PATHNAME} does not exist. */ static Path getScriptFile() throws NoSuchFileException { - final Configuration config = Configuration.getInstance(); - // The script name may be an absolute pathname or a filename. - final String configValue = - config.getString(Key.DELEGATE_SCRIPT_PATHNAME, ""); - if (!configValue.isEmpty()) { - Path script = findScript(configValue); + String value = System.getProperty("cantaloupe.delegate_script"); + if (value == null || value.isBlank()) { + final Configuration config = Configuration.getInstance(); + // The script name may be an absolute pathname or a filename. + value = config.getString(Key.DELEGATE_SCRIPT_PATHNAME, ""); + } + if (!value.isBlank()) { + Path script = findScript(value); if (!Files.exists(script)) { - throw new NoSuchFileException( - "File not found: " + script.toString()); + throw new NoSuchFileException("File not found: " + script); } return script; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java index d3a9fa949..cb3222ca6 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java @@ -252,6 +252,16 @@ public String getJdbcSourceDatabaseIdentifier() throws ScriptException { return (String) result; } + /** + * @return Return value of {@link DelegateMethod#JDBCSOURCE_LAST_MODIFIED}. + * @since 6.0 + */ + @Override + public String getJdbcSourceLastModified() throws ScriptException { + Object result = invoke(DelegateMethod.JDBCSOURCE_LAST_MODIFIED); + return (String) result; + } + /** * @return Return value of {@link DelegateMethod#JDBCSOURCE_MEDIA_TYPE}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java index ad0dd04b3..f460cc415 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java @@ -67,7 +67,14 @@ public interface JavaContext { * "Field2Name": value * ], * "xmp_string": "...", - * "xmp_model": https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html + * "xmp_model": See https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html, + * "xmp_elements": { + * "Field1Name": "value", + * "Field2Name": [ + * "value1", + * "value2" + * ] + * }, * "native": { * # structure varies * } @@ -83,6 +90,10 @@ public interface JavaContext { *
  • The {@code xmp_model} key contains a {@link * org.apache.jena.rdf.model.Model} object pre-loaded with the * contents of {@code xmp_string}.
    • + *
  • The {@code xmp_elements} key contains a view of the embedded XMP + * data as key-value pairs. This is convenient to use, but may not work + * correctly with all XMP fields—in particular, those that cannot + * be expressed as key-value pairs. *
  • The {@code native} key refers to format-specific metadata.
    • * * diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java index cba44062f..7d825ae44 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java @@ -179,6 +179,11 @@ public interface JavaDelegate { *
    String (required for HTTP Basic authentication)
    *
    {@code headers}
    *
    Map of request header name-value pairs (optional)
    + *
    {@code send_head_request}
    + *
    Boolean (optional). Defaults to {@code true}. See the + * documentation of the {@code + * HttpSource.BasicLookupStrategy.send_head_requests} + * configuration key.
    * * *
  • {@code null} if not found
    • @@ -199,13 +204,28 @@ public interface JavaDelegate { */ String getJDBCSourceDatabaseIdentifier(); + /** + *

    Returns either the last-modified timestamp of an image in ISO 8601 + * format, or an SQL statement that can be used to retrieve it from a + * {@code TIMESTAMP}-type column in the database. In the latter case, the + * {@code SELECT} and {@code FROM} clauses should be in uppercase in order + * to be autodetected.

    + * + *

    Implementing this method is optional, but may be necessary for + * certain features (like {@code Last-Modified} response headers) to + * work.

    + * + * @since 6.0 + */ + String getJDBCSourceLastModified(); + /** * Returns either the media (MIME) type of an image, or an SQL statement - * that can be used to retrieve it, if it is stored in the database. In the - * latter case, the {@code SELECT} and {@code FROM} clauses should be in - * uppercase in order to be autodetected. If {@code null} is returned, the - * media type will be inferred some other way, such as by identifier - * extension or magic bytes. + * that can be used to retrieve it from a {@code CHAR}-type column in the + * database. In the latter case, the {@code SELECT} and {@code FROM} + * clauses should be in uppercase in order to be autodetected. If {@code + * null} is returned, the media type will be inferred some other way, such + * as by identifier extension or magic bytes. */ String getJDBCSourceMediaType(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java index 79140f038..f13c939ca 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java @@ -63,6 +63,14 @@ public String getJdbcSourceDatabaseIdentifier() { return delegate.getJDBCSourceDatabaseIdentifier(); } + /** + * @since 6.0 + */ + @Override + public String getJdbcSourceLastModified() { + return delegate.getJDBCSourceLastModified(); + } + @Override public String getJdbcSourceLookupSQL() { return delegate.getJDBCSourceLookupSQL(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java index 21c9b37e3..ba049467e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java +++ b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java @@ -2,6 +2,7 @@ import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -73,6 +74,7 @@ private void initializeServer() { ServerConnector connector; HttpConfiguration config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2c = @@ -95,10 +97,11 @@ private void initializeServer() { // Initialize HTTPS. if (isHTTPS1Enabled || isHTTPS2Enabled) { config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.addCustomizer(new SecureRequestCustomizer()); - final SslContextFactory contextFactory = + final SslContextFactory.Server contextFactory = new SslContextFactory.Server(); contextFactory.setKeyStorePath(keyStorePath.toString()); contextFactory.setKeyStorePassword(keyStorePassword); diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java index 12682aee2..522f3c10e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java @@ -17,6 +17,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -205,11 +206,19 @@ public enum Serialization { * *

    Introduced in application version 5.0.

    */ - VERSION_4(4); + VERSION_4(4), + + /** + *

    Added a {@code serializationTimestamp} key containing an ISO 8601 + * timestamp in UTC.

    + * + *

    Introduced in application version 6.0.

    + */ + VERSION_5(5); private final int version; - static final Serialization CURRENT = VERSION_4; + static final Serialization CURRENT = VERSION_5; Serialization(int version) { this.version = version; @@ -220,11 +229,12 @@ int getVersion() { } } - private String appVersion = Application.getVersion(); + private String appVersion = Application.getVersion(); private Identifier identifier; private MediaType mediaType; - private Metadata metadata = new Metadata(); - private Serialization serialization = Serialization.CURRENT; + private Metadata metadata = new Metadata(); + private Serialization serialization = Serialization.CURRENT; + private Instant serializationTimestamp; /** * Ordered list of subimages. The main image is at index {@code 0}. @@ -268,19 +278,24 @@ public Info() { images.add(new Image()); } + /** + * N.B.: the {@link #getSerializationTimestamp() serialization timestamp} + * is not considered. + */ @Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj instanceof Info) { Info other = (Info) obj; - return Objects.equals(other.getApplicationVersion(), getApplicationVersion()) && - Objects.equals(other.getSerialization(), getSerialization()) && - Objects.equals(other.getIdentifier(), getIdentifier()) && - Objects.equals(other.getMetadata(), getMetadata()) && - Objects.equals(other.getSourceFormat(), getSourceFormat()) && - other.getNumResolutions() == getNumResolutions() && - other.getImages().equals(getImages()); + boolean a = Objects.equals(other.getApplicationVersion(), getApplicationVersion()); + boolean b = Objects.equals(other.getSerialization(), getSerialization()); + boolean c = Objects.equals(other.getIdentifier(), getIdentifier()); + boolean d = Objects.equals(other.getMetadata(), getMetadata()); + boolean e = Objects.equals(other.getSourceFormat(), getSourceFormat()); + boolean f = other.getNumResolutions() == getNumResolutions(); + boolean g = other.getImages().equals(getImages()); + return a && b && c && d && e && f && g; } return super.equals(obj); } @@ -367,6 +382,20 @@ public Serialization getSerialization() { return serialization; } + /** + * N.B.: Although it would be possible for most {@link + * edu.illinois.library.cantaloupe.cache.DerivativeCache}s to obtain a + * serialization timestamp by other means, such as e.g. filesystem + * attributes, storing it within the serialized instance makes a separate + * I/O call unnecessary. + * + * @return Timestamp that the instance was serialized. + * @since 6.0 + */ + public Instant getSerializationTimestamp() { + return serializationTimestamp; + } + /** * @return Size of the main image. */ @@ -484,6 +513,14 @@ public void setPersistable(boolean isComplete) { this.isPersistable = isComplete; } + /** + * @param timestamp Time at which the instance is serialized. + * @since 6.0 + */ + public void setSerializationTimestamp(Instant timestamp) { + this.serializationTimestamp = timestamp; + } + /** * @param version One of the {@link Serialization} versions. This value is * not serialized (the current serialization version is diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java b/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java index 0800eee0a..e88570eb7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import static edu.illinois.library.cantaloupe.image.InfoSerializer.APPLICATION_VERSION_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.IDENTIFIER_KEY; @@ -15,6 +16,7 @@ import static edu.illinois.library.cantaloupe.image.InfoSerializer.MEDIA_TYPE_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.METADATA_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.NUM_RESOLUTIONS_KEY; +import static edu.illinois.library.cantaloupe.image.InfoSerializer.SERIALIZATION_TIMESTAMP_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.SERIALIZATION_VERSION_KEY; /** @@ -28,10 +30,16 @@ final class InfoDeserializer extends JsonDeserializer { public Info deserialize(JsonParser parser, DeserializationContext deserializationContext) throws IOException { // N.B.: keys may or may not exist in different serializations, - // documented inline. Even for keys that are supposed to always exist, - // we have to check for them anyway because they may not exist in tests. - final Info info = new Info(); + // documented inline. Even keys that are supposed to always exist may + // not exist in tests, so we have to check for them anyway. + final Info info = new Info(); final JsonNode node = parser.getCodec().readTree(parser); + { // serializationTimestamp (does not exist in < 6.0 serializations) + JsonNode timestampNode = node.get(SERIALIZATION_TIMESTAMP_KEY); + if (timestampNode != null) { + info.setSerializationTimestamp(Instant.parse(timestampNode.textValue())); + } + } { // applicationVersion (does not exist in < 5.0 serializations) JsonNode appVersionNode = node.get(APPLICATION_VERSION_KEY); if (appVersionNode != null) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java b/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java index ba70a4499..f337ee27a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; /** * Serializes an {@link Info}. @@ -15,13 +16,14 @@ */ final class InfoSerializer extends JsonSerializer { - static final String APPLICATION_VERSION_KEY = "applicationVersion"; - static final String IDENTIFIER_KEY = "identifier"; - static final String IMAGES_KEY = "images"; - static final String MEDIA_TYPE_KEY = "mediaType"; - static final String METADATA_KEY = "metadata"; - static final String NUM_RESOLUTIONS_KEY = "numResolutions"; - static final String SERIALIZATION_VERSION_KEY = "serializationVersion"; + static final String APPLICATION_VERSION_KEY = "applicationVersion"; + static final String IDENTIFIER_KEY = "identifier"; + static final String IMAGES_KEY = "images"; + static final String MEDIA_TYPE_KEY = "mediaType"; + static final String METADATA_KEY = "metadata"; + static final String NUM_RESOLUTIONS_KEY = "numResolutions"; + static final String SERIALIZATION_TIMESTAMP_KEY = "serializationTimestamp"; + static final String SERIALIZATION_VERSION_KEY = "serializationVersion"; @Override public void serialize(Info info, @@ -29,19 +31,27 @@ public void serialize(Info info, SerializerProvider serializerProvider) throws IOException { generator.writeStartObject(); // application version - generator.writeStringField(APPLICATION_VERSION_KEY, Application.getVersion()); + generator.writeStringField(APPLICATION_VERSION_KEY, + Application.getVersion()); // serialization version - generator.writeNumberField(SERIALIZATION_VERSION_KEY, Info.Serialization.CURRENT.getVersion()); + generator.writeNumberField(SERIALIZATION_VERSION_KEY, + Info.Serialization.CURRENT.getVersion()); + // serialization timestamp + generator.writeStringField(SERIALIZATION_TIMESTAMP_KEY, + Instant.now().toString()); // identifier if (info.getIdentifier() != null) { - generator.writeStringField(IDENTIFIER_KEY, info.getIdentifier().toString()); + generator.writeStringField(IDENTIFIER_KEY, + info.getIdentifier().toString()); } // mediaType if (info.getMediaType() != null) { - generator.writeStringField(MEDIA_TYPE_KEY, info.getMediaType().toString()); + generator.writeStringField(MEDIA_TYPE_KEY, + info.getMediaType().toString()); } // numResolutions - generator.writeNumberField(NUM_RESOLUTIONS_KEY, info.getNumResolutions()); + generator.writeNumberField(NUM_RESOLUTIONS_KEY, + info.getNumResolutions()); // images generator.writeArrayFieldStart(IMAGES_KEY); info.getImages().forEach(image -> { diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index 2d04a64e6..8a2d26819 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -4,11 +4,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import edu.illinois.library.cantaloupe.Application; import edu.illinois.library.cantaloupe.image.exif.Directory; import edu.illinois.library.cantaloupe.image.exif.Tag; import edu.illinois.library.cantaloupe.image.iptc.DataSet; -import edu.illinois.library.cantaloupe.util.StringUtils; +import edu.illinois.library.cantaloupe.image.xmp.MapReader; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.NodeIterator; @@ -17,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -45,9 +46,6 @@ public class Metadata { private static final String XMP_ORIENTATION_PREDICATE = "http://ns.adobe.com/tiff/1.0/Orientation"; - private static final String XMP_TOOLKIT = Application.getName() + " " + - Application.getVersion(); - protected Directory exif; protected List iptcDataSets; protected String xmp; @@ -63,27 +61,6 @@ public class Metadata { */ private transient Orientation orientation; - /** - * Returns an XMP string encapsulated in an {@literal x:xmpmeta} element, - * which is itself encapsulated in an {@literal xpacket} PI. - * - * @param xmp XMP string with an {@literal rdf:RDF} root element. - * @return Encapsulated XMP data packet. - */ - public static String encapsulateXMP(String xmp) { - final StringBuilder b = new StringBuilder(); - b.append(""); - b.append(""); - b.append(xmp); - b.append(""); - // Append the magic trailer - b.append(" ".repeat(2048)); - b.append(""); - return b.toString(); - } - @Override public boolean equals(Object obj) { if (obj == this) { @@ -173,16 +150,32 @@ private void readOrientationFromXMP() { } /** - * Returns an RDF/XML string in UTF-8 encoding. The root element is - * {@literal rdf:RDF}, and there is no packet wrapper. - * - * @return XMP data packet. + * @return RDF/XML string in UTF-8 encoding. The root element is {@literal + * rdf:RDF}, and there is no packet wrapper. */ @JsonProperty public Optional getXMP() { return Optional.ofNullable(xmp); } + /** + * @return Map of elements found in the XMP data. If none are found, the + * map is empty. + */ + @JsonIgnore + public Map getXMPElements() { + loadXMP(); + if (xmpModel != null) { + try { + MapReader reader = new MapReader(xmpModel); + return reader.readElements(); + } catch (IOException e) { + LOGGER.warn("getXMPElements(): {}", e.getMessage()); + } + } + return Collections.emptyMap(); + } + /** * @return XMP model backed by the contents of {@link #getXMP()}. */ @@ -295,7 +288,7 @@ public void setXMP(byte[] xmp) { */ public void setXMP(String xmp) { if (xmp != null) { - this.xmp = StringUtils.trimXMP(xmp); + this.xmp = Utils.trimXMP(xmp); } else { this.xmp = null; this.xmpModel = null; @@ -310,7 +303,9 @@ public void setXMP(String xmp) { * { * "exif": See {@link Directory#toMap()}, * "iptc": See {@link DataSet#toMap()}, - * "xmp": "...", + * "xmp_string": "...", + * "xmp_model": [Jena model], + * "xmp_elements": {@link Map} * "native": String * }} * @@ -334,6 +329,7 @@ public Map toMap() { // XMP getXMP().ifPresent(xmp -> map.put("xmp_string", xmp)); getXMPModel().ifPresent(model -> map.put("xmp_model", model)); + map.put("xmp_elements", getXMPElements()); // Native metadata getNativeMetadata().ifPresent(nm -> map.put("native", nm)); return Collections.unmodifiableMap(map); diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java new file mode 100644 index 000000000..51b608fe6 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java @@ -0,0 +1,148 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; +import org.apache.jena.riot.RIOT; +import org.apache.jena.riot.RiotException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Extracts label-value pairs from an XMP model, making them available in a + * {@link Map}. + * + * @author Alex Dolski UIUC + * @since 6.0 + */ +public final class MapReader { + + private static final Logger LOGGER = + LoggerFactory.getLogger(MapReader.class); + + private static final Map PREFIXES = Map.ofEntries( + Map.entry("http://ns.adobe.com/camera-raw-settings/1.0/", "crs"), + Map.entry("http://purl.org/dc/elements/1.1/", "dc"), + Map.entry("http://purl.org/dc/terms/", "dcterms"), + Map.entry("http://ns.adobe.com/exif/1.0/", "exif"), + Map.entry("http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/", "Iptc4xmpCore"), + Map.entry("http://ns.adobe.com/iX/1.0/", "iX"), + Map.entry("http://ns.adobe.com/pdf/1.3/", "pdf"), + Map.entry("http://ns.adobe.com/photoshop/1.0/", "photoshop"), + Map.entry("http://ns.adobe.com/tiff/1.0/", "tiff"), + Map.entry("http://ns.adobe.com/xap/1.0/", "xmp"), + Map.entry("http://ns.adobe.com/xap/1.0/bj/", "xmpBJ"), + Map.entry("http://ns.adobe.com/xmp/1.0/DynamicMedia/", "xmpDM"), + Map.entry("http://ns.adobe.com/xmp/identifier/qual/1.0/", "xmpidq"), + Map.entry("http://ns.adobe.com/xap/1.0/mm/", "xmpMM"), + Map.entry("http://ns.adobe.com/xap/1.0/rights/", "xmpRights"), + Map.entry("http://ns.adobe.com/xap/1.0/t/pg/", "xmpTPg")); + + private final Model model; + private final Map elements = new TreeMap<>(); + private boolean hasReadElements; + + /** + * @param xmp XMP string. {@code } must be the root element. + * @see Utils#trimXMP + */ + public MapReader(String xmp) throws IOException { + RIOT.init(); + this.model = ModelFactory.createDefaultModel(); + try (StringReader reader = new StringReader(xmp)) { + model.read(reader, null, "RDF/XML"); + } catch (RiotException | NullPointerException e) { + // The XMP string may be invalid RDF/XML, or there may be a bug + // in Jena (that would be the NPE). Not much we can do. + throw new IOException(e); + } + } + + /** + * @param model XMP model, already initialized. + */ + public MapReader(Model model) { + this.model = model; + } + + public Map readElements() throws IOException { + if (!hasReadElements) { + StmtIterator it = model.listStatements(); + while (it.hasNext()) { + Statement stmt = it.next(); + //System.out.println(stmt.getSubject() + " " + stmt.getSubject().isAnon()); + //System.out.println(" " + stmt.getPredicate()); + //System.out.println(" " + stmt.getObject() + " " + stmt.getObject().isLiteral()); + //System.out.println("---------------------------"); + if (!stmt.getSubject().isAnon()) { + recurse(stmt); + } + } + LOGGER.trace("readElements(): read {} elements", elements.size()); + hasReadElements = true; + } + return Collections.unmodifiableMap(elements); + } + + private void recurse(Statement stmt) { + recurse(stmt, null); + } + + private void recurse(Statement stmt, String predicateOverride) { + String predicate = stmt.getPredicate().toString(); + if (stmt.getObject().isLiteral()) { + addElement(label(predicateOverride != null ? predicateOverride : predicate), + stmt.getObject().asLiteral().getValue()); + } else { + StmtIterator it = model.listStatements( + stmt.getObject().asResource(), null, (RDFNode) null); + while (it.hasNext()) { + Statement substmt = it.next(); + predicateOverride = null; + if (substmt.getPredicate().toString().matches("(.*)#_\\d+\\b")) { + predicateOverride = predicate; + } + recurse(substmt, predicateOverride); + } + } + } + + private void addElement(String label, Object value) { + if (elements.containsKey(label)) { + if (elements.get(label) instanceof List) { + @SuppressWarnings("unchecked") + List valueList = (List) elements.get(label); + valueList.add(value); + } else { + List valueList = new ArrayList<>(); + valueList.add(elements.get(label)); + valueList.add(value); + elements.put(label, valueList); + } + } else { + elements.put(label, value); + } + } + + private String label(String uri) { + for (Map.Entry entry : PREFIXES.entrySet()) { + if (uri.startsWith(entry.getKey())) { + String prefix = entry.getValue(); + String[] parts = uri.split("/"); + return prefix + ":" + parts[parts.length - 1]; + } + } + return uri; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java new file mode 100644 index 000000000..747d63aef --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java @@ -0,0 +1,44 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.Application; + +public final class Utils { + + private static final String XMP_TOOLKIT = Application.getName() + " " + + Application.getVersion(); + + /** + * Returns an XMP string encapsulated in an {@literal x:xmpmeta} element, + * which is itself encapsulated in an {@literal xpacket} PI. + * + * @param xmp XMP string with an {@literal rdf:RDF} root element. + * @return Encapsulated XMP data packet. + */ + public static String encapsulateXMP(String xmp) { + final StringBuilder b = new StringBuilder(); + b.append(""); + b.append(""); + b.append(xmp); + b.append(""); + // Append the magic trailer + b.append(" ".repeat(2048)); + b.append(""); + return b.toString(); + } + + /** + * Strips any enclosing tags or other content around the {@literal rdf:RDF} + * element within an RDF/XML XMP string. + */ + public static String trimXMP(String xmp) { + final int start = xmp.indexOf(" -1 && end > -1) { + xmp = xmp.substring(start, end + 10); + } + return xmp; + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java new file mode 100644 index 000000000..acd0e10b0 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java @@ -0,0 +1,6 @@ +/** + *

    Contains an XMP parser.

    + * + * @since 6.0 + */ +package edu.illinois.library.cantaloupe.image.xmp; \ No newline at end of file diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java index f575915e6..209ed6028 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java @@ -4,11 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.imageio.ImageIO; import javax.imageio.spi.IIORegistry; import javax.imageio.spi.ServiceRegistry; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; import java.util.ArrayList; import java.util.Iterator; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java index 0a4627f40..347d57e78 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java @@ -2,6 +2,7 @@ import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.image.Metadata; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.AbstractIIOImageWriter; import edu.illinois.library.cantaloupe.processor.codec.BufferedImageSequence; import edu.illinois.library.cantaloupe.processor.codec.ImageWriter; @@ -62,7 +63,7 @@ protected void addMetadata(final IIOMetadataNode baseTree) { final Metadata metadata = encode.getMetadata(); if (metadata != null) { metadata.getXMP().ifPresent(xmp -> { - xmp = Metadata.encapsulateXMP(xmp); + xmp = Utils.encapsulateXMP(xmp); // Get the /ApplicationExtensions node, creating it if necessary. final NodeList appExtensionsList = diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java index 945db16e5..9640511b0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import edu.illinois.library.cantaloupe.image.Metadata; -import edu.illinois.library.cantaloupe.util.StringUtils; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +71,7 @@ public Optional getXMP() { try { xmp = reader.getXMP(); if (xmp != null) { - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } catch (IOException e) { LOGGER.warn("getXMP(): {}", e.getMessage()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java index b4b0de716..c73507dd4 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java @@ -1,8 +1,7 @@ package edu.illinois.library.cantaloupe.processor.codec.jpeg; -import edu.illinois.library.cantaloupe.image.Metadata; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.util.ArrayUtils; -import edu.illinois.library.cantaloupe.util.StringUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.RDFNode; @@ -29,7 +28,7 @@ static byte[] assembleAPP1Segment(String xmp) { try { final ByteArrayOutputStream os = new ByteArrayOutputStream(); final byte[] headerBytes = Constants.STANDARD_XMP_SEGMENT_HEADER; - final byte[] xmpBytes = Metadata.encapsulateXMP(xmp). + final byte[] xmpBytes = Utils.encapsulateXMP(xmp). getBytes(StandardCharsets.UTF_8); // write segment marker os.write(Marker.APP1.marker()); @@ -61,12 +60,12 @@ static String assembleXMP(final List xmpChunks) throws IOException { final int numChunks = xmpChunks.size(); if (numChunks > 0) { standardXMP = new String(xmpChunks.get(0), StandardCharsets.UTF_8); - standardXMP = StringUtils.trimXMP(standardXMP); + standardXMP = Utils.trimXMP(standardXMP); if (numChunks > 1) { String extendedXMP = new String( mergeChunks(xmpChunks.subList(1, numChunks)), StandardCharsets.UTF_8); - extendedXMP = StringUtils.trimXMP(extendedXMP); + extendedXMP = Utils.trimXMP(extendedXMP); return mergeXMPModels(standardXMP, extendedXMP); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java index 2ab574dcd..4db63c068 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java @@ -2,11 +2,11 @@ import edu.illinois.library.cantaloupe.image.Rectangle; import edu.illinois.library.cantaloupe.image.ScaleConstraint; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.operation.ReductionFactor; import edu.illinois.library.cantaloupe.operation.Scale; import edu.illinois.library.cantaloupe.processor.SourceFormatException; import edu.illinois.library.cantaloupe.util.Stopwatch; -import edu.illinois.library.cantaloupe.util.StringUtils; import kdu_jni.Jp2_threadsafe_family_src; import kdu_jni.Jpx_codestream_source; import kdu_jni.Jpx_input_box; @@ -527,7 +527,7 @@ private void readMetadata() throws IOException { iptc = Arrays.copyOfRange(buffer, 16, buffer.length); } else if (isXMP) { xmp = new String(buffer, StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } finally { box.Close(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java index 3e2af6ab4..830e5a934 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.processor.codec.png; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.IIOMetadata; -import edu.illinois.library.cantaloupe.util.StringUtils; import org.w3c.dom.NodeList; import javax.imageio.metadata.IIOMetadataNode; @@ -71,7 +71,7 @@ public Optional getXMP() { .getAttribute("text") .getBytes(Charset.forName("UTF-8")); xmp = new String(xmpBytes, StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java index db2a8ec9a..077ab2517 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.image.exif.Directory; import edu.illinois.library.cantaloupe.image.iptc.DataSet; import edu.illinois.library.cantaloupe.image.iptc.Reader; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.IIOMetadata; -import edu.illinois.library.cantaloupe.util.StringUtils; import it.geosolutions.imageio.plugins.tiff.TIFFDirectory; import it.geosolutions.imageio.plugins.tiff.TIFFField; import org.slf4j.Logger; @@ -97,7 +97,7 @@ public Optional getXMP() { xmp = new String( (byte[]) xmpField.getData(), StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } return Optional.ofNullable(xmp); diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java index 20181b298..7b2c04e8d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java @@ -21,7 +21,7 @@ import edu.illinois.library.cantaloupe.util.StringUtils; import org.slf4j.Logger; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -41,7 +41,7 @@ * more of the HTTP-method-specific methods {@link #doGET()} etc., and may * optionally use {@link #doInit()} and {@link #destroy()}.

    * - *

    Unlike {@link javax.servlet.http.HttpServlet}s, instances are only used + *

    Unlike {@link jakarta.servlet.http.HttpServlet}s, instances are only used * once and not shared across threads.

    */ public abstract class AbstractResource { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java index cbad65f02..94551c415 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java @@ -1,8 +1,8 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java index 2223870ec..b84025a3d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java @@ -8,9 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java index d05cd3fa8..46e6d627f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java @@ -129,7 +129,7 @@ public void write(OutputStream responseOS) throws IOException { "cache simultaneously"); copyOrProcess(teeOS); cacheOS.flush(); - cacheOS.setCompletelyWritten(true); + cacheOS.setComplete(true); } else { copyOrProcess(responseOS); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java index b5e6f38b2..31f1cfb6f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java @@ -1,5 +1,6 @@ package edu.illinois.library.cantaloupe.resource; +import edu.illinois.library.cantaloupe.async.TaskQueue; import edu.illinois.library.cantaloupe.cache.CacheFacade; import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.config.Key; @@ -14,6 +15,7 @@ import edu.illinois.library.cantaloupe.processor.ProcessorFactory; import edu.illinois.library.cantaloupe.processor.SourceFormatException; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.status.HealthChecker; import edu.illinois.library.cantaloupe.source.Source; import edu.illinois.library.cantaloupe.source.SourceFactory; @@ -23,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -181,6 +184,13 @@ public interface Callback { */ boolean authorize() throws Exception; + /** + * Called immediately after the source image has first been accessed. + * + * @param result Information about the source image. + */ + void sourceAccessed(StatResult result); + /** * Called when image information is available; always before {@link * #willProcessImage(Processor, Info)} and {@link @@ -230,6 +240,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override @@ -335,7 +348,8 @@ public void handle(OutputStream outputStream) throws Exception { final Optional sourceImage = cacheFacade.getSourceCacheFile(identifier); if (sourceImage.isEmpty() || isResolvingFirst()) { try { - source.checkAccess(); + StatResult result = source.stat(); + callback.sourceAccessed(result); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. @@ -416,6 +430,23 @@ public void handle(OutputStream outputStream) throws Exception { format, identifier); } } + if (config.getBoolean(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, false)) { + TaskQueue.getInstance().submit(() -> { + try { + cacheFacade.getSourceCacheFile(identifier).ifPresent(file -> { + try { + getLogger().debug("Deleting {}", file); + Files.delete(file); + } catch (IOException e) { + getLogger().warn("Failed to delete file from source cache: {}", + e.getMessage()); + } + }); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } throw new SourceFormatException(); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java index 74dd9cd02..61e140b81 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java @@ -14,6 +14,7 @@ import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.source.Source; import edu.illinois.library.cantaloupe.source.SourceFactory; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +51,7 @@ public class InformationRequestHandler extends AbstractRequestHandler */ public static final class Builder { - private InformationRequestHandler handler; + private final InformationRequestHandler handler; private Builder(InformationRequestHandler handler) { this.handler = handler; @@ -151,7 +152,14 @@ public interface Callback { boolean authorize() throws Exception; /** - * Called when the list of available output formats have been obtained + * Called immediately after a source image has been accessed. + * + * @param result Information about the source image. + */ + void sourceAccessed(StatResult result); + + /** + * Called when a list of available output formats has been obtained * from a {@link Processor}. */ void knowAvailableOutputFormats(Set availableOutputFormats); @@ -168,6 +176,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult sourceAvailable) { + } + @Override public void knowAvailableOutputFormats(Set availableOutputFormats) { } }; @@ -249,7 +260,8 @@ public Info handle() throws Exception { final Optional optSrcImage = cacheFacade.getSourceCacheFile(identifier); if (optSrcImage.isEmpty() || isResolvingFirst()) { try { - source.checkAccess(); + StatResult result = source.stat(); + callback.sourceAccessed(result); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java index 55588eab4..9e36c9a38 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java @@ -6,7 +6,7 @@ import edu.illinois.library.cantaloupe.http.Query; import edu.illinois.library.cantaloupe.http.Reference; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java index 1b4fe0d45..4bfb4111e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java @@ -89,7 +89,7 @@ public final class Route { MAPPINGS.put(Pattern.compile("^" + CONFIGURATION_PATH + "$"), edu.illinois.library.cantaloupe.resource.api.ConfigurationResource.class); MAPPINGS.put(Pattern.compile("^" + HEALTH_PATH + "$"), - edu.illinois.library.cantaloupe.resource.api.HealthResource.class); + edu.illinois.library.cantaloupe.resource.health.HealthResource.class); MAPPINGS.put(Pattern.compile("^" + STATUS_PATH + "$"), edu.illinois.library.cantaloupe.resource.api.StatusResource.class); MAPPINGS.put(Pattern.compile("^" + TASKS_PATH + "$"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java index 6c5796ace..e41473d71 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java @@ -22,17 +22,13 @@ public void doInit() throws Exception { throw new EndpointDisabledException(); } - if (requiresAuth()) { - authenticateUsingBasic(BASIC_REALM, user -> { - final String configUser = config.getString(Key.API_USERNAME, ""); - if (!configUser.isEmpty() && configUser.equals(user)) { - return config.getString(Key.API_SECRET); - } - return null; - }); - } + authenticateUsingBasic(BASIC_REALM, user -> { + final String configUser = config.getString(Key.API_USERNAME, ""); + if (!configUser.isEmpty() && configUser.equals(user)) { + return config.getString(Key.API_SECRET); + } + return null; + }); } - abstract boolean requiresAuth(); - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java index a7d9b2e8d..5722814e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java @@ -24,6 +24,9 @@ @JsonSubTypes.Type( name = "PurgeInfoCache", value = PurgeInfoCacheCommand.class), + @JsonSubTypes.Type( + name = "PurgeInfosFromCache", + value = PurgeInfosFromCacheCommand.class), @JsonSubTypes.Type( name = "PurgeInvalidFromCache", value = PurgeInvalidFromCacheCommand.class), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java index b1f223c88..bed8838fe 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java @@ -77,9 +77,4 @@ public void doPUT() throws IOException { getResponse().setStatus(Status.NO_CONTENT.getCode()); } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java new file mode 100644 index 000000000..d0cbbb9c4 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java @@ -0,0 +1,24 @@ +package edu.illinois.library.cantaloupe.resource.api; + +import edu.illinois.library.cantaloupe.cache.CacheFacade; + +import java.util.concurrent.Callable; + +/** + * @since 6.0 + */ +final class PurgeInfosFromCacheCommand extends Command + implements Callable { + + @Override + public T call() throws Exception { + new CacheFacade().purgeInfos(); + return null; + } + + @Override + String getVerb() { + return "PurgeInfosFromCache"; + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java index 6feeb8d0f..1ed1bf0ff 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java @@ -43,9 +43,4 @@ public void doGET() throws IOException { .write(getResponse().getOutputStream(), features); } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java index dc5ab6af9..84deec047 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java @@ -52,9 +52,4 @@ public void doGET() throws Exception { } } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java index 4e0009df5..aae5ffe80 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java @@ -74,9 +74,4 @@ public void doPOST() throws Exception { } } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java similarity index 75% rename from src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java index 89d323dc2..854998c9f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java @@ -1,9 +1,11 @@ -package edu.illinois.library.cantaloupe.resource.api; +package edu.illinois.library.cantaloupe.resource.health; import com.fasterxml.jackson.databind.SerializationFeature; import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.config.Key; import edu.illinois.library.cantaloupe.http.Method; +import edu.illinois.library.cantaloupe.resource.AbstractResource; +import edu.illinois.library.cantaloupe.resource.EndpointDisabledException; import edu.illinois.library.cantaloupe.resource.JacksonRepresentation; import edu.illinois.library.cantaloupe.status.Health; import edu.illinois.library.cantaloupe.status.HealthChecker; @@ -16,7 +18,7 @@ /** * Provides health checks via the HTTP API. */ -public class HealthResource extends AbstractAPIResource { +public class HealthResource extends AbstractResource { private static final Logger LOGGER = LoggerFactory.getLogger(HealthResource.class); @@ -27,6 +29,17 @@ public class HealthResource extends AbstractAPIResource { private static final Map SERIALIZATION_FEATURES = Map.of(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); + @Override + public void doInit() throws Exception { + super.doInit(); + getResponse().setHeader("Cache-Control", "no-cache"); + + final Configuration config = Configuration.getInstance(); + if (!config.getBoolean(Key.HEALTH_ENDPOINT_ENABLED, false)) { + throw new EndpointDisabledException(); + } + } + @Override protected Logger getLogger() { return LOGGER; @@ -57,9 +70,4 @@ public void doGET() throws IOException { SERIALIZATION_FEATURES); } - @Override - boolean requiresAuth() { - return false; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java index 0bc631794..24e1b0b19 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java @@ -9,8 +9,21 @@ import edu.illinois.library.cantaloupe.operation.ScaleByPixels; import edu.illinois.library.cantaloupe.resource.PublicResource; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + public abstract class IIIFResource extends PublicResource { + protected void setLastModifiedHeader(Instant lastModified) { + getResponse().setHeader("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .format(lastModified)); + } + /** * When the size expressed in the endpoint URI is {@code max}, and the * resulting image dimensions are larger than {@link Key#MAX_PIXELS}, the diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java index 0b6ab0cbc..1bcb9fc55 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java @@ -12,6 +12,7 @@ import edu.illinois.library.cantaloupe.operation.Scale; import edu.illinois.library.cantaloupe.processor.Processor; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +77,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java similarity index 90% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java index 3320be6ed..62ada7bf1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java @@ -8,8 +8,8 @@ import java.util.List; /** - * Class whose instances are intended to be serialized to JSON for use in IIIF - * Image Information responses. + * Class whose instances are intended to be serialized as JSON for use in + * information responses. * * @see IIIF Image * API 1.1 @@ -19,7 +19,7 @@ @JsonPropertyOrder({ "@context", "@id", "width", "height", "scale_factors", "tile_width", "tile_height", "formats", "qualities", "profile" }) @JsonInclude(JsonInclude.Include.NON_NULL) -class ImageInfo { +final class Information { @JsonProperty("@context") public final String context = "http://library.stanford.edu/iiif/image-api/1.1/context.json"; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java similarity index 88% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java index c0ebeba89..9a63063df 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java @@ -12,7 +12,10 @@ import java.util.Set; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final int MIN_SIZE = 64; @@ -21,11 +24,11 @@ final class ImageInfoFactory { */ private static final int DEFAULT_MIN_TILE_SIZE = 512; - ImageInfo newImageInfo(final String imageURI, - final Set availableOutputFormats, - final Info info, - final int imageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final String imageURI, + final Set availableOutputFormats, + final Info info, + final int imageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -59,7 +62,7 @@ ImageInfo newImageInfo(final String imageURI, // Create an Info instance, which will eventually be serialized // to JSON and sent as the response body. - final ImageInfo imageInfo = new ImageInfo(); + final Information imageInfo = new Information(); imageInfo.id = imageURI; imageInfo.width = virtualSize.intWidth(); imageInfo.height = virtualSize.intHeight(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java index 079bc4b07..4a4c776de 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java @@ -14,6 +14,7 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -56,6 +57,14 @@ class CustomCallback implements InformationRequestHandler.Callback { public boolean authorize() throws Exception { return InformationResource.this.preAuthorize(); } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -72,13 +81,13 @@ public void knowAvailableOutputFormats(Set formats) { .build()) { try { Info info = handler.handle(); - ImageInfo iiifInfo = new ImageInfoFactory().newImageInfo( + Information iiifInfo = new InformationFactory().newImageInfo( getImageURI(), availableOutputFormats, info, getPageIndex(), getMetaIdentifier().getScaleConstraint()); - addHeaders(iiifInfo); + addHeaders(info, iiifInfo); new JacksonRepresentation(iiifInfo) .write(getResponse().getOutputStream()); } catch (ResourceException e) { @@ -92,10 +101,16 @@ public void knowAvailableOutputFormats(Set formats) { } } - private void addHeaders(ImageInfo info) { + private void addHeaders(Info info, Information iiifInfo) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedMediaType()); + // Link getResponse().setHeader("Link", - String.format("<%s>;rel=\"profile\";", info.profile)); + String.format("<%s>;rel=\"profile\";", iiifInfo.profile)); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + setLastModifiedHeader(info.getSerializationTimestamp()); + } } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java index 64aab388c..5673c699a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java @@ -15,6 +15,7 @@ import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; import edu.illinois.library.cantaloupe.resource.iiif.SizeRestrictedException; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +24,7 @@ import java.util.Map; /** - * Handles IIIF Image API 2.x image requests. + * Handles image requests. * * @see Image * Request Operations @@ -84,6 +85,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { if (Size.ScaleMode.MAX.equals(params.getSize().getScaleMode())) { @@ -162,8 +170,8 @@ private void validateSize(Dimension resultingSize, Dimension virtualSize) throws SizeRestrictedException { final var config = Configuration.getInstance(); if (config.getBoolean(Key.IIIF_RESTRICT_TO_SIZES, false)) { - var factory = new ImageInfoFactory(); - factory.getSizes(virtualSize).stream() + new InformationFactory().getSizes(virtualSize) + .stream() .filter(s -> s.width == resultingSize.intWidth() && s.height == resultingSize.intHeight()) .findAny() diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java similarity index 69% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java index 280e0890c..56538adbe 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java @@ -7,18 +7,18 @@ import java.util.List; /** - *

    Class whose instances are intended to be serialized to JSON for use in - * IIIF Image Information responses.

    + *

    Class whose instances are intended to be serialized as JSON for use in + * information responses.

    * - *

    Extends Map in order to support arbitrary keys, and LinkedHashMap in - * order to preserve key order.

    + *

    Extends {@link LinkedHashMap} in order to support arbitrary keys and + * preserve key order.

    * - * @see IIIF Image - * API 2.0: Image Information + * @see IIIF Image + * API 2.1: Image Information * @see jackson-databind * docs */ -class ImageInfo extends LinkedHashMap { +class Information extends LinkedHashMap { @JsonPropertyOrder({ "width", "height" }) public static final class Size { @@ -29,7 +29,7 @@ public static final class Size { public Size() {} public Size(Integer width, Integer height) { - this.width = width; + this.width = width; this.height = height; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java similarity index 91% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java index 3059b453b..aee1f8cc0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java @@ -24,10 +24,13 @@ import java.util.Map; import java.util.Set; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final Logger LOGGER = - LoggerFactory.getLogger(ImageInfoFactory.class); + LoggerFactory.getLogger(InformationFactory.class); /** * Will be used if {@link Key#IIIF_MIN_SIZE} is not set. @@ -52,7 +55,7 @@ final class ImageInfoFactory { private double maxScale; private int maxPixels, minSize, minTileSize; - ImageInfoFactory() { + InformationFactory() { var config = Configuration.getInstance(); maxPixels = config.getInt(Key.MAX_PIXELS, 0); maxScale = config.getDouble(Key.MAX_SCALE, Double.MAX_VALUE); @@ -70,11 +73,11 @@ final class ImageInfoFactory { * list. * @param scaleConstraint May be {@code null}. */ - ImageInfo newImageInfo(final Set processorOutputFormats, - final String imageURI, - final Info info, - final int infoImageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final Set processorOutputFormats, + final String imageURI, + final Info info, + final int infoImageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -89,7 +92,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // Create a Map instance, which will eventually be serialized to JSON // and returned in the response body. - final ImageInfo responseInfo = new ImageInfo<>(); + final Information responseInfo = new Information<>(); responseInfo.put("@context", "http://iiif.io/api/image/2/context.json"); responseInfo.put("@id", imageURI); responseInfo.put("protocol", "http://iiif.io/api/image"); @@ -98,7 +101,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // sizes -- this will be a 2^n series that will work for both multi- // and monoresolution images. - final List sizes = getSizes(virtualSize); + final List sizes = getSizes(virtualSize); responseInfo.put("sizes", sizes); // The max reduction factor is the maximum number of times the full @@ -115,7 +118,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // calculate a tile size close to minTileSize pixels. // Otherwise, use the smallest multiple of the tile size above that // of image resolution 0. - final List tiles = new ArrayList<>(); + final List tiles = new ArrayList<>(); responseInfo.put("tiles", tiles); info.getImages().forEach(image -> @@ -125,7 +128,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, minTileSize))); for (Dimension uniqueTileSize : uniqueTileSizes) { - final ImageInfo.Tile tile = new ImageInfo.Tile(); + final Information.Tile tile = new Information.Tile(); tile.width = (int) Math.ceil(uniqueTileSize.width()); tile.height = (int) Math.ceil(uniqueTileSize.height()); // Add every scale factor up to 2^RFmax. @@ -202,10 +205,10 @@ ImageInfo newImageInfo(final Set processorOutputFormats, * @param virtualSize Orientation-aware and {@link ScaleConstraint * scale-constrained} full size. */ - List getSizes(Dimension virtualSize) { + List getSizes(Dimension virtualSize) { // This will be a 2^n series that will work for both multi- and // monoresolution images. - final List sizes = new ArrayList<>(); + final List sizes = new ArrayList<>(); // The min reduction factor is the smallest number of reductions that // are required in order to fit within maxPixels. @@ -222,7 +225,7 @@ List getSizes(Dimension virtualSize) { i *= 2) { final int width = (int) Math.round(virtualSize.width() / i); final int height = (int) Math.round(virtualSize.height() / i); - sizes.add(0, new ImageInfo.Size(width, height)); + sizes.add(0, new Information.Size(width, height)); } return sizes; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java index 55e684951..19c695694 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java @@ -15,13 +15,14 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.script.ScriptException; /** - * Handles IIIF Image API 2.x information requests. + * Handles information requests. * * @see Information * Requests @@ -45,7 +46,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -63,6 +64,14 @@ class CustomCallback implements InformationRequestHandler.Callback { public boolean authorize() throws Exception { return InformationResource.this.preAuthorize(); } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -77,9 +86,9 @@ public void knowAvailableOutputFormats(Set formats) { .withRequestContext(getRequestContext()) .withCallback(new CustomCallback()) .build()) { - addHeaders(); try { Info info = handler.handle(); + addHeaders(info); newRepresentation(info, availableOutputFormats) .write(getResponse().getOutputStream()); } catch (ResourceException e) { @@ -93,8 +102,13 @@ public void knowAvailableOutputFormats(Set formats) { } } - private void addHeaders() { + private void addHeaders(Info info) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedMediaType()); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + setLastModifiedHeader(info.getSerializationTimestamp()); + } } /** @@ -123,16 +137,16 @@ private String getNegotiatedMediaType() { private JacksonRepresentation newRepresentation(Info info, Set availableOutputFormats) { - final ImageInfoFactory factory = new ImageInfoFactory(); + final InformationFactory factory = new InformationFactory(); factory.setDelegateProxy(getDelegateProxy()); - final ImageInfo imageInfo = factory.newImageInfo( + final Information iiifInfo = factory.newImageInfo( availableOutputFormats, getImageURI(), info, getPageIndex(), getMetaIdentifier().getScaleConstraint()); - return new JacksonRepresentation(imageInfo); + return new JacksonRepresentation(iiifInfo); } private JacksonRepresentation newHTTP4xxRepresentation( diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java index 815afe9d4..e97de343c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java @@ -17,6 +17,7 @@ import edu.illinois.library.cantaloupe.resource.ScaleRestrictedException; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; import edu.illinois.library.cantaloupe.resource.iiif.SizeRestrictedException; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,7 @@ import java.util.Map; /** - * Handles IIIF Image API 3.x image requests. + * Handles image requests. * * @see Image * Requests @@ -87,6 +88,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { if (Size.Type.MAX.equals(params.getSize().getType())) { @@ -204,8 +212,8 @@ private void validateSize(Dimension virtualSize, Dimension resultingSize) throws SizeRestrictedException { final Configuration config = Configuration.getInstance(); if (config.getBoolean(Key.IIIF_RESTRICT_TO_SIZES, false)) { - var factory = new ImageInfoFactory(); - factory.getSizes(virtualSize).stream() + new InformationFactory().getSizes(virtualSize) + .stream() .filter(s -> s.width == resultingSize.intWidth() && s.height == resultingSize.intHeight()) .findAny() diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java similarity index 87% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java index 1f08ff38e..67dcaf22f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java @@ -7,8 +7,8 @@ import java.util.List; /** - *

    Class whose instances are intended to be serialized to JSON for use in - * IIIF Image Information responses.

    + *

    Class whose instances are intended to be serialized as JSON for use in + * information responses.

    * *

    Extends {@link LinkedHashMap} in order to support arbitrary keys and * preserve key order.

    @@ -16,7 +16,7 @@ * @see IIIF * Image API 3.0: Technical Properties */ -class ImageInfo extends LinkedHashMap { +class Information extends LinkedHashMap { @JsonPropertyOrder({ "width", "height" }) public static final class Size { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java similarity index 90% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java index 800e0d1cd..4c10dfbff 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java @@ -22,10 +22,13 @@ import java.util.Set; import java.util.stream.Collectors; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final Logger LOGGER = - LoggerFactory.getLogger(ImageInfoFactory.class); + LoggerFactory.getLogger(InformationFactory.class); private static final String CONTEXT = "http://iiif.io/api/image/3/context.json"; @@ -50,7 +53,7 @@ final class ImageInfoFactory { private long maxPixels; private int minSize, minTileSize; - ImageInfoFactory() { + InformationFactory() { var config = Configuration.getInstance(); maxPixels = config.getInt(Key.MAX_PIXELS, 0); maxScale = config.getDouble(Key.MAX_SCALE, Double.MAX_VALUE); @@ -67,11 +70,11 @@ final class ImageInfoFactory { * list. * @param scaleConstraint May be {@code null}. */ - ImageInfo newImageInfo(final Set processorOutputFormats, - final String imageURI, - final Info info, - final int infoImageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final Set processorOutputFormats, + final String imageURI, + final Info info, + final int infoImageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -86,7 +89,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // Create an instance, which will later be serialized as JSON and // returned in the response body. - final ImageInfo responseInfo = new ImageInfo<>(); + final Information responseInfo = new Information<>(); responseInfo.put("@context", CONTEXT); responseInfo.put("id", imageURI); responseInfo.put("type", TYPE); @@ -162,10 +165,10 @@ ImageInfo newImageInfo(final Set processorOutputFormats, * @param virtualSize Orientation-aware and {@link ScaleConstraint * scale-constrained} full size. */ - List getSizes(Dimension virtualSize) { + List getSizes(Dimension virtualSize) { // This will be a 2^n series that will work for both multi- and // monoresolution images. - final List sizes = new ArrayList<>(); + final List sizes = new ArrayList<>(); // The min reduction factor is the smallest number of reductions that // are required in order to fit within maxPixels. @@ -182,7 +185,7 @@ List getSizes(Dimension virtualSize) { i *= 2) { final int width = (int) Math.round(virtualSize.width() / i); final int height = (int) Math.round(virtualSize.height() / i); - sizes.add(0, new ImageInfo.Size(width, height)); + sizes.add(0, new Information.Size(width, height)); } return sizes; } @@ -199,10 +202,10 @@ List getSizes(Dimension virtualSize) { * deliver (which may or may not match the physical tile size or a multiple * of it). */ - List getTiles(Dimension virtualSize, - Orientation orientation, - List images) { - final List tiles = new ArrayList<>(); + List getTiles(Dimension virtualSize, + Orientation orientation, + List images) { + final List tiles = new ArrayList<>(); final Set uniqueTileSizes = new HashSet<>(); images.forEach(image -> @@ -217,7 +220,7 @@ List getTiles(Dimension virtualSize, ImageInfoUtil.maxReductionFactor(virtualSize, minSize); for (Dimension uniqueTileSize : uniqueTileSizes) { - final ImageInfo.Tile tile = new ImageInfo.Tile(); + final Information.Tile tile = new Information.Tile(); tile.width = (int) Math.ceil(uniqueTileSize.width()); tile.height = (int) Math.ceil(uniqueTileSize.height()); // Add every scale factor up to 2^RFmax. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java index 383b1067f..125130cf9 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java @@ -15,6 +15,7 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +46,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -63,6 +64,14 @@ class CustomCallback implements InformationRequestHandler.Callback { public boolean authorize() throws Exception { return InformationResource.this.preAuthorize(); } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -77,9 +86,9 @@ public void knowAvailableOutputFormats(Set formats) { .withRequestContext(getRequestContext()) .withCallback(new CustomCallback()) .build()) { - addHeaders(); try { Info info = handler.handle(); + addHeaders(info); newRepresentation(info, availableOutputFormats) .write(getResponse().getOutputStream()); } catch (ResourceException e) { @@ -93,8 +102,13 @@ public void knowAvailableOutputFormats(Set formats) { } } - private void addHeaders() { + private void addHeaders(Info info) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedContentType()); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + setLastModifiedHeader(info.getSerializationTimestamp()); + } } /** @@ -125,16 +139,16 @@ private String getNegotiatedContentType() { private JacksonRepresentation newRepresentation(Info info, Set availableOutputFormats) { - final ImageInfoFactory factory = new ImageInfoFactory(); + final InformationFactory factory = new InformationFactory(); factory.setDelegateProxy(getDelegateProxy()); - final ImageInfo imageInfo = factory.newImageInfo( + final Information iiifInfo = factory.newImageInfo( availableOutputFormats, getImageURI(), info, getPageIndex(), getMetaIdentifier().getScaleConstraint()); - return new JacksonRepresentation(imageInfo); + return new JacksonRepresentation(iiifInfo); } private JacksonRepresentation newHTTP4xxRepresentation( diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java index cf3a57cbc..af45a0aa2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java @@ -226,8 +226,11 @@ private static synchronized CloudBlobClient getClientInstance() { } @Override - public void checkAccess() throws IOException { - fetchBlob(); + public StatResult stat() throws IOException { + CloudBlockBlob blob = fetchBlob(); + StatResult result = new StatResult(); + result.setLastModified(blob.getProperties().getLastModified().toInstant()); + return result; } private CloudBlockBlob fetchBlob() throws IOException { @@ -275,6 +278,7 @@ String getBlobKey() throws IOException { if (objectKey == null) { final LookupStrategy strategy = LookupStrategy.from(Key.AZURESTORAGESOURCE_LOOKUP_STRATEGY); + //noinspection SwitchStatementWithTooFewBranches switch (strategy) { case DELEGATE_SCRIPT: try { diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java index d2b4a4bb2..ab6cafd6a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java @@ -118,7 +118,7 @@ public T next() { private Path path; @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { final Path file = getFile(); if (!Files.exists(file)) { throw new NoSuchFileException("Failed to resolve " + @@ -126,6 +126,9 @@ public void checkAccess() throws IOException { } else if (!Files.isReadable(file)) { throw new AccessDeniedException("File is not readable: " + file); } + StatResult result = new StatResult(); + result.setLastModified(Files.getLastModifiedTime(file).toInstant()); + return result; } /** @@ -139,6 +142,7 @@ public Path getFile() throws IOException { if (path == null) { final LookupStrategy strategy = LookupStrategy.from(Key.FILESYSTEMSOURCE_LOOKUP_STRATEGY); + //noinspection SwitchStatementWithTooFewBranches switch (strategy) { case DELEGATE_SCRIPT: try { @@ -169,7 +173,7 @@ private Path getPathWithBasicStrategy() { final String suffix = config.getString(Key.FILESYSTEMSOURCE_PATH_SUFFIX, ""); final Identifier sanitizedId = sanitizedIdentifier(); - return Paths.get(prefix + sanitizedId.toString() + suffix); + return Paths.get(prefix + sanitizedId + suffix); } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java b/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java index fa3d2033d..1b4bebc0d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java @@ -10,27 +10,7 @@ class HTTPRequestInfo { private final Headers headers = new Headers(); private String uri, username, secret; - - HTTPRequestInfo(String uri) { - this.uri = uri; - } - - HTTPRequestInfo(String uri, String username, String secret) { - this(uri); - this.username = username; - this.secret = secret; - } - - HTTPRequestInfo(String uri, - String username, - String secret, - Map headers) { - this(uri, username, secret); - if (headers != null) { - headers.forEach((key, value) -> - this.headers.add(key, value.toString())); - } - } + private boolean isSendingHeadRequest = true; String getBasicAuthToken() { if (getUsername() != null && getSecret() != null) { @@ -57,6 +37,35 @@ String getUsername() { return username; } + boolean isSendingHeadRequest() { + return isSendingHeadRequest; + } + + void setHeaders(Map headers) { + if (headers != null) { + headers.forEach((key, value) -> + this.headers.add(key, value.toString())); + } else { + this.headers.clear(); + } + } + + void setSecret(String secret) { + this.secret = secret; + } + + void setSendingHeadRequest(boolean isUsingHeadRequest) { + this.isSendingHeadRequest = isUsingHeadRequest; + } + + void setURI(String uri) { + this.uri = uri; + } + + void setUsername(String username) { + this.username = username; + } + @Override public String toString() { return getURI() + ""; diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index 6d3ef2293..e9c2af51d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -19,6 +19,8 @@ import javax.net.ssl.X509TrustManager; import javax.script.ScriptException; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; @@ -28,9 +30,14 @@ import java.security.SecureRandom; import java.security.cert.X509Certificate; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; @@ -66,12 +73,16 @@ * issues the following server requests:

    * *
      - *
    1. {@literal HEAD}
      2. - *
    2. If server supports ranges: + *
    3. If {@link Key#HTTPSOURCE_SEND_HEAD_REQUESTS} is {@code true}, or + * the delegate method returns {@code true} for the equivalent key, a + * {@code HEAD} request. Otherwise, a ranged {@code GET} request specifying + * a small range of the beginning of the resource.
      4. + *
    4. If a {@code HEAD} request was sent: *
        - *
      1. If {@link FormatIterator#next()} } needs to check magic + *
      2. If {@link FormatIterator#next()} needs to check magic bytes, + * and the server supports ranges: *
          - *
        1. Ranged {@literal GET}
          2. + *
        2. Ranged {@code GET}
          3. *
        *
        3. *
      3. If {@link HTTPStreamFactory#newSeekableStream()} is used: @@ -83,7 +94,7 @@ *
        4. *
      4. Else if {@link HTTPStreamFactory#newInputStream()} is used: *
          - *
        1. {@literal GET} to retrieve the full image bytes
          2. + *
        2. {@code GET} to retrieve the full image bytes
          3. *
        *
        5. *
      @@ -112,18 +123,31 @@ class HttpSource extends AbstractSource implements Source { /** - * Encapsulates the status code and headers of a HEAD response. + * Encapsulates the status code, headers, and body (if available) of a + * {code HEAD} or ranged {@code GET} response. The range specifies a small + * part of the beginning of the resource to use for the purpose of + * inferring its format. */ - private static class HEADResponseInfo { + private static class ResourceInfo { - int status; - Headers headers; + private String requestMethod; + private int status; + private Headers headers; - static HEADResponseInfo fromResponse(Response response) - throws IOException { - HEADResponseInfo info = new HEADResponseInfo(); - info.status = response.code(); - info.headers = response.headers(); + /** + * Response entity with a maximum length of {@link #RANGE_LENGTH}. + */ + private byte[] entity; + + static ResourceInfo fromResponse(Response response) throws IOException { + ResourceInfo info = new ResourceInfo(); + info.requestMethod = response.request().method(); + info.status = response.code(); + info.headers = response.headers(); + if ("GET".equals(response.request().method()) && + response.body() != null) { + info.entity = response.body().bytes(); + } return info; } @@ -131,37 +155,13 @@ boolean acceptsRanges() { return "bytes".equals(headers.get("Accept-Ranges")); } - long getContentLength() { + long contentLength() { String value = headers.get("Content-Length"); return (value != null) ? Long.parseLong(value) : 0; } - } - - /** - * Encapsulates the status code, headers, and body of a ranged GET - * response. The range specifies a small part of the beginning of the - * resource to use for the purpose of inferring its format. - */ - private static class RangedGETResponseInfo extends HEADResponseInfo { - - private static final int RANGE_LENGTH = 32; - - /** - * Ranged response entity, with a maximum length of {@link - * #RANGE_LENGTH}. - */ - private byte[] entity; - - static RangedGETResponseInfo fromResponse(Response response) - throws IOException { - RangedGETResponseInfo info = new RangedGETResponseInfo(); - info.status = response.code(); - info.headers = response.headers(); - if (response.body() != null) { - info.entity = response.body().bytes(); - } - return info; + String contentType() { + return headers.get("Content-Type"); } Format detectFormat() throws IOException { @@ -175,6 +175,18 @@ Format detectFormat() throws IOException { return format; } + Instant lastModified() { + String str = headers.get("Last-Modified"); + if (str != null) { + TemporalAccessor ta = DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .parse(str); + return Instant.from(ta); + } + return null; + } + } /** @@ -183,13 +195,13 @@ Format detectFormat() throws IOException { * extension, the format is inferred from that.
      5. *
    5. Otherwise, if the identifier contains a recognized filename * extension, the format is inferred from that.
      6. - *
    6. Otherwise, if a {@literal Content-Type} header is present in the - * {@link #fetchHEADResponseInfo() HEAD response}, and its value is - * specific enough (not {@literal application/octet-stream}, for + *
    7. Otherwise, if a {@code Content-Type} header is present in the + * {@link #getResourceInfo HEAD response}, and its value is + * specific enough (not {@code application/octet-stream}, for * example), a format is inferred from that.
      8. - *
    8. Otherwise, if the {@literal HEAD} response contains an {@literal - * Accept-Ranges: bytes} header, a {@literal GET} request is sent with - * a {@literal Range} header specifying a small range of data from the + *
    9. Otherwise, if the {@literal HEAD} response contains an {@code + * Accept-Ranges: bytes} header, a {@code GET} request is sent with a + * {@code Range} header specifying a small range of data from the * beginning of the resource, and a format is inferred from the magic * bytes in the response entity.
      10. *
    10. Otherwise, {@link Format#UNKNOWN} is returned.
      11. @@ -200,36 +212,41 @@ Format detectFormat() throws IOException { class FormatIterator implements Iterator { /** - * Infers a {@link Format} based on the {@literal Content-Type} header in - * the {@link #fetchHEADResponseInfo() HEAD response}. + * Infers a {@link Format} based on the {@code Content-Type} header in + * the initial response.. */ private class ContentTypeHeaderChecker implements FormatChecker { /** - * @return Format from the {@literal Content-Type} header, or {@link + * @return Format from the {@code Content-Type} header, or {@link * Format#UNKNOWN} if that header is missing or invalid. */ @Override public Format check() { try { final HTTPRequestInfo requestInfo = getRequestInfo(); - final HEADResponseInfo responseInfo = fetchHEADResponseInfo(); + final ResourceInfo resourceInfo = getResourceInfo(); - if (responseInfo.status >= 200 && responseInfo.status < 300) { - String field = responseInfo.headers.get("Content-Type"); - if (field != null) { - Format format = MediaType.fromContentType(field).toFormat(); + if (resourceInfo.status >= 200 && resourceInfo.status < 300) { + String value = resourceInfo.contentType(); + if (value != null) { + Format format = MediaType.fromContentType(value).toFormat(); if (Format.UNKNOWN.equals(format)) { - LOGGER.debug("Unrecognized Content-Type header value for HEAD {}: {}", - requestInfo.getURI(), field); + LOGGER.debug("Unrecognized Content-Type header value for {} {}: {}", + resourceInfo.requestMethod, + requestInfo.getURI(), + value); } return format; } else { - LOGGER.debug("No Content-Type header for HEAD {}", + LOGGER.debug("No Content-Type header for {} {}", + resourceInfo.requestMethod, requestInfo.getURI()); } } else { - LOGGER.debug("HEAD {} returned status {}", - requestInfo.getURI(), responseInfo.status); + LOGGER.debug("{} {} returned status {}", + resourceInfo.requestMethod, + requestInfo.getURI(), + resourceInfo.status); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); @@ -240,10 +257,12 @@ public Format check() { private class ByteChecker implements FormatChecker { /** - * If the {@link #fetchHEADResponseInfo() HEAD response} contains an - * {@literal Accept-Ranges: bytes} header, issues an HTTP {@literal GET} - * request for a small {@literal Range} of the beginning of the resource - * and checks the magic bytes in the response body. + * If the {@link #getResourceInfo initial response} is from a + * {@code HEAD} request, issues an HTTP {@code GET} request for a + * small range of the beginning of the resource. (If it is a {@code + * GET} response, that data has already been received.) Then, a + * source format is inferred from the magic bytes in the response + * entity. * * @return Inferred source format, or {@link Format#UNKNOWN}. */ @@ -251,28 +270,43 @@ private class ByteChecker implements FormatChecker { public Format check() { try { final HTTPRequestInfo requestInfo = getRequestInfo(); - if (fetchHEADResponseInfo().acceptsRanges()) { - final RangedGETResponseInfo responseInfo - = fetchRangedGETResponseInfo(); - if (responseInfo.status >= 200 && responseInfo.status < 300) { - Format format = responseInfo.detectFormat(); - if (!Format.UNKNOWN.equals(format)) { - LOGGER.debug("Inferred {} format from magic bytes for GET {}", - format, requestInfo.getURI()); - return format; - } else { - LOGGER.debug("Unable to infer a format from magic bytes for GET {}", - requestInfo.getURI()); - } + // If a request hasn't yet been sent, send one. This may be + // a HEAD or a ranged GET. It's not safe to send a ranged + // GET without first checking (via HEAD) whether the + // resource supports ranges--unless we are told via the + // configuration not to send HEADs. + if (resourceInfo == null) { + resourceInfo = getResourceInfo(); + } + // If it was a HEAD, we need to know whether the resource + // supports ranged requests. If it does, send one. + if ("HEAD".equals(resourceInfo.requestMethod)) { + if (resourceInfo.acceptsRanges()) { + resourceInfo = fetchResourceInfoViaGET(); } else { - LOGGER.debug("GET {} returned status {}", - requestInfo.getURI(), responseInfo.status); + LOGGER.debug("Server did not supply an " + + "`Accept-Ranges: bytes` header in response " + + "to HEAD {}, and all other attempts to "+ + "infer a format failed.", + requestInfo.getURI()); + return Format.UNKNOWN; + } + } + if (resourceInfo.status >= 200 && resourceInfo.status < 300) { + Format format = resourceInfo.detectFormat(); + if (!Format.UNKNOWN.equals(format)) { + LOGGER.debug("Inferred {} format from magic bytes for GET {}", + format, requestInfo.getURI()); + return format; + } else { + LOGGER.debug("Unable to infer a format from magic bytes for GET {}", + requestInfo.getURI()); } } else { - LOGGER.info("Server did not supply an " + - "`Accept-Ranges: bytes` header for HEAD {}, and all " + - "other attempts to infer a format failed.", - requestInfo.getURI()); + LOGGER.debug("{} {} returned status {}", + resourceInfo.requestMethod, + requestInfo.getURI(), + resourceInfo.status); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); @@ -336,29 +370,38 @@ public Format check() { static final Logger LOGGER = LoggerFactory.getLogger(HttpSource.class); + static final String USER_AGENT = String.format( + "%s/%s (%s/%s; java/%s; %s/%s)", + HttpSource.class.getSimpleName(), + Application.getVersion(), + Application.getName(), + Application.getVersion(), + System.getProperty("java.version"), + System.getProperty("os.name"), + System.getProperty("os.version")); + private static final int DEFAULT_REQUEST_TIMEOUT = 30; + private static final int RANGE_LENGTH = 32; private static OkHttpClient httpClient; /** - * Cached {@link #fetchHEADResponseInfo() HEAD response info}. - */ - private HEADResponseInfo headResponseInfo; - - /** - * Cached {@link #fetchRangedGETResponseInfo() ranged GET response info}. + * Cached by {@link #getRequestInfo()}. */ - private RangedGETResponseInfo rangedGETResponseInfo; + private HTTPRequestInfo requestInfo; /** - * Cached by {@link #getRequestInfo()}. + * Cached {@link #getResourceInfo resource info} from the initial request, + * which may be either a {@code HEAD} or ranged {@code GET}, depending on + * the configuration. */ - private HTTPRequestInfo requestInfo; + private ResourceInfo resourceInfo; - private final FormatIterator formatIterator = new FormatIterator<>(); + private final FormatIterator formatIterator = + new FormatIterator<>(); /** - * @return Shared instance. + * @return Already-initialized instance shared by all threads. */ static synchronized OkHttpClient getHTTPClient() { if (httpClient == null) { @@ -367,12 +410,24 @@ static synchronized OkHttpClient getHTTPClient() { .connectTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .readTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .writeTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS); - final Configuration config = Configuration.getInstance(); - final boolean allowInsecure = config.getBoolean( - Key.HTTPSOURCE_ALLOW_INSECURE, false); - if (allowInsecure) { + final String proxyHost = + config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST, ""); + if (!proxyHost.isBlank()) { + final int proxyPort = + config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT); + if (proxyPort == 0) { + throw new RuntimeException("Proxy port setting " + + Key.HTTPSOURCE_HTTP_PROXY_PORT + " must be set"); + } + LOGGER.debug("Using HTTP proxy: {}:{}", proxyHost, proxyPort); + Proxy httpProxy = new Proxy(Proxy.Type.HTTP, + new InetSocketAddress(proxyHost, proxyPort)); + builder.proxy(httpProxy); + } + + if (config.getBoolean(Key.HTTPSOURCE_ALLOW_INSECURE, false)) { try { X509TrustManager[] tm = new X509TrustManager[]{ new X509TrustManager() { @@ -412,17 +467,6 @@ private static Duration getRequestTimeout() { return Duration.ofSeconds(timeout); } - static String getUserAgent() { - return String.format("%s/%s (%s/%s; java/%s; %s/%s)", - HttpSource.class.getSimpleName(), - Application.getVersion(), - Application.getName(), - Application.getVersion(), - System.getProperty("java.version"), - System.getProperty("os.name"), - System.getProperty("os.version")); - } - /** * @see #request(HTTPRequestInfo, String, Map) */ @@ -447,7 +491,7 @@ static Response request(HTTPRequestInfo requestInfo, Request.Builder builder = new Request.Builder() .method(method, null) .url(requestInfo.getURI()) - .addHeader("User-Agent", getUserAgent()); + .addHeader("User-Agent", USER_AGENT); // Add credentials. if (requestInfo.getUsername() != null && requestInfo.getSecret() != null) { @@ -477,10 +521,9 @@ static String toString(Headers headers) { } @Override - public void checkAccess() throws IOException { - fetchHEADResponseInfo(); - - final int status = headResponseInfo.status; + public StatResult stat() throws IOException { + ResourceInfo info = getResourceInfo(); + final int status = info.status; if (status >= 400) { final String statusLine = "HTTP " + status; if (status == 404 || status == 410) { // not found or gone @@ -491,6 +534,9 @@ public void checkAccess() throws IOException { throw new IOException(statusLine); } } + StatResult result = new StatResult(); + result.setLastModified(info.lastModified()); + return result; } @Override @@ -498,72 +544,46 @@ public FormatIterator getFormatIterator() { return formatIterator; } - @Override - public StreamFactory newStreamFactory() throws IOException { - HTTPRequestInfo info; - try { - info = getRequestInfo(); - } catch (IOException e) { - throw e; - } catch (Exception e) { - LOGGER.error("newStreamFactory(): {}", e.getMessage()); - throw new IOException(e); - } - - if (info != null) { - LOGGER.debug("Resolved {} to {}", identifier, info.getURI()); - fetchHEADResponseInfo(); - return new HTTPStreamFactory( - info, - headResponseInfo.getContentLength(), - headResponseInfo.acceptsRanges()); - } - return null; - } - /** - *

      Issues a {@literal HEAD} request and caches parts of the response in - * {@link #headResponseInfo}.

      + * Issues a {@code HEAD} or ranged {@code GET} request (depending on the + * configuration) and caches the result in {@link #resourceInfo}. */ - private HEADResponseInfo fetchHEADResponseInfo() throws IOException { - if (headResponseInfo == null) { - try (Response response = request("HEAD")) { - headResponseInfo = HEADResponseInfo.fromResponse(response); + private ResourceInfo getResourceInfo() throws IOException { + if (resourceInfo == null) { + try { + requestInfo = getRequestInfo(); + if (requestInfo.isSendingHeadRequest()) { + fetchResourceInfoViaHEAD(); + } else { + fetchResourceInfoViaGET(); + } + } catch (Exception e) { + LOGGER.error("fetchResourceInfo(): {}", e.getMessage()); + throw new IOException(e.getMessage(), e); } } - return headResponseInfo; + return resourceInfo; } - /** - *

      Issues a {@literal GET} request specifying a small range of data and - * caches parts of the response in {@link #rangedGETResponseInfo}.

      - */ - private RangedGETResponseInfo fetchRangedGETResponseInfo() - throws IOException { - if (rangedGETResponseInfo == null) { - Map extraHeaders = Map.of("Range", - "bytes=0-" + (RangedGETResponseInfo.RANGE_LENGTH - 1)); - try (Response response = request("GET", extraHeaders)) { - rangedGETResponseInfo = - RangedGETResponseInfo.fromResponse(response); - } + private ResourceInfo fetchResourceInfoViaHEAD() throws Exception { + requestInfo = getRequestInfo(); + try (Response response = request("HEAD", Collections.emptyMap())) { + resourceInfo = ResourceInfo.fromResponse(response); } - return rangedGETResponseInfo; + return resourceInfo; } - private Response request(String method) throws IOException { - return request(method, Collections.emptyMap()); + private ResourceInfo fetchResourceInfoViaGET() throws Exception { + requestInfo = getRequestInfo(); + var extraHeaders = Map.of("Range", "bytes=0-" + (RANGE_LENGTH - 1)); + try (Response response = request("GET", extraHeaders)) { + resourceInfo = ResourceInfo.fromResponse(response); + } + return resourceInfo; } private Response request(String method, Map extraHeaders) throws IOException { - HTTPRequestInfo requestInfo; - try { - requestInfo = getRequestInfo(); - } catch (Exception e) { - LOGGER.error("request(): {}", e.getMessage()); - throw new IOException(e.getMessage(), e); - } return request(requestInfo, method, extraHeaders); } @@ -575,33 +595,33 @@ HTTPRequestInfo getRequestInfo() throws Exception { if (requestInfo == null) { final LookupStrategy strategy = LookupStrategy.from(Key.HTTPSOURCE_LOOKUP_STRATEGY); - switch (strategy) { - case DELEGATE_SCRIPT: - requestInfo = getRequestInfoUsingScriptStrategy(); - break; - default: - requestInfo = getRequestInfoUsingBasicStrategy(); - break; + if (LookupStrategy.DELEGATE_SCRIPT.equals(strategy)) { + requestInfo = newRequestInfoUsingScriptStrategy(); + } else { + requestInfo = newRequestInfoUsingBasicStrategy(); } } return requestInfo; } - private HTTPRequestInfo getRequestInfoUsingBasicStrategy() { - final Configuration config = Configuration.getInstance(); + private HTTPRequestInfo newRequestInfoUsingBasicStrategy() { + final var config = Configuration.getInstance(); final String prefix = config.getString(Key.HTTPSOURCE_URL_PREFIX, ""); final String suffix = config.getString(Key.HTTPSOURCE_URL_SUFFIX, ""); - return new HTTPRequestInfo( - prefix + identifier.toString() + suffix, - config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME), - config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + + final HTTPRequestInfo info = new HTTPRequestInfo(); + info.setURI(prefix + identifier.toString() + suffix); + info.setUsername(config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME)); + info.setSecret(config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + info.setSendingHeadRequest(config.getBoolean(Key.HTTPSOURCE_SEND_HEAD_REQUESTS, true)); + return info; } /** * @throws NoSuchFileException if the remote resource was not found. * @throws ScriptException if the delegate method throws an exception. */ - private HTTPRequestInfo getRequestInfoUsingScriptStrategy() + private HTTPRequestInfo newRequestInfoUsingScriptStrategy() throws NoSuchFileException, ScriptException { final DelegateProxy proxy = getDelegateProxy(); final Map result = proxy.getHttpSourceResourceInfo(); @@ -612,13 +632,44 @@ private HTTPRequestInfo getRequestInfoUsingScriptStrategy() " returned nil for " + identifier); } - final String uri = (String) result.get("uri"); - final String username = (String) result.get("username"); - final String secret = (String) result.get("secret"); + final String uri = (String) result.get("uri"); + final String username = (String) result.get("username"); + final String secret = (String) result.get("secret"); @SuppressWarnings("unchecked") - final Map headers = (Map) result.get("headers"); + final Map headers = (Map) result.get("headers"); + final boolean isHeadEnabled = !result.containsKey("send_head_request") || + (boolean) result.get("send_head_request"); + + final HTTPRequestInfo info = new HTTPRequestInfo(); + info.setURI(uri); + info.setUsername(username); + info.setSecret(secret); + info.setHeaders(headers); + info.setSendingHeadRequest(isHeadEnabled); + return info; + } - return new HTTPRequestInfo(uri, username, secret, headers); + @Override + public StreamFactory newStreamFactory() throws IOException { + HTTPRequestInfo info; + try { + info = getRequestInfo(); + } catch (IOException e) { + throw e; + } catch (Exception e) { + LOGGER.error("newStreamFactory(): {}", e.getMessage()); + throw new IOException(e); + } + + if (info != null) { + LOGGER.debug("Resolved {} to {}", identifier, info.getURI()); + getResourceInfo(); + return new HTTPStreamFactory( + info, + resourceInfo.contentLength(), + resourceInfo.acceptsRanges()); + } + return null; } @Override @@ -628,9 +679,8 @@ public void setIdentifier(Identifier identifier) { } private void reset() { - requestInfo = null; - headResponseInfo = null; - rangedGETResponseInfo = null; + requestInfo = null; + resourceInfo = null; } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java index faca97960..d937a391b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java @@ -21,6 +21,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -28,7 +30,7 @@ /** *

      Maps an identifier to a binary/BLOB field in a relational database.

      * - *

      A custom schema is not required; any schema will work. However, several + *

      A custom schema is not required; most schemas will work. However, several * delegate methods must be implemented in order to obtain the information * needed to run the SQL queries.

      * @@ -211,23 +213,11 @@ static synchronized Connection getConnection() throws SQLException { } @Override - public void checkAccess() throws IOException { - try (Connection connection = getConnection()) { - final String sql = getLookupSQL(); - - // Check that the image exists. - try (PreparedStatement statement = connection.prepareStatement(sql)) { - statement.setString(1, getDatabaseIdentifier()); - LOGGER.debug(sql); - try (ResultSet result = statement.executeQuery()) { - if (!result.next()) { - throw new NoSuchFileException(sql); - } - } - } - } catch (ScriptException | SQLException e) { - throw new IOException(e); - } + public StatResult stat() throws IOException { + Instant lastModified = getLastModified(); + StatResult result = new StatResult(); + result.setLastModified(lastModified); + return result; } /** @@ -243,6 +233,41 @@ public FormatIterator getFormatIterator() { return formatIterator; } + Instant getLastModified() throws IOException { + try { + String methodResult = getDelegateProxy().getJdbcSourceLastModified(); + if (methodResult != null) { + // the delegate method result may be an ISO 8601 string, or an + // SQL statement to look it up. + if (methodResult.toUpperCase().startsWith("SELECT")) { + // It's called readability, IntelliJ! + //noinspection UnnecessaryLocalVariable + final String sql = methodResult; + LOGGER.debug(sql); + try (Connection connection = getConnection(); + PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, getDatabaseIdentifier()); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + Timestamp value = resultSet.getTimestamp(1); + if (value != null) { + return value.toInstant(); + } + } else { + throw new NoSuchFileException(sql); + } + } + } + } else { + return Instant.parse(methodResult); + } + } + } catch (SQLException | ScriptException e) { + throw new IOException(e); + } + return null; + } + /** * @return Result of the {@link DelegateMethod#JDBCSOURCE_LOOKUP_SQL} * method. diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java index 6c9e49d62..3d06f7807 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java @@ -36,7 +36,7 @@ class S3HTTPImageInputStreamClient implements HTTPImageInputStreamClient { @Override public Response sendHEADRequest() throws IOException { - final S3Client client = S3Source.getClientInstance(); + final S3Client client = S3Source.getClientInstance(objectInfo); final String bucket = objectInfo.getBucketName(); final String key = objectInfo.getKey(); try { diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java index cfaa8afe6..bf11539ca 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java @@ -1,19 +1,32 @@ package edu.illinois.library.cantaloupe.source; +/** + * Contains information needed to access an object in S3. + */ final class S3ObjectInfo { - private String bucketName, key; + private String region, endpoint, accessKeyID, secretAccessKey, bucketName, + key; private long length = -1; - S3ObjectInfo(String key, String bucketName) { - this.key = key; - this.bucketName = bucketName; + /** + * @return Access key ID. May be {@code null}. + */ + String getAccessKeyID() { + return accessKeyID; } String getBucketName() { return bucketName; } + /** + * @return Service endpoint URI. May be {@code null}. + */ + String getEndpoint() { + return endpoint; + } + String getKey() { return key; } @@ -22,13 +35,61 @@ long getLength() { return length; } + /** + * @return Endpoint AWS region. Only used by AWS endpoints. May be {@code + * null}. + */ + String getRegion() { + return region; + } + + /** + * @return Secret access key. May be {@code null}. + */ + String getSecretAccessKey() { + return secretAccessKey; + } + + void setAccessKeyID(String accessKeyID) { + this.accessKeyID = accessKeyID; + } + + void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + void setKey(String key) { + this.key = key; + } + void setLength(long length) { this.length = length; } + void setRegion(String region) { + this.region = region; + } + + void setSecretAccessKey(String secretAccessKey) { + this.secretAccessKey = secretAccessKey; + } + @Override public String toString() { - return getBucketName() + "/" + getKey(); + StringBuilder b = new StringBuilder(); + b.append("[endpoint: ").append(getEndpoint()).append("] "); + b.append("[region: ").append(getRegion()).append("] "); + String tmp = getAccessKeyID() != null ? "******" : "null"; + b.append("[accessKeyID: ").append(tmp).append("] "); + tmp = getSecretAccessKey() != null ? "******" : "null"; + b.append("[secretAccessKey: ").append(tmp).append("] "); + b.append("[bucket: ").append(getBucketName()).append("] "); + b.append("[key: ").append(getKey()).append("]"); + return b.toString(); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java index 6b18d98a0..93db8621f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java @@ -28,6 +28,8 @@ import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; +import java.time.Instant; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -79,13 +81,12 @@ *
    * * @author Alex Dolski UIUC - * @see - * Minio Java Client API Reference */ final class S3Source extends AbstractSource implements Source { private static class S3ObjectAttributes { String contentType; + Instant lastModified; long length; } @@ -189,11 +190,16 @@ public T next() { LoggerFactory.getLogger(S3Source.class); /** - * Range used to infer the source image format. + * Byte range used to infer the source image format. */ private static final Range FORMAT_INFERENCE_RANGE = new Range(0, 32); - private static S3Client client; + /** + * The keys are endpoint URIs. The default client's key is {@code null}. + * This is not thread-safe, so should only be accessed via {@link + * #getClientInstance(S3ObjectInfo)}. + */ + private static final Map CLIENTS = new HashMap<>(); /** * Cached by {@link #getObjectInfo()}. @@ -207,25 +213,44 @@ public T next() { private FormatIterator formatIterator = new FormatIterator<>(); - static synchronized S3Client getClientInstance() { + static synchronized S3Client getClientInstance(S3ObjectInfo info) { + String endpoint = info.getEndpoint(); + S3Client client = CLIENTS.get(endpoint); if (client == null) { final Configuration config = Configuration.getInstance(); - final String endpointStr = config.getString(Key.S3SOURCE_ENDPOINT); + if (endpoint == null) { + endpoint = config.getString(Key.S3SOURCE_ENDPOINT); + } + // Convert the endpoint string into a URI which is required by the + // client builder. URI endpointURI = null; - if (endpointStr != null) { + if (endpoint != null) { try { - endpointURI = new URI(endpointStr); + endpointURI = new URI(endpoint); } catch (URISyntaxException e) { LOGGER.error("Invalid URI for {}: {}", Key.S3SOURCE_ENDPOINT, e.getMessage()); } } + String region = info.getRegion(); + if (region == null) { + region = config.getString(Key.S3SOURCE_REGION); + } + String accessKeyID = info.getAccessKeyID(); + if (accessKeyID == null) { + accessKeyID = config.getString(Key.S3SOURCE_ACCESS_KEY_ID); + } + String secretAccessKey = info.getSecretAccessKey(); + if (secretAccessKey == null) { + secretAccessKey = config.getString(Key.S3SOURCE_SECRET_KEY); + } client = new S3ClientBuilder() - .accessKeyID(config.getString(Key.S3SOURCE_ACCESS_KEY_ID)) - .secretAccessKey(config.getString(Key.S3SOURCE_SECRET_KEY)) + .accessKeyID(accessKeyID) + .secretAccessKey(secretAccessKey) .endpointURI(endpointURI) - .region(config.getString(Key.S3SOURCE_REGION)) + .region(region) .build(); + CLIENTS.put(endpoint, client); } return client; } @@ -248,7 +273,7 @@ static InputStream newObjectInputStream(S3ObjectInfo info) */ static InputStream newObjectInputStream(S3ObjectInfo info, Range range) throws IOException { - final S3Client client = getClientInstance(); + final S3Client client = getClientInstance(info); try { GetObjectRequest request; if (range != null) { @@ -275,8 +300,11 @@ static InputStream newObjectInputStream(S3ObjectInfo info, } @Override - public void checkAccess() throws IOException { - getObjectAttributes(); + public StatResult stat() throws IOException { + S3ObjectAttributes attrs = getObjectAttributes(); + StatResult result = new StatResult(); + result.setLastModified(attrs.lastModified); + return result; } @Override @@ -290,14 +318,15 @@ private S3ObjectAttributes getObjectAttributes() throws IOException { final S3ObjectInfo info = getObjectInfo(); final String bucket = info.getBucketName(); final String key = info.getKey(); - final S3Client client = getClientInstance(); + final S3Client client = getClientInstance(info); try { HeadObjectResponse response = client.headObject(HeadObjectRequest.builder() .bucket(bucket) .key(key) .build()); - objectAttributes = new S3ObjectAttributes(); - objectAttributes.length = response.contentLength(); + objectAttributes = new S3ObjectAttributes(); + objectAttributes.length = response.contentLength(); + objectAttributes.lastModified = response.lastModified(); } catch (NoSuchBucketException | NoSuchKeyException e) { throw new NoSuchFileException(info.toString()); } catch (S3Exception e) { @@ -323,6 +352,7 @@ private S3ObjectAttributes getObjectAttributes() throws IOException { */ S3ObjectInfo getObjectInfo() throws IOException { if (objectInfo == null) { + //noinspection SwitchStatementWithTooFewBranches switch (LookupStrategy.from(Key.S3SOURCE_LOOKUP_STRATEGY)) { case DELEGATE_SCRIPT: try { @@ -349,7 +379,10 @@ private S3ObjectInfo getObjectInfoUsingBasicStrategy() { final String keyPrefix = config.getString(Key.S3SOURCE_PATH_PREFIX, ""); final String keySuffix = config.getString(Key.S3SOURCE_PATH_SUFFIX, ""); final String key = keyPrefix + identifier.toString() + keySuffix; - return new S3ObjectInfo(key, bucketName); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucketName); + info.setKey(key); + return info; } /** @@ -372,12 +405,18 @@ private S3ObjectInfo getObjectInfoUsingDelegateStrategy() } if (result.containsKey("bucket") && result.containsKey("key")) { - String bucketName = result.get("bucket"); - String objectKey = result.get("key"); - return new S3ObjectInfo(objectKey, bucketName); + final S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(result.get("bucket")); + info.setKey(result.get("key")); + // These may be null. + info.setRegion(result.get("region")); + info.setEndpoint(result.get("endpoint")); + info.setAccessKeyID(result.get("access_key_id")); + info.setSecretAccessKey(result.get("secret_access_key")); + return info; } else { throw new IllegalArgumentException( - "Returned hash does not include bucket and key"); + "Returned hash must include bucket and key"); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/Source.java index bc21d30f9..f3901a31e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/Source.java @@ -20,7 +20,7 @@ *
      *
    1. {@link #setIdentifier(Identifier)} and * {@link #setDelegateProxy(DelegateProxy)} (in either order)
      2. - *
    2. {@link #checkAccess()}
      3. + *
    3. {@link #stat()}
      4. *
    4. Any other methods
      5. *
    5. {@link #shutdown()}
      6. *
    @@ -39,17 +39,19 @@ public interface Source { void setIdentifier(Identifier identifier); /** - *

    Checks the accessibility of the source image.

    + *

    Checks the accessibility of the source image and returns some limited + * metadata.

    * *

    Will be called only once.

    * + * @return Instance with as many of its properties set as possible. * @throws NoSuchFileException if an image corresponding to the set * identifier does not exist. * @throws AccessDeniedException if an image corresponding to the set * identifier is not readable. * @throws IOException if there is some other issue accessing the image. */ - void checkAccess() throws IOException; + StatResult stat() throws IOException; /** * N.B.: This default implementation throws an {@link @@ -104,10 +106,11 @@ default Path getFile() throws IOException { default void shutdown() {} /** - * N.B. 1: This method's return value affects the behavior of {@link - * #getFile()}. See the documentation of that method for more information. + *

    N.B. 1: This method's return value affects the behavior of {@link + * #getFile()}. See the documentation of that method for more + * information.

    * - * N.B. 2: The default implementation returns {@code false}. + *

    N.B. 2: The default implementation returns {@code false}.

    * * @return Whether the source image can be accessed via a {@link * java.nio.file.Path}. diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java b/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java new file mode 100644 index 000000000..f9f5710da --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java @@ -0,0 +1,20 @@ +package edu.illinois.library.cantaloupe.source; + +import java.time.Instant; + +/** + * Holds some limited metadata about a source image. + */ +public final class StatResult { + + private Instant lastModified; + + public Instant getLastModified() { + return lastModified; + } + + void setLastModified(Instant lastModified) { + this.lastModified = lastModified; + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java b/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java index a751586bb..ce9b21600 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java +++ b/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java @@ -17,7 +17,8 @@ public final class ArrayUtils { * @return Chunked data. */ public static List chunkify(byte[] bytes, int maxChunkSize) { - final List chunks = new ArrayList<>(10); + final int listSize = (int) Math.ceil(bytes.length / (double) maxChunkSize); + final List chunks = new ArrayList<>(listSize); if (bytes.length <= maxChunkSize) { chunks.add(bytes); } else { diff --git a/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java b/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java index c8e4c38c5..015bf0d60 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java +++ b/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java @@ -217,19 +217,6 @@ public static long toByteSize(String str) { return Math.round(number * Math.pow(1024, exponent)); } - /** - * Strips any enclosing tags or other content around the {@literal rdf:RDF} - * element within an RDF/XML XMP string. - */ - public static String trimXMP(String xmp) { - final int start = xmp.indexOf(" -1 && end > -1) { - xmp = xmp.substring(start, end + 10); - } - return xmp; - } - /** * Returns an array of strings, one for each line in the string after it * has been wrapped to fit lines of maxWidth. Lines end with any diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 134f11b17..8890f1648 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -238,7 +238,9 @@ data-content="Location of the delegate script. This can be an absolute path, or a filename; if only a filename is specified, it will be searched for in the same folder as the config file, - and then the current working directory.">? + and then the current working directory. The delegate script + pathname can also be set using the <code>-Dcantaloupe.delegate_script</code> + VM argument, which overrides this value.">?

    Health Check

    + + + + + + + + + + + + - - - - + + + + + + + +
    + ? + +
    + +
    +
    + Proxy Host + ? + + +
    + Proxy Port + ? + + +
    Lookup Strategy
    URL Prefix + Basic Lookup Strategy: URL Prefix
    URL Suffix + Basic Lookup Strategy: URL Suffix
    Basic Auth Username + Basic Lookup Strategy: Basic Auth Username
    Basic Auth Secret + Basic Lookup Strategy: Basic Auth Secret
    + Basic Lookup Strategy: HEAD Requests + ? + +
    + +
    +
    Chunking
    + ? + +
    + +
    +
    DPI { @@ -216,7 +234,7 @@ void testNewDerivativeImageOutputStream() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // Wait for it to upload @@ -245,7 +263,7 @@ void testNewDerivativeImageOutputStreamDoesNotLeaveDetritusWhenStreamIsIncomplet try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(false); // the whole point of the test + outputStream.setComplete(false); // the whole point of the test } // Wait for it to upload @@ -292,7 +310,7 @@ void testPurge() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add the info @@ -335,7 +353,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList1)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.put(id1, new Info()); @@ -348,7 +366,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList2)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.put(id2, new Info()); @@ -388,7 +406,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Seed another derivative image @@ -399,7 +417,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } Thread.sleep(ASYNC_WAIT); @@ -414,13 +432,60 @@ void testPurgeWithOperationList() throws Exception { assertExists(instance, ops2); } + /* purgeInfos() */ + + @Test + void testPurgeInfos() throws Exception { + DerivativeCache instance = newInstance(); + Identifier identifier = new Identifier(IMAGE); + OperationList opList = OperationList.builder() + .withIdentifier(identifier) + .withOperations(new Encode(Format.get("jpg"))) + .build(); + Info info = new Info(); + + // assert that a particular image doesn't exist + try (InputStream is = instance.newDerivativeImageInputStream(opList)) { + assertNull(is); + } + + // assert that a particular info doesn't exist + assertFalse(instance.getInfo(identifier).isPresent()); + + // add the image + try (CompletableOutputStream outputStream = + instance.newDerivativeImageOutputStream(opList)) { + Path fixture = TestUtil.getImage(IMAGE); + Files.copy(fixture, outputStream); + outputStream.setComplete(true); + } + + // add the info + instance.put(identifier, info); + + Thread.sleep(ASYNC_WAIT); + + // assert that they've been added + assertExists(instance, opList); + assertNotNull(instance.getInfo(identifier)); + + // purge infos + instance.purgeInfos(); + + // assert that the info has been purged + assertFalse(instance.getInfo(identifier).isPresent()); + + // assert that the image has NOT been purged + assertExists(instance, opList); + } + /* purgeInvalid() */ @Test void testPurgeInvalid() throws Exception { DerivativeCache instance = newInstance(); - Identifier id1 = new Identifier(IMAGE); - OperationList ops1 = OperationList.builder() + Identifier id1 = new Identifier(IMAGE); + OperationList ops1 = OperationList.builder() .withIdentifier(id1) .withOperations(new Encode(Format.get("jpg"))) .build(); @@ -432,7 +497,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add an Info @@ -455,7 +520,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(fixture2, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add another info diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java index 13c490d59..e9111a778 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java @@ -189,7 +189,7 @@ void testNewDerivativeImageInputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -217,7 +217,7 @@ void testNewDerivativeImageOutputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { assertNotNull(os); - os.setCompletelyWritten(true); + os.setComplete(true); } } @@ -253,7 +253,7 @@ void testPurge() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -299,7 +299,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -343,7 +343,7 @@ void testPurgeAsyncWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -378,7 +378,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -392,6 +392,34 @@ void testPurgeWithOperationList() throws Exception { } } + /* purgeInfos() */ + + @Test + void testPurgeInfos() throws Exception { + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.DERIVATIVE_CACHE_TTL, 1); + + enableDerivativeCache(); + DerivativeCache derivCache = CacheFactory.getDerivativeCache().get(); + + Identifier identifier = new Identifier("jpg"); + Info info = new Info(); + + // Add info to the derivative cache. + derivCache.put(identifier, info); + + // Assert that everything has been added. + assertNotNull(derivCache.getInfo(identifier)); + + instance.purgeInfos(); + + Thread.sleep(ASYNC_WAIT); + + // Assert that it's gone. + assertEquals(0, InfoService.getInstance().getInfoCache().size()); + assertFalse(derivCache.getInfo(identifier).isPresent()); + } + /* purgeInvalid() */ @Test @@ -419,7 +447,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java index 747efba7e..39e9abffb 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -59,7 +60,7 @@ public void setUp() throws Exception { public void tearDown() throws IOException { try { Files.walkFileTree(fixturePath, new DeletingFileVisitor()); - } catch (DirectoryNotEmptyException e) { + } catch (NoSuchFileException | DirectoryNotEmptyException e) { // This happens in Windows 7 (maybe other versions?) sometimes; not // sure why, but it shouldn't result in a test failure. System.err.println(e.getMessage()); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java index 42a73409d..cb34460c9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java @@ -98,7 +98,7 @@ void testDumpToPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.dumpToPersistentStore(); @@ -127,7 +127,7 @@ void testGetByteSize() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } assertEquals(5439, instance.getByteSize()); @@ -231,7 +231,7 @@ void testLoadFromPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.dumpToPersistentStore(); @@ -296,7 +296,7 @@ void testPurgeExcessWithExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } assertEquals(5439, instance.getByteSize()); @@ -317,7 +317,7 @@ void testPurgeExcessWithNoExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } long size = instance.getByteSize(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java index edbfdb998..83895cb70 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java @@ -106,7 +106,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } Crop crop = new CropByPixels(50, 50, 50, 50); @@ -119,7 +119,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } crop = new CropByPixels(10, 20, 50, 90); @@ -132,7 +132,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // persist some infos corresponding to the above images diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java b/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java index 79c2a6ae2..989b2b110 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java @@ -42,6 +42,11 @@ public void purge(OperationList opList) throws IOException { throw new IOException("I'm broken"); } + @Override + public void purgeInfos() throws IOException { + throw new IOException("I'm broken"); + } + @Override public void purgeInvalid() throws IOException { throw new IOException("I'm broken"); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java b/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java index 1ce5b75b7..7611af407 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java @@ -13,7 +13,7 @@ class MockCache implements DerivativeCache, SourceCache { private boolean isCleanUpCalled, isInitializeCalled, isOnCacheWorkerCalled, - isPurgeInvalidCalled, isShutdownCalled; + isPurgeInfosCalled, isPurgeInvalidCalled, isShutdownCalled; @Override public void cleanUp() { @@ -48,6 +48,10 @@ boolean isOnCacheWorkerCalled() { return isOnCacheWorkerCalled; } + public boolean isPurgeInfosCalled() { + return isPurgeInfosCalled; + } + boolean isPurgeInvalidCalled() { return isPurgeInvalidCalled; } @@ -89,6 +93,11 @@ public void purge(Identifier identifier) {} @Override public void purge(OperationList opList) {} + @Override + public void purgeInfos() { + isPurgeInfosCalled = true; + } + @Override public void purgeInvalid() { isPurgeInvalidCalled = true; diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index d900a64b3..6f00ec903 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -249,7 +249,7 @@ void testNewDerivativeImageInputStreamUpdatesLastModifiedTime() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Wait for it to finish, hopefully. @@ -302,7 +302,7 @@ void testPurgeWithKeyPrefix() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // Add a cached info @@ -379,7 +379,7 @@ void testPurgeInvalidWithKeyPrefix() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add a cached Info diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java new file mode 100644 index 000000000..da3a1d60b --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java @@ -0,0 +1,318 @@ +package edu.illinois.library.cantaloupe.cache; + +import edu.illinois.library.cantaloupe.test.BaseTest; +import edu.illinois.library.cantaloupe.test.ConfigurationConstants; +import edu.illinois.library.cantaloupe.test.TestUtil; +import edu.illinois.library.cantaloupe.util.S3ClientBuilder; +import edu.illinois.library.cantaloupe.util.S3Utils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.SecureRandom; + +import static org.junit.jupiter.api.Assertions.*; + +class S3MultipartAsyncOutputStreamTest extends BaseTest { + + private static S3Client client; + + @BeforeAll + public static void beforeClass() throws Exception { + BaseTest.beforeClass(); + S3Utils.createBucket(client(), getBucket()); + } + + @AfterAll + public static void afterClass() throws Exception { + BaseTest.afterClass(); + if (client != null) { + client.close(); + } + } + + private static synchronized S3Client client() { + if (client == null) { + client = new S3ClientBuilder() + .endpointURI(getEndpoint()) + .region(getRegion()) + .accessKeyID(getAccessKeyId()) + .secretAccessKey(getSecretKey()) + .build(); + } + return client; + } + + private static void delete(String key) { + DeleteObjectRequest request = DeleteObjectRequest.builder() + .bucket(getBucket()) + .key(key) + .build(); + client.deleteObject(request); + } + + private static String getAccessKeyId() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_ACCESS_KEY_ID.getKey()); + } + + private static String getBucket() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_BUCKET.getKey()); + } + + private static URI getEndpoint() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + String endpointStr = testConfig.getString(ConfigurationConstants.S3_ENDPOINT.getKey()); + if (endpointStr != null && !endpointStr.isBlank()) { + try { + return new URI(endpointStr); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + return null; + } + + private static String getRegion() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_REGION.getKey()); + } + + private static String getSecretKey() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_SECRET_KEY.getKey()); + } + + private static byte[] readBytes(String key) throws IOException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(getBucket()) + .key(key) + .build(); + ResponseInputStream is = client().getObject(request); + return is.readAllBytes(); + } + + @Test + void closeMarksInstanceComplete() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/closeMarksInstanceComplete"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] bytes = new byte[1024 * 1024]; + new SecureRandom().nextBytes(bytes); + instance.write(bytes); + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + assertTrue(instance.isComplete()); + } finally { + delete(key); + } + } + + @Test + void write1WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write1WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + for (byte b : expectedBytes) { + instance.write(b); + } + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + @Test + void write1WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write1WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + for (byte b : expectedBytes) { + instance.write(b); + } + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object larger than the part size is written correctly. + */ + @Test + void write2WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write2WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + instance.write(expectedBytes); + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object smaller than the part size is written correctly. + */ + @Test + void write2WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write2WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + instance.write(expectedBytes); + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object larger than the part size is written correctly. + */ + @Test + void write3WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write3WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + instance.write(expectedBytes, 0, expectedBytes.length); + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object smaller than the part size is written correctly. + */ + @Test + void write3WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write3WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + instance.write(expectedBytes, 0, expectedBytes.length); + instance.setComplete(true); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + +} diff --git a/src/test/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyServiceTest.java b/src/test/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyServiceTest.java index 312db5315..9f10fac50 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyServiceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyServiceTest.java @@ -8,12 +8,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -public class DelegateProxyServiceTest extends BaseTest { +class DelegateProxyServiceTest extends BaseTest { private DelegateProxyService instance; @@ -21,10 +22,12 @@ public class DelegateProxyServiceTest extends BaseTest { public void setUp() throws Exception { super.setUp(); + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + TestUtil.getFixture("delegates.rb").toString()); + Configuration config = Configuration.getInstance(); config.setProperty(Key.DELEGATE_SCRIPT_ENABLED, true); - config.setProperty(Key.DELEGATE_SCRIPT_PATHNAME, - TestUtil.getFixture("delegates.rb").toString()); + config.clearProperty(Key.DELEGATE_SCRIPT_PATHNAME); DelegateProxyService.clearInstance(); @@ -34,26 +37,25 @@ public void setUp() throws Exception { /* getJavaDelegate() */ @Test - void testGetJavaDelegate() { - // This is hard to test as we don't have a JavaDelegate on the - // classpath. + void getJavaDelegate() { + // TODO: This is hard to test as we don't have a JavaDelegate on the classpath. } /* isDelegateAvailable() */ @Test - void testIsDelegateAvailableWithJavaDelegateAvailable() { + void isDelegateAvailableWithJavaDelegateAvailable() { // This is hard to test as we don't have a JavaDelegate on the // classpath. } @Test - void testIsDelegateAvailableWithNoJavaDelegateAndScriptEnabled() { + void isDelegateAvailableWithNoJavaDelegateAndScriptEnabled() { assertTrue(DelegateProxyService.isDelegateAvailable()); } @Test - void testIsDelegateAvailableWithNoJavaDelegateAndScriptDisabled() { + void isDelegateAvailableWithNoJavaDelegateAndScriptDisabled() { Configuration config = Configuration.getInstance(); config.setProperty(Key.DELEGATE_SCRIPT_ENABLED, false); assertFalse(DelegateProxyService.isDelegateAvailable()); @@ -62,7 +64,7 @@ void testIsDelegateAvailableWithNoJavaDelegateAndScriptDisabled() { /* isScriptEnabled() */ @Test - void testIsScriptEnabled() { + void isScriptEnabled() { Configuration config = Configuration.getInstance(); config.setProperty(Key.DELEGATE_SCRIPT_ENABLED, false); assertFalse(DelegateProxyService.isScriptEnabled()); @@ -74,34 +76,100 @@ void testIsScriptEnabled() { /* getScriptFile() */ @Test - void testGetScriptFileWithPresentValidScript() throws Exception { - Path file = DelegateProxyService.getScriptFile(); - assertNotNull(file); + void getScriptFileWithValidScriptInVMArgumentAndConfiguration() + throws Exception { + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getFixture("delegates.rb").toString()); + + Path actual = DelegateProxyService.getScriptFile(); + assertTrue(Files.readString(actual).contains("CustomDelegate")); } @Test - void testGetScriptFileWithPresentInvalidScript() throws Exception { - Configuration config = Configuration.getInstance(); - config.setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + void getScriptFileWithValidScriptInVMArgumentAndInvalidScriptInConfiguration() + throws Exception { + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getImage("txt")); + + Path actual = DelegateProxyService.getScriptFile(); + assertTrue(Files.readString(actual).contains("CustomDelegate")); + } + + @Test + void getScriptFileWithValidScriptInVMArgumentAndMissingScriptInConfiguration() + throws Exception { + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + "/bogus/bogus/bogus"); + + Path actual = DelegateProxyService.getScriptFile(); + assertTrue(Files.readString(actual).contains("CustomDelegate")); + } + + @Test + void getScriptFileWithInvalidScriptInVMArgumentAndValidScriptInConfiguration() + throws Exception { + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, TestUtil.getImage("txt").toString()); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getFixture("delegates.rb").toString()); - Path file = DelegateProxyService.getScriptFile(); - assertNotNull(file); + Path actual = DelegateProxyService.getScriptFile(); + assertEquals("some text", Files.readString(actual)); } @Test - void testGetScriptFileWithNoScript() throws Exception { - Configuration config = Configuration.getInstance(); - config.setProperty(Key.DELEGATE_SCRIPT_PATHNAME, ""); + void getScriptFileWithInvalidScriptInVMArgumentAndConfiguration() + throws Exception { + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + TestUtil.getImage("txt").toString()); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getImage("txt").toString()); - assertNull(DelegateProxyService.getScriptFile()); + Path actual = DelegateProxyService.getScriptFile(); + assertEquals("some text", Files.readString(actual)); } @Test - void testGetScriptFileWithBogusScript() { - Configuration config = Configuration.getInstance(); - config.setProperty(Key.DELEGATE_SCRIPT_PATHNAME, - "/bla/bla/blaasdfasdfasfd"); + void getScriptFileWithInvalidScriptInVMArgumentAndMissingScriptInConfiguration() + throws Exception { + final Path invalidScript = TestUtil.getImage("txt"); + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + invalidScript.toString()); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + "/bogus/bogus/bogus"); + + Path actual = DelegateProxyService.getScriptFile(); + assertEquals("some text", Files.readString(actual)); + } + + @Test + void getScriptFileWithMissingScriptInVMArgumentAndValidScriptConfiguration() { + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + "/bogus/bogus/bogus"); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getFixture("delegates.rb").toString()); + + assertThrows(NoSuchFileException.class, + DelegateProxyService::getScriptFile); + } + + @Test + void getScriptFileWithMissingScriptInVMArgumentAndInvalidScriptInConfiguration() { + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + "/bogus/bogus/bogus"); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + TestUtil.getFixture("txt").toString()); + + assertThrows(NoSuchFileException.class, + DelegateProxyService::getScriptFile); + } + + @Test + void getScriptFileWithMissingScriptInVMArgumentAndConfiguration() { + System.setProperty(DelegateProxyService.DELEGATE_SCRIPT_VM_ARGUMENT, + "/bogus/bogus/bogus"); + Configuration.getInstance().setProperty(Key.DELEGATE_SCRIPT_PATHNAME, + "/bogus/bogus/bogus"); assertThrows(NoSuchFileException.class, DelegateProxyService::getScriptFile); @@ -110,13 +178,13 @@ void testGetScriptFileWithBogusScript() { /* newDelegateProxy() */ @Test - void testNewDelegateProxyWithJavaDelegateAvailable() { + void newDelegateProxyWithJavaDelegateAvailable() { // This is hard to test as we don't have a JavaDelegate on the // classpath. } @Test - void testNewDelegateProxyWithDelegateScriptEnabled() throws Exception { + void newDelegateProxyWithDelegateScriptEnabled() throws Exception { RequestContext context = new RequestContext(); DelegateProxy actual = instance.newDelegateProxy(context); assertNotNull(actual); diff --git a/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java b/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java index dcfa39d1b..5252db1de 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java @@ -237,6 +237,14 @@ void testGetJdbcSourceDatabaseIdentifier() throws Exception { assertEquals("cats", result); } + /* getJdbcSourceLastModified() */ + + @Test + void testGetJdbcSourceLastModified() throws Exception { + String result = instance.getJdbcSourceLastModified(); + assertEquals("SELECT last_modified FROM items WHERE filename = ?", result); + } + /* getJdbcSourceMediaType() */ @Test diff --git a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java index 84c79a215..1e6002d58 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java @@ -9,8 +9,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java index c724ae6e9..58c8785e9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java @@ -12,6 +12,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import static org.junit.jupiter.api.Assertions.*; @@ -207,7 +208,8 @@ void testFromJSONWithPath() throws Exception { Files.write(tempFile, json.getBytes(StandardCharsets.UTF_8)); Info info = Info.fromJSON(tempFile); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } finally { if (tempFile != null) { Files.deleteIfExists(tempFile); @@ -223,7 +225,8 @@ void testFromJSONWithInputStream() throws Exception { InputStream inputStream = new ByteArrayInputStream(json.getBytes()); Info info = Info.fromJSON(inputStream); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } /* fromJSON(String) */ @@ -232,14 +235,15 @@ void testFromJSONWithInputStream() throws Exception { void testFromJSONWithString() throws Exception { String json = instance.toJSON(); Info info = Info.fromJSON(json); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } /* fromJSON() serialization */ @Test void testFromJSONWithVersion2Serialization() throws Exception { - String v34json = "{\n" + + String v2json = "{\n" + " \"mediaType\": \"image/jpeg\",\n" + " \"images\": [\n" + " {\n" + @@ -250,7 +254,7 @@ void testFromJSONWithVersion2Serialization() throws Exception { " }\n" + " ]\n" + "}"; - Info actual = Info.fromJSON(v34json); + Info actual = Info.fromJSON(v2json); Info expected = Info.builder() .withFormat(Format.get("jpg")) .withSize(100, 80) @@ -261,7 +265,7 @@ void testFromJSONWithVersion2Serialization() throws Exception { @Test void testFromJSONWithVersion3Serialization() throws Exception { - String v4json = "{\n" + + String v3json = "{\n" + " \"identifier\": \"cats\",\n" + " \"mediaType\": \"image/jpeg\",\n" + " \"numResolutions\": 3,\n" + @@ -275,7 +279,7 @@ void testFromJSONWithVersion3Serialization() throws Exception { " }\n" + " ]\n" + "}"; - Info actual = Info.fromJSON(v4json); + Info actual = Info.fromJSON(v3json); Info expected = Info.builder() .withIdentifier(new Identifier("cats")) .withFormat(Format.get("jpg")) @@ -288,7 +292,7 @@ void testFromJSONWithVersion3Serialization() throws Exception { @Test void testFromJSONWithVersion4Serialization() throws Exception { - String v5json = "{\n" + + String v4json = "{\n" + " \"applicationVersion\": \"5.0\",\n" + " \"serializationVersion\": 4,\n" + " \"identifier\": \"cats\",\n" + @@ -306,9 +310,10 @@ void testFromJSONWithVersion4Serialization() throws Exception { " \"xmp\": \"\"\n" + " }\n" + "}"; + Info actual = Info.fromJSON(v4json); + Metadata metadata = new Metadata(); metadata.setXMP(""); - Info actual = Info.fromJSON(v5json); Info expected = Info.builder() .withIdentifier(new Identifier("cats")) .withFormat(Format.get("jpg")) @@ -318,6 +323,45 @@ void testFromJSONWithVersion4Serialization() throws Exception { .withTileSize(50, 40) .build(); expected.setApplicationVersion("5.0"); + actual.setSerializationVersion(Info.Serialization.CURRENT.getVersion()); + assertEquals(expected, actual); + } + + @Test + void testFromJSONWithVersion5Serialization() throws Exception { + Instant timestamp = Instant.now(); + String v6json = "{\n" + + " \"applicationVersion\": \"6.0\",\n" + + " \"serializationVersion\": 5,\n" + + " \"serializationTimestamp\": \"" + timestamp.toString() + "\",\n" + + " \"identifier\": \"cats\",\n" + + " \"mediaType\": \"image/jpeg\",\n" + + " \"numResolutions\": 3,\n" + + " \"images\": [\n" + + " {\n" + + " \"width\": 100,\n" + + " \"height\": 80,\n" + + " \"tileWidth\": 50,\n" + + " \"tileHeight\": 40\n" + + " }\n" + + " ],\n" + + " \"metadata\": {\n" + + " \"xmp\": \"\"\n" + + " }\n" + + "}"; + Metadata metadata = new Metadata(); + metadata.setXMP(""); + Info actual = Info.fromJSON(v6json); + Info expected = Info.builder() + .withIdentifier(new Identifier("cats")) + .withFormat(Format.get("jpg")) + .withNumResolutions(3) + .withMetadata(metadata) + .withSize(100, 80) + .withTileSize(50, 40) + .build(); + expected.setApplicationVersion("6.0"); + expected.setSerializationTimestamp(timestamp); assertEquals(expected, actual); } @@ -373,6 +417,20 @@ void testEqualsWithDifferentSerializationVersions() { assertNotEquals(instance, info2); } + @Test + void testEqualsWithDifferentSerializationTimestamps() { + Info info2 = Info.builder() + .withIdentifier(instance.getIdentifier()) + .withSize(instance.getSize()) + .withTileSize(instance.getImages().get(0).getTileSize()) + .withFormat(instance.getSourceFormat()) + .withNumResolutions(instance.getNumResolutions()) + .withMetadata(instance.getMetadata()) + .build(); + info2.setSerializationTimestamp(Instant.now()); + assertEquals(instance, info2); + } + @Test void testEqualsWithDifferentIdentifiers() { Info info2 = Info.builder() @@ -633,6 +691,23 @@ void testHashCodeWithEqualInstances() { assertEquals(instance.hashCode(), info2.hashCode()); } + @Test + void testHashCodeWithDifferentSerializationTimestamps() { + Metadata metadata2 = new Metadata(); + metadata2.setXMP(""); + + Info info2 = Info.builder() + .withIdentifier(instance.getIdentifier()) + .withSize(instance.getSize()) + .withTileSize(instance.getImages().get(0).getTileSize()) + .withFormat(instance.getSourceFormat()) + .withNumResolutions(instance.getNumResolutions()) + .withMetadata(metadata2) + .build(); + info2.setSerializationTimestamp(Instant.now()); + assertEquals(instance.hashCode(), info2.hashCode()); + } + @Test void testHashCodeWithDifferentIdentifiers() { Info info2 = Info.builder() @@ -859,6 +934,15 @@ void testSetPersistable() { assertFalse(instance.isPersistable()); } + /* setSerializationTimestamp() */ + + @Test + void testSetSerializationTimestamp() { + Instant timestamp = Instant.now(); + instance.setSerializationTimestamp(timestamp); + assertEquals(timestamp, instance.getSerializationTimestamp()); + } + /* setSerializationVersion() */ @Test @@ -889,6 +973,7 @@ void testToJSONContents() throws Exception { assertEquals("{" + "\"applicationVersion\":\"" + Application.getVersion() + "\"," + "\"serializationVersion\":" + Info.Serialization.CURRENT.getVersion() + "," + + "\"serializationTimestamp\":\"0000-00-00T00:00:00.000000Z\"," + "\"identifier\":\"cats\"," + "\"mediaType\":\"image/jpeg\"," + "\"numResolutions\":3," + @@ -903,7 +988,7 @@ void testToJSONContents() throws Exception { "\"xmp\":\"\"" + "}" + "}", - instance.toJSON()); + obscureTimestamps(instance.toJSON())); } @Test @@ -923,7 +1008,8 @@ void testToJSONOmitsNullValues() throws Exception { @Test void testToString() throws Exception { - assertEquals(instance.toJSON(), instance.toString()); + assertEquals(obscureTimestamps(instance.toJSON()), + obscureTimestamps(instance.toString())); } /* writeAsJSON() */ @@ -932,7 +1018,20 @@ void testToString() throws Exception { void testWriteAsJSON() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); instance.writeAsJSON(baos); - assertArrayEquals(baos.toByteArray(), instance.toJSON().getBytes()); + + String expected = baos.toString(StandardCharsets.UTF_8); + String actual = new String(instance.toJSON().getBytes(), + StandardCharsets.UTF_8); + assertEquals(obscureTimestamps(expected), obscureTimestamps(actual)); + } + + /** + * Converts any ISO-8601 timestamps in the given string to + * {@literal 0000-00-00T00:00:00.000000Z}. + */ + private static String obscureTimestamps(String inString) { + return inString.replaceAll("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d+Z", + "0000-00-00T00:00:00.000000Z"); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java index 92c69b590..de3d914f6 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.image.exif.Tag; import edu.illinois.library.cantaloupe.image.exif.TagSet; import edu.illinois.library.cantaloupe.image.iptc.DataSet; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.ImageReader; import edu.illinois.library.cantaloupe.processor.codec.ImageReaderFactory; import edu.illinois.library.cantaloupe.test.BaseTest; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.Test; import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -34,17 +36,11 @@ public void setUp() throws Exception { instance = new Metadata(); } - @Test - void testEncapsulateXMP() { - final String xmp = ""; - String actual = Metadata.encapsulateXMP(xmp); - assertTrue(actual.startsWith("")); - } + /* equals() */ @Test void testEqualsWithEqualInstances() { - Directory exif = new Directory(TagSet.EXIF); + Directory exif = new Directory(TagSet.EXIF); List iptc = List.of(new DataSet( edu.illinois.library.cantaloupe.image.iptc.Tag.CITY, "Urbana".getBytes())); @@ -119,9 +115,11 @@ void testEqualsWithDifferentXMP() { assertNotEquals(m1, m2); } + /* getEXIF() */ + @Test void testGetEXIFWithPresentEXIFData() throws Exception { - Path fixture = TestUtil.getImage("jpg-exif.jpg"); + Path fixture = TestUtil.getImage("jpg-exif.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -134,7 +132,7 @@ void testGetEXIFWithPresentEXIFData() throws Exception { @Test void testGetEXIFWithNoEXIFData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -145,9 +143,11 @@ void testGetEXIFWithNoEXIFData() throws Exception { } } + /* getIPTC() */ + @Test void testGetIPTCWithPresentIPTCData() throws Exception { - Path fixture = TestUtil.getImage("jpg-iptc.jpg"); + Path fixture = TestUtil.getImage("jpg-iptc.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -160,7 +160,7 @@ void testGetIPTCWithPresentIPTCData() throws Exception { @Test void testGetIPTCWithNoIPTCData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -171,6 +171,8 @@ void testGetIPTCWithNoIPTCData() throws Exception { } } + /* getNativeMetadata() */ + @Test void testGetNativeMetadataWithPresentData() throws Exception { Path fixture = TestUtil.getImage("png-nativemetadata.png"); @@ -249,9 +251,40 @@ void testGetOrientationWithMalformedXMP() { assertEquals(Orientation.ROTATE_0, instance.getOrientation()); } + /* getXMPElements() */ + + @Test + void testGetXMPElementsWithPresentXMPData() throws Exception { + Path fixture = TestUtil.getImage("jpg-xmp.jpg"); + ImageReader reader = new ImageReaderFactory() + .newImageReader(Format.get("jpg"), fixture); + try { + Metadata metadata = reader.getMetadata(0); + Map model = metadata.getXMPElements(); + assertEquals(6, model.size()); + } finally { + reader.dispose(); + } + } + + @Test + void testGetXMPElementsWithNoXMPData() throws Exception { + Path fixture = TestUtil.getImage("jpg"); + ImageReader reader = new ImageReaderFactory() + .newImageReader(Format.get("jpg"), fixture); + try { + Metadata metadata = reader.getMetadata(0); + assertTrue(metadata.getXMPElements().isEmpty()); + } finally { + reader.dispose(); + } + } + + /* getXMPModel() */ + @Test void testGetXMPModelWithPresentXMPData() throws Exception { - Path fixture = TestUtil.getImage("jpg-xmp.jpg"); + Path fixture = TestUtil.getImage("jpg-xmp.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -265,7 +298,7 @@ void testGetXMPModelWithPresentXMPData() throws Exception { @Test void testGetXMPModelWithNoXMPData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -276,10 +309,12 @@ void testGetXMPModelWithNoXMPData() throws Exception { } } + /* hashCode() */ + @Test void testHashCodeWithEqualInstances() { Directory exif = new Directory(TagSet.EXIF); - String xmp = "cats"; + String xmp = "cats"; Metadata m1 = new Metadata(); m1.setEXIF(exif); @@ -348,18 +383,24 @@ void testHashCodeWithDifferentXMP() { assertNotEquals(m1.hashCode(), m2.hashCode()); } + /* setEXIF() */ + @Test void testSetEXIFWithNullArgument() { instance.setEXIF(null); assertFalse(instance.getEXIF().isPresent()); } + /* setIPTC() */ + @Test void testSetIPTCWithNullArgument() { instance.setIPTC(null); assertFalse(instance.getIPTC().isPresent()); } + /* setXMP() */ + @Test void testSetXMPWithNullByteArrayArgument() { instance.setXMP((byte[]) null); @@ -380,6 +421,8 @@ void testSetXMPTrimsData() { assertTrue(xmp.endsWith("")); } + /* toMap() */ + @Test void testToMap() { // assemble the expected map structure @@ -404,6 +447,7 @@ void testToMap() { edu.illinois.library.cantaloupe.image.iptc.Tag.CITY, "Urbana".getBytes()).toMap())); expectedMap.put("xmp_string", ""); + expectedMap.put("xmp_elements", Collections.emptyMap()); expectedMap.put("native", Map.of("key1", "value1", "key2", "value2")); // assemble the Metadata diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java new file mode 100644 index 000000000..ab440fbc9 --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java @@ -0,0 +1,42 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.test.TestUtil; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class MapReaderTest { + + @Test + void readElements1() throws Exception { + String xmp = Files.readString(TestUtil.getFixture("xmp/xmp.xmp")); + xmp = Utils.trimXMP(xmp); + MapReader reader = new MapReader(xmp); + Map elements = reader.readElements(); + + print(elements); + + assertEquals(18, elements.size()); + } + + @Test + void readElements2() throws Exception { + String xmp = Files.readString(TestUtil.getFixture("xmp/xmp2.xmp")); + xmp = Utils.trimXMP(xmp); + MapReader reader = new MapReader(xmp); + Map elements = reader.readElements(); + assertEquals(61, elements.size()); + } + + private static void print(Map elements) { + System.out.println("------ ELEMENTS -------"); + for (Map.Entry entry : elements.entrySet()) { + System.out.println(entry.getKey()); + System.out.println(entry.getValue()); + System.out.println("-------------"); + } + } +} \ No newline at end of file diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java new file mode 100644 index 000000000..386ced649 --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java @@ -0,0 +1,48 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.test.BaseTest; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class UtilsTest extends BaseTest { + + /* encapsulateXMP() */ + + @Test + void testEncapsulateXMP() { + final String xmp = ""; + String actual = Utils.encapsulateXMP(xmp); + assertTrue(actual.startsWith("")); + } + + /* trimXMP() */ + + @Test + void testTrimXMPWithTrimmableXMP() { + String xmp = "" + + "" + + "" + + "" + + ""; + String result = Utils.trimXMP(xmp); + assertTrue(result.startsWith("")); + } + + @Test + void testTrimXMPWithNonTrimmableXMP() { + String xmp = "" + + ""; + String result = Utils.trimXMP(xmp); + assertSame(xmp, result); + } + + @Test + void testTrimXMPWithNullArgument() { + assertThrows(NullPointerException.class, + () -> Utils.trimXMP(null)); + } + +} \ No newline at end of file diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java index dd998a04d..74a752a69 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.processor.codec.jpeg; -import edu.illinois.library.cantaloupe.image.Metadata; import edu.illinois.library.cantaloupe.image.Rectangle; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.util.Rational; @@ -104,7 +104,7 @@ public void testSetXMP() throws Exception { final ByteArrayOutputStream expectedSegment = new ByteArrayOutputStream(); final byte[] headerBytes = "http://ns.adobe.com/xap/1.0/\0".getBytes(); - final byte[] xmpBytes = Metadata.encapsulateXMP(xmp). + final byte[] xmpBytes = Utils.encapsulateXMP(xmp). getBytes(StandardCharsets.UTF_8); // write segment marker expectedSegment.write(new byte[]{(byte) 0xff, (byte) 0xe1}); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java index 99f8486ec..123bc2120 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java index 641f9e518..7c561f226 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java @@ -1,5 +1,6 @@ package edu.illinois.library.cantaloupe.resource; +import edu.illinois.library.cantaloupe.Application; import edu.illinois.library.cantaloupe.cache.CacheFacade; import edu.illinois.library.cantaloupe.cache.CompletableOutputStream; import edu.illinois.library.cantaloupe.cache.DerivativeCache; @@ -11,29 +12,35 @@ import edu.illinois.library.cantaloupe.operation.ValidationException; import edu.illinois.library.cantaloupe.processor.Processor; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; +import edu.illinois.library.cantaloupe.processor.SourceFormatException; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; +import edu.illinois.library.cantaloupe.test.WebServer; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -public class ImageRequestHandlerTest extends BaseTest { +class ImageRequestHandlerTest extends BaseTest { @Nested - public class BuilderTest { + class BuilderTest { @Test - void testBuildWithNoOperationListSet() { + void buildWithNoOperationListSet() { assertThrows(NullPointerException.class, () -> ImageRequestHandler.builder().build()); } @Test - void testOptionallyWithDelegateProxyWithNonNullArguments() { + void optionallyWithDelegateProxyWithNonNullArguments() { DelegateProxy proxy = TestUtil.newDelegateProxy(); ImageRequestHandler handler = ImageRequestHandler.builder() .optionallyWithDelegateProxy(proxy, proxy.getRequestContext()) @@ -44,7 +51,7 @@ void testOptionallyWithDelegateProxyWithNonNullArguments() { } @Test - void testOptionallyWithDelegateProxyWithNullDelegateProxy() { + void optionallyWithDelegateProxyWithNullDelegateProxy() { RequestContext context = new RequestContext(); ImageRequestHandler handler = ImageRequestHandler.builder() .optionallyWithDelegateProxy(null, context) @@ -56,7 +63,7 @@ void testOptionallyWithDelegateProxyWithNullDelegateProxy() { } @Test - void testOptionallyWithDelegateProxyWithNullRequestContext() { + void optionallyWithDelegateProxyWithNullRequestContext() { DelegateProxy delegateProxy = TestUtil.newDelegateProxy(); ImageRequestHandler handler = ImageRequestHandler.builder() .withOperationList(new OperationList()) @@ -68,7 +75,7 @@ void testOptionallyWithDelegateProxyWithNullRequestContext() { } @Test - void testOptionallyWithDelegateProxyWithNullArguments() { + void optionallyWithDelegateProxyWithNullArguments() { ImageRequestHandler handler = ImageRequestHandler.builder() .withOperationList(new OperationList()) .optionallyWithDelegateProxy(null, null) @@ -79,14 +86,14 @@ void testOptionallyWithDelegateProxyWithNullArguments() { } @Test - void testWithDelegateProxyWithNullDelegateProxy() { + void withDelegateProxyWithNullDelegateProxy() { assertThrows(IllegalArgumentException.class, () -> ImageRequestHandler.builder() .withDelegateProxy(null, new RequestContext())); } @Test - void testWithDelegateProxyWithNullRequestContext() { + void withDelegateProxyWithNullRequestContext() { DelegateProxy delegateProxy = TestUtil.newDelegateProxy(); assertThrows(IllegalArgumentException.class, () -> ImageRequestHandler.builder() @@ -97,6 +104,7 @@ void testWithDelegateProxyWithNullRequestContext() { private static class IntrospectiveCallback implements ImageRequestHandler.Callback { private boolean isPreAuthorizeCalled, isAuthorizeCalled, + isSourceAccessedCalled, isWillStreamImageFromDerivativeCacheCalled, isInfoAvailableCalled, isWillProcessImageCalled; @@ -112,6 +120,11 @@ public boolean authorize() { return true; } + @Override + public void sourceAccessed(StatResult result) { + isSourceAccessedCalled = true; + } + @Override public void willStreamImageFromDerivativeCache() { isWillStreamImageFromDerivativeCacheCalled = true; @@ -129,13 +142,13 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleCallsPreAuthorizationCallback() throws Exception { + void handleCallsPreAuthorizationCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -155,13 +168,13 @@ void testHandleCallsPreAuthorizationCallback() throws Exception { } @Test - void testHandleCallsAuthorizationCallback() throws Exception { + void handleCallsAuthorizationCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -181,13 +194,39 @@ void testHandleCallsAuthorizationCallback() throws Exception { } @Test - void testHandleCallsCacheStreamingCallback() throws Exception { + void handleCallsSourceAccessedCallback() throws Exception { + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); + config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); + config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, + TestUtil.getImagesPath() + "/"); + } + + // Configure the request. + final OperationList opList = new OperationList(); + opList.setIdentifier(new Identifier("jpg-rgb-64x48x8.jpg")); + opList.add(new Encode(Format.get("jpg"))); + + final IntrospectiveCallback callback = new IntrospectiveCallback(); + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withCallback(callback) + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + handler.handle(outputStream); + assertTrue(callback.isSourceAccessedCalled); + } + } + + @Test + void handleCallsCacheStreamingCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); config.setProperty(Key.DERIVATIVE_CACHE_ENABLED, true); config.setProperty(Key.DERIVATIVE_CACHE, "HeapCache"); } @@ -218,7 +257,7 @@ void testHandleCallsCacheStreamingCallback() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(new byte[] { 0x35, 0x35, 0x35 }); - os.setCompletelyWritten(true); + os.setComplete(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); @@ -233,13 +272,13 @@ void testHandleCallsCacheStreamingCallback() throws Exception { } @Test - void testHandleCallsInfoAvailableCallback() throws Exception { + void handleCallsInfoAvailableCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -259,13 +298,13 @@ void testHandleCallsInfoAvailableCallback() throws Exception { } @Test - void testHandleCallsProcessingCallback() throws Exception { + void handleCallsProcessingCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -285,12 +324,12 @@ void testHandleCallsProcessingCallback() throws Exception { } @Test - void testHandleProcessesImage() throws Exception { + void handleProcessesImage() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -310,13 +349,13 @@ void testHandleProcessesImage() throws Exception { } @Test - void testHandleStreamsFromDerivativeCache() throws Exception { + void handleStreamsFromDerivativeCache() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); config.setProperty(Key.DERIVATIVE_CACHE_ENABLED, true); config.setProperty(Key.DERIVATIVE_CACHE, "HeapCache"); } @@ -348,7 +387,7 @@ void testHandleStreamsFromDerivativeCache() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(expected); - os.setCompletelyWritten(true); + os.setComplete(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); @@ -363,12 +402,12 @@ void testHandleStreamsFromDerivativeCache() throws Exception { } @Test - void testHandleWithFailedPreAuthorization() throws Exception { + void handleWithFailedPreAuthorization() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -387,6 +426,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override @@ -405,12 +447,12 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleWithFailedAuthorization() throws Exception { + void handleWithFailedAuthorization() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -429,6 +471,9 @@ public boolean authorize() { return false; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override @@ -447,12 +492,12 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleWithIllegalPageIndex() throws Exception { + void handleWithIllegalPageIndex() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -474,12 +519,12 @@ void testHandleWithIllegalPageIndex() throws Exception { } @Test - void testHandleWithInvalidOperationList() throws Exception { + void handleWithInvalidOperationList() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -497,4 +542,132 @@ void testHandleWithInvalidOperationList() throws Exception { } } + @Test + void handleDeletesIncompatibleSourceCachedImageWhenSoConfigured() + throws Exception { + final WebServer server = new WebServer(); + try { + server.start(); + + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "HttpSource"); + config.setProperty(Key.HTTPSOURCE_URL_PREFIX, + server.getHTTPURI().toString() + "/"); + config.setProperty(Key.PROCESSOR_FALLBACK, + edu.illinois.library.cantaloupe.processor.MockStreamProcessor.class.getName()); + config.setProperty(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY, + "CacheStrategy"); + config.setProperty(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, + true); // what this test is testing + config.setProperty(Key.SOURCE_CACHE, "FilesystemCache"); + config.setProperty(Key.SOURCE_CACHE_TTL, 300); + config.setProperty(Key.FILESYSTEMCACHE_PATHNAME, + Application.getTempPath().toString()); + } + + // Configure the request. + final OperationList opList = new OperationList(); + final Identifier identifier = new Identifier("jpg-rgb-64x48x8.jpg"); + final Metadata metadata = new Metadata(); + opList.setIdentifier(identifier); + Encode encode = new Encode(Format.get("jpg")); + encode.setCompression(Compression.JPEG); + encode.setQuality(80); + encode.setMetadata(metadata); + opList.add(encode); + + final CacheFacade cacheFacade = new CacheFacade(); + + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + // The first request should cause the source image to be + // source-cached... + handler.handle(outputStream); + // Overwrite the source-cached image with garbage, destroying + // any format-signifying magic bytes. + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + Files.write(file, "This is garbage".getBytes(StandardCharsets.UTF_8)); + // Send the same request again. The source cache will be + // consulted instead of the source. + handler.handle(outputStream); + fail("Expected a SourceFormatException"); + } catch (SourceFormatException e) { + // The delete happens asynchronously, so give it some time. + Thread.sleep(2000); + assertFalse(cacheFacade.getSourceCacheFile(identifier).isPresent()); + } + } finally { + server.stop(); + } + } + + @Test + void handleDoesNotDeleteIncompatibleSourceCachedImageWhenNotConfiguredTo() + throws Exception { + final WebServer server = new WebServer(); + try { + server.start(); + + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "HttpSource"); + config.setProperty(Key.HTTPSOURCE_URL_PREFIX, + server.getHTTPURI().toString() + "/"); + config.setProperty(Key.PROCESSOR_FALLBACK, + edu.illinois.library.cantaloupe.processor.MockStreamProcessor.class.getName()); + config.setProperty(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY, + "CacheStrategy"); + config.setProperty(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, + false); // what this test is testing + config.setProperty(Key.SOURCE_CACHE, "FilesystemCache"); + config.setProperty(Key.SOURCE_CACHE_TTL, 300); + config.setProperty(Key.FILESYSTEMCACHE_PATHNAME, + Application.getTempPath().toString()); + } + + // Configure the request. + final OperationList opList = new OperationList(); + final Identifier identifier = new Identifier("jpg-rgb-64x48x8.jpg"); + final Metadata metadata = new Metadata(); + opList.setIdentifier(identifier); + Encode encode = new Encode(Format.get("jpg")); + encode.setCompression(Compression.JPEG); + encode.setQuality(80); + encode.setMetadata(metadata); + opList.add(encode); + + final CacheFacade cacheFacade = new CacheFacade(); + + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + // The first request should cause the source image to be + // source-cached... + handler.handle(outputStream); + // Overwrite the source-cached image with garbage, destroying + // any format-signifying magic bytes. + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + Files.write(file, "This is garbage".getBytes(StandardCharsets.UTF_8)); + // Send the same request again. The source cache will be + // consulted instead of the source. + handler.handle(outputStream); + fail("Expected a SourceFormatException"); + } catch (SourceFormatException e) { + // The source-cached file is not supposed to get deleted, but + // it will happen asynchronously if it does, so give it some + // time. + Thread.sleep(2000); + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + assertTrue(Files.exists(file)); + Files.delete(file); + } + } finally { + server.stop(); + } + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java index 9778cc096..cd6c2c319 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.config.Key; import edu.illinois.library.cantaloupe.image.*; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import org.junit.jupiter.api.Nested; @@ -40,6 +41,7 @@ void testBuildWithDelegateProxyButNoRequestContextSet() { private static class IntrospectiveCallback implements InformationRequestHandler.Callback { private boolean isAuthorizeCalled, + isSourceAccessedCalled, isKnowAvailableOutputFormatsCalled; @Override @@ -48,6 +50,11 @@ public boolean authorize() { return true; } + @Override + public void sourceAccessed(StatResult result) { + isSourceAccessedCalled = true; + } + @Override public void knowAvailableOutputFormats(Set formats) { isKnowAvailableOutputFormatsCalled = true; @@ -74,6 +81,25 @@ void testHandleCallsAuthorizationCallback() throws Exception { } } + @Test + void testHandleCallsSourceAccessedCallback() throws Exception { + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); + config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, + TestUtil.getImagesPath() + "/"); + } + + final IntrospectiveCallback callback = new IntrospectiveCallback(); + try (InformationRequestHandler handler = InformationRequestHandler.builder() + .withCallback(callback) + .withIdentifier(new Identifier("jpg-rgb-64x48x8.jpg")) + .build()) { + handler.handle(); + assertTrue(callback.isSourceAccessedCalled); + } + } + @Test void testHandleCallsAvailableOutputFormatsCallback() throws Exception { { // Configure the application. @@ -225,6 +251,9 @@ public boolean authorize() { return false; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void knowAvailableOutputFormats(Set availableOutputFormats) { } }) diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java index ef6d5926a..1bdb796ab 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java @@ -1,18 +1,18 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.AsyncContext; -import javax.servlet.DispatcherType; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletContext; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; -import javax.servlet.http.HttpUpgradeHandler; -import javax.servlet.http.Part; +import jakarta.servlet.AsyncContext; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import jakarta.servlet.http.HttpUpgradeHandler; +import jakarta.servlet.http.Part; import java.io.BufferedReader; import java.security.Principal; import java.util.Collection; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java index 749e314f3..638cd4599 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.http.Header; import edu.illinois.library.cantaloupe.http.Headers; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletResponse; import java.io.PrintWriter; import java.util.Collection; import java.util.Locale; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/RouteTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/RouteTest.java index 742467048..67a13bb4b 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/RouteTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/RouteTest.java @@ -3,6 +3,7 @@ import edu.illinois.library.cantaloupe.resource.admin.AdminResource; import edu.illinois.library.cantaloupe.resource.api.TaskResource; import edu.illinois.library.cantaloupe.resource.api.TasksResource; +import edu.illinois.library.cantaloupe.resource.health.HealthResource; import edu.illinois.library.cantaloupe.test.BaseTest; import org.junit.jupiter.api.Test; @@ -43,8 +44,7 @@ void testForPathWithConfigurationRoute() { @Test void testForPathWithHealthRoute() { Route route = Route.forPath(Route.HEALTH_PATH); - assertEquals(edu.illinois.library.cantaloupe.resource.api.HealthResource.class, - route.getResource()); + assertEquals(HealthResource.class, route.getResource()); } @Test diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java index 4c506b916..a3092c0dd 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java @@ -291,6 +291,8 @@ void testEndpointsSection() throws Exception { inputNamed(Key.API_ENABLED).click(); inputNamed(Key.API_USERNAME).sendKeys("cats"); inputNamed(Key.API_SECRET).sendKeys("dogs"); + inputNamed(Key.HEALTH_ENDPOINT_ENABLED).click(); + inputNamed(Key.HEALTH_DEPENDENCY_CHECK).click(); inputNamed(Key.META_IDENTIFIER_TRANSFORMER).sendKeys("Bla"); css("#cl-endpoints li > a[href=\"#StandardMetaIdentifierTransformer\"]").click(); inputNamed(Key.STANDARD_META_IDENTIFIER_TRANSFORMER_DELIMITER).sendKeys("---"); @@ -313,6 +315,8 @@ void testEndpointsSection() throws Exception { assertTrue(config.getBoolean(Key.API_ENABLED)); assertEquals("cats", config.getString(Key.API_USERNAME)); assertEquals("dogs", config.getString(Key.API_SECRET)); + assertTrue(config.getBoolean(Key.HEALTH_ENDPOINT_ENABLED)); + assertTrue(config.getBoolean(Key.HEALTH_DEPENDENCY_CHECK)); assertEquals("StandardMetaIdentifierTransformer", config.getString(Key.META_IDENTIFIER_TRANSFORMER)); assertEquals("---", @@ -367,6 +371,8 @@ void testSourceSection() throws Exception { inputNamed(Key.HTTPSOURCE_CHUNK_SIZE).sendKeys("412"); inputNamed(Key.HTTPSOURCE_CHUNK_CACHE_ENABLED).click(); inputNamed(Key.HTTPSOURCE_CHUNK_CACHE_MAX_SIZE).sendKeys("333"); + inputNamed(Key.HTTPSOURCE_HTTP_PROXY_HOST).sendKeys("example.org"); + inputNamed(Key.HTTPSOURCE_HTTP_PROXY_PORT).sendKeys("12345"); inputNamed(Key.HTTPSOURCE_REQUEST_TIMEOUT).sendKeys("13"); selectNamed(Key.HTTPSOURCE_LOOKUP_STRATEGY). selectByValue("BasicLookupStrategy"); @@ -374,6 +380,7 @@ void testSourceSection() throws Exception { inputNamed(Key.HTTPSOURCE_URL_SUFFIX).sendKeys("/suffix"); inputNamed(Key.HTTPSOURCE_BASIC_AUTH_USERNAME).sendKeys("username"); inputNamed(Key.HTTPSOURCE_BASIC_AUTH_SECRET).sendKeys("password"); + inputNamed(Key.HTTPSOURCE_SEND_HEAD_REQUESTS).click(); // JdbcSource css("#cl-source li > a[href=\"#JdbcSource\"]").click(); inputNamed(Key.JDBCSOURCE_JDBC_URL).sendKeys("cats://dogs"); @@ -451,6 +458,10 @@ void testSourceSection() throws Exception { config.getBoolean(Key.HTTPSOURCE_CHUNK_CACHE_ENABLED)); assertEquals("333", config.getString(Key.HTTPSOURCE_CHUNK_CACHE_MAX_SIZE)); + assertEquals("example.org", + config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST)); + assertEquals("12345", + config.getString(Key.HTTPSOURCE_HTTP_PROXY_PORT)); assertEquals("13", config.getString(Key.HTTPSOURCE_REQUEST_TIMEOUT)); assertEquals("BasicLookupStrategy", @@ -463,6 +474,8 @@ void testSourceSection() throws Exception { config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME)); assertEquals("password", config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + assertTrue( + config.getBoolean(Key.HTTPSOURCE_SEND_HEAD_REQUESTS)); // JdbcSource assertEquals("cats://dogs", config.getString(Key.JDBCSOURCE_JDBC_URL)); @@ -490,6 +503,7 @@ void testProcessorsSection() throws Exception { selectByValue("StreamStrategy"); selectNamed(Key.PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY). selectByValue("CacheStrategy"); + inputNamed(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE).click(); inputNamed(Key.PROCESSOR_DPI).sendKeys("300"); selectNamed(Key.PROCESSOR_BACKGROUND_COLOR).selectByValue("white"); selectNamed(Key.PROCESSOR_UPSCALE_FILTER). @@ -535,6 +549,7 @@ void testProcessorsSection() throws Exception { config.getString(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY)); assertEquals("CacheStrategy", config.getString(Key.PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY)); + assertTrue(config.getBoolean(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE)); assertEquals(300, config.getInt(Key.PROCESSOR_DPI)); assertEquals("white", config.getString(Key.PROCESSOR_BACKGROUND_COLOR)); assertEquals("triangle", diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java similarity index 88% rename from src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java index 0d029493f..97a3a02e8 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java @@ -1,4 +1,4 @@ -package edu.illinois.library.cantaloupe.resource.api; +package edu.illinois.library.cantaloupe.resource.health; import edu.illinois.library.cantaloupe.Application; import edu.illinois.library.cantaloupe.config.Configuration; @@ -7,6 +7,7 @@ import edu.illinois.library.cantaloupe.http.Headers; import edu.illinois.library.cantaloupe.http.ResourceException; import edu.illinois.library.cantaloupe.http.Response; +import edu.illinois.library.cantaloupe.resource.ResourceTest; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.status.Health; import edu.illinois.library.cantaloupe.status.HealthChecker; @@ -17,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; -public class HealthResourceTest extends AbstractAPIResourceTest { +public class HealthResourceTest extends ResourceTest { @BeforeEach @Override @@ -25,6 +26,9 @@ public void setUp() throws Exception { super.setUp(); HealthChecker.getSourceUsages().clear(); HealthChecker.overrideHealth(null); + Configuration config = Configuration.getInstance(); + config.setProperty(Key.HEALTH_ENDPOINT_ENABLED, true); + client = newClient(""); } @Override @@ -35,7 +39,7 @@ protected String getEndpointPath() { @Test void testGETWithEndpointDisabled() throws Exception { Configuration config = Configuration.getInstance(); - config.setProperty(Key.API_ENABLED, false); + config.setProperty(Key.HEALTH_ENDPOINT_ENABLED, false); try { client.send(); fail("Expected exception"); @@ -116,22 +120,6 @@ void testGETWithRedStatus() throws Exception { } } - @Override // because this endpoint doesn't require auth - @Test - public void testGETWithNoCredentials() throws Exception { - client.setUsername(null); - client.setSecret(null); - client.send(); - } - - @Override // because this endpoint doesn't require auth - @Test - public void testGETWithInvalidCredentials() throws Exception { - client.setUsername("invalid"); - client.setSecret("invalid"); - client.send(); - } - @Test void testGETResponseBody() throws Exception { Response response = client.send(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java index 3e11a6786..183b002f0 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java @@ -16,6 +16,7 @@ import edu.illinois.library.cantaloupe.source.AccessDeniedSource; import edu.illinois.library.cantaloupe.source.PathStreamFactory; import edu.illinois.library.cantaloupe.source.Source; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.source.StreamFactory; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.test.TestUtil; @@ -29,7 +30,12 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; import java.util.Iterator; +import java.util.Locale; import static edu.illinois.library.cantaloupe.test.Assert.HTTPAssert.*; import static edu.illinois.library.cantaloupe.test.Assert.PathAssert.*; @@ -276,6 +282,29 @@ public void testForbidden(URI uri) { assertStatus(403, uri); } + public void testLastModifiedHeaderWhenDerivativeCacheIsEnabled(URI uri) + throws Exception { + initializeFilesystemCache(); + + Client client = newClient(uri); + try { + // request a resource once to cache it + client.send(); + // request it again to get the Last-Modified header + Response response = client.send(); + String value = response.getHeaders().getFirstValue("Last-Modified"); + TemporalAccessor ta = DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .parse(value); + Instant instant = Instant.from(ta); + // assert that the header value is less than 2 seconds in the past + assertTrue(Instant.now().getEpochSecond() - instant.getEpochSecond() < 2); + } finally { + client.stop(); + } + } + public void testNotFound(URI uri) { assertStatus(404, uri); } @@ -329,7 +358,7 @@ public void testRecoveryFromIncorrectSourceFormat(URI uri) throws Exception { public static class NotCheckingAccessSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { throw new IOException("checkAccess called!"); } @@ -404,7 +433,9 @@ public void testSourceCheckAccessNotCalledWithSourceCacheHit(Identifier identifi public static class NotReadingSourceFormatSource implements Source { @Override - public void checkAccess() {} + public StatResult stat() { + return null; + } @Override public Iterator getFormatIterator() { diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java index 40eb7924d..b83580287 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java @@ -544,7 +544,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/full/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -554,6 +554,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java similarity index 92% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java index f5db42c76..ff0d5c304 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java @@ -15,10 +15,10 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private String imageUri; - private ImageInfo imageInfo; + private Information imageInfo; private Processor processor; @BeforeEach @@ -35,9 +35,12 @@ public void setUp() throws Exception { TestUtil.getImage("jpg-rgb-594x522x8-baseline.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( - imageUri, processor.getAvailableOutputFormats(), - info, 0, new ScaleConstraint(1, 1)); + imageInfo = new InformationFactory().newImageInfo( + imageUri, + processor.getAvailableOutputFormats(), + info, + 0, + new ScaleConstraint(1, 1)); } @Override @@ -56,7 +59,7 @@ private void setUpForRotatedImage() throws Exception { TestUtil.getImage("jpg-xmp-orientation-90.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); } @@ -68,7 +71,7 @@ private void setUpForScaleConstrainedImage() throws Exception { TestUtil.getImage("jpg-rgb-594x522x8-baseline.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 2)); } @@ -160,7 +163,7 @@ void newImageInfoTileWidthWithTiledImage() throws Exception { ((FileProcessor) processor).setSourceFile( TestUtil.getImage("tif-rgb-1res-64x56x8-tiled-uncompressed.tif")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); @@ -191,7 +194,7 @@ void newImageInfoTileHeightWithTiledImage() throws Exception { ((FileProcessor) processor).setSourceFile( TestUtil.getImage("tif-rgb-1res-64x56x8-tiled-uncompressed.tif")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java index a8d570e7c..af9dfa9db 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java @@ -451,9 +451,9 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); - assertEquals("http://localhost:" + getHTTPPort() + - Route.IIIF_1_PATH + "/" + IMAGE, info.id); + Information info = mapper.readValue(json, Information.class); + assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_1_PATH + "/" + IMAGE, + info.id); } @Test @@ -466,7 +466,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_1_PATH + "/" + IMAGE, info.id); } @@ -482,7 +482,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_1_PATH + path, info.id); } @@ -498,9 +498,9 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); - assertEquals("http://localhost:" + getHTTPPort() + - Route.IIIF_1_PATH + path, info.id); + Information info = mapper.readValue(json, Information.class); + assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_1_PATH + path, + info.id); } @Test @@ -516,7 +516,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_1_PATH + "/originalID", info.id); } @@ -535,7 +535,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_1_PATH + "/" + IMAGE, info.id); } @@ -549,7 +549,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -560,6 +560,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server @@ -578,6 +580,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java index 0c3b089f4..b99b2b38d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java @@ -526,7 +526,7 @@ void testInformationRequestContentType() throws Exception { */ @Test void testInformationRequestJSON() { - // this will be tested in ImageInfoFactoryTest + // this will be tested in InformationFactoryTest } /** diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java index d2f6008ea..e8c2c9ee7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java @@ -646,7 +646,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/full/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -656,6 +656,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java similarity index 76% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java index 7d311e8ed..2a6907c6a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java @@ -17,21 +17,21 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private static final Set PROCESSOR_FORMATS = Set.of( Format.get("gif"), Format.get("jpg"), Format.get("png")); - private ImageInfoFactory instance; + private InformationFactory instance; @BeforeEach public void setUp() throws Exception { super.setUp(); - instance = new ImageInfoFactory(); + instance = new InformationFactory(); } - private ImageInfo invokeNewImageInfo() { + private Information invokeNewImageInfo() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); return instance.newImageInfo(PROCESSOR_FORMATS, imageURI, info, 0, @@ -40,26 +40,26 @@ private ImageInfo invokeNewImageInfo() { @Test void testNewImageInfoContext() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image/2/context.json", info.get("@context")); } @Test void testNewImageInfoID() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://example.org/bla", info.get("@id")); } @Test void testNewImageInfoProtocol() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image", info.get("protocol")); } @Test void testNewImageInfoWidth() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1500, info.get("width")); } @@ -75,10 +75,10 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1200, imageInfo.get("width")); + assertEquals(1200, iiifInfo.get("width")); } @Test @@ -87,15 +87,15 @@ void testNewImageInfoWidthWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(750, imageInfo.get("width")); + assertEquals(750, iiifInfo.get("width")); } @Test void testNewImageInfoHeight() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1200, info.get("height")); } @@ -111,10 +111,10 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1500, imageInfo.get("height")); + assertEquals(1500, iiifInfo.get("height")); } @Test @@ -123,19 +123,19 @@ void testNewImageInfoHeightWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(600, imageInfo.get("height")); + assertEquals(600, iiifInfo.get("height")); } @Test void testNewImageInfoSizes() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -152,11 +152,11 @@ void testNewImageInfoSizes() { @Test void testNewImageInfoSizesMinSize() { instance.setMinSize(500); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(2, sizes.size()); assertEquals(750, (int) sizes.get(0).width); assertEquals(600, (int) sizes.get(0).height); @@ -167,11 +167,11 @@ void testNewImageInfoSizesMinSize() { @Test void testNewImageInfoSizesMaxSize() { instance.setMaxPixels(10000); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(1, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -189,12 +189,12 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(75, (int) sizes.get(0).width); assertEquals(94, (int) sizes.get(0).height); @@ -214,12 +214,12 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1500, 1200) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(4, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -233,11 +233,11 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { @Test void testNewImageInfoTilesWithUntiledMonoResolutionImage() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -257,12 +257,12 @@ void testNewImageInfoTilesWithUntiledMultiResolutionImage() { .withSize(3000, 2000) .withNumResolutions(3) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -284,12 +284,12 @@ void testNewImageInfoMinTileSize() { .withTileSize(1000, 1000) .build(); instance.setMinTileSize(1000); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1000, (int) tiles.get(0).width); assertEquals(1000, (int) tiles.get(0).height); } @@ -307,12 +307,12 @@ public Orientation getOrientation() { }) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(56, (int) tiles.get(0).width); assertEquals(64, (int) tiles.get(0).height); } @@ -324,12 +324,12 @@ void testNewImageInfoTilesWithScaleConstrainedImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(32, (int) tiles.get(0).width); assertEquals(28, (int) tiles.get(0).height); } @@ -341,12 +341,12 @@ void testNewImageInfoTilesWithTiledImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(64, (int) tiles.get(0).width); assertEquals(56, (int) tiles.get(0).height); @@ -357,15 +357,15 @@ void testNewImageInfoTilesWithTiledImage() { @Test void testNewImageInfoProfile() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals("http://iiif.io/api/image/2/level2.json", profile.get(0)); } @Test void testNewImageInfoFormats() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); // If some are present, we will assume the rest are. (The exact // contents of the sets are processor-dependent.) assertTrue(((Set) ((Map) profile.get(1)).get("formats")).contains("gif")); @@ -373,8 +373,8 @@ void testNewImageInfoFormats() { @Test void testNewImageInfoQualities() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); // If some are present, we will assume the rest are. (The exact // contents of the sets are processor-dependent.) assertTrue(((Set) ((Map) profile.get(1)).get("qualities")).contains("color")); @@ -385,8 +385,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixels() { final int maxPixels = 100; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals(maxPixels, ((Map) profile.get(1)).get("maxArea")); } @@ -395,8 +395,8 @@ void testNewImageInfoMaxAreaWithZeroMaxPixels() { final int maxPixels = 0; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertFalse(((Map) profile.get(1)).containsKey("maxArea")); } @@ -406,16 +406,16 @@ void testNewImageInfoMaxAreaWithAllowUpscalingDisabled() { instance.setMaxPixels(maxPixels); instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals(1500 * 1200, ((Map) profile.get(1)).get("maxArea")); } @Test void testNewImageInfoSupports() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertTrue(supportsSet.contains("baseUriRedirect")); assertTrue(supportsSet.contains("canonicalLinkHeader")); @@ -429,9 +429,9 @@ void testNewImageInfoSupports() { @Test void testNewImageInfoSupportsWhenUpscalingIsAllowed() { instance.setMaxScale(9.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertTrue(supportsSet.contains("sizeAboveFull")); } @@ -439,9 +439,9 @@ void testNewImageInfoSupportsWhenUpscalingIsAllowed() { @Test void testNewImageInfoSupportsWhenUpscalingIsDisallowed() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertFalse(supportsSet.contains("sizeAboveFull")); } @@ -450,10 +450,10 @@ void testNewImageInfoSupportsWhenUpscalingIsDisallowed() { void testNewImageInfoSupportsWithScaleConstraint() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 4)); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertFalse(supportsSet.contains("sizeAboveFull")); } @@ -463,12 +463,12 @@ void testNewImageInfoDelegateKeys() { DelegateProxy proxy = TestUtil.newDelegateProxy(); instance.setDelegateProxy(proxy); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals("Copyright My Great Organization. All rights reserved.", - imageInfo.get("attribution")); + iiifInfo.get("attribution")); assertEquals("http://example.org/license.html", - imageInfo.get("license")); + iiifInfo.get("license")); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java index 2143ad5da..021d93927 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java @@ -422,7 +422,7 @@ void testGETScaleConstraintIsRespected() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals(32, info.get("width")); assertEquals(28, info.get("height")); } @@ -473,7 +473,7 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } @@ -488,7 +488,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } @@ -504,7 +504,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_2_PATH + path, info.get("@id")); } @@ -520,7 +520,7 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_2_PATH + path, info.get("@id")); } @@ -538,7 +538,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_2_PATH + "/originalID", info.get("@id")); } @@ -557,7 +557,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } @@ -571,7 +571,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(7, headers.size()); + assertEquals(8, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -582,6 +582,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary @@ -598,6 +600,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java index 81714d12c..45c4bb997 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java @@ -558,7 +558,7 @@ void testInformationRequestCORSHeader() throws Exception { */ @Test void testInformationRequestJSON() { - // this will be tested in ImageInfoFactoryTest + // this will be tested in InformationFactoryTest } /** @@ -585,7 +585,7 @@ void testComplianceLevel() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); List profile = (List) info.get("profile"); assertEquals("http://iiif.io/api/image/2/level2.json", profile.get(0)); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java index 2b834f61b..d5d0a00b7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java @@ -596,7 +596,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/max/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -606,6 +606,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java similarity index 74% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java index 77b750a30..7b187ba95 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java @@ -16,20 +16,20 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private static final Set PROCESSOR_FORMATS = Set.of(Format.get("gif"), Format.get("jpg"), Format.get("png")); - private ImageInfoFactory instance; + private InformationFactory instance; @BeforeEach public void setUp() throws Exception { super.setUp(); - instance = new ImageInfoFactory(); + instance = new InformationFactory(); } - private ImageInfo invokeNewImageInfo() { + private Information invokeNewImageInfo() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); return instance.newImageInfo(PROCESSOR_FORMATS, imageURI, info, 0, @@ -38,38 +38,38 @@ private ImageInfo invokeNewImageInfo() { @Test void testNewImageInfoContext() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image/3/context.json", info.get("@context")); } @Test void testNewImageInfoID() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://example.org/bla", info.get("id")); } @Test void testNewImageInfoType() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("ImageService3", info.get("type")); } @Test void testNewImageInfoProtocol() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image", info.get("protocol")); } @Test void testNewImageInfoProfile() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("level2", info.get("profile")); } @Test void testNewImageInfoWidth() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1500, info.get("width")); } @@ -85,11 +85,11 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1200, imageInfo.get("width")); + assertEquals(1200, iiifInfo.get("width")); } @Test @@ -98,16 +98,16 @@ void testNewImageInfoWidthWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(750, imageInfo.get("width")); + assertEquals(750, iiifInfo.get("width")); } @Test void testNewImageInfoHeight() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1200, info.get("height")); } @@ -123,11 +123,11 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1500, imageInfo.get("height")); + assertEquals(1500, iiifInfo.get("height")); } @Test @@ -136,11 +136,11 @@ void testNewImageInfoHeightWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(600, imageInfo.get("height")); + assertEquals(600, iiifInfo.get("height")); } @Test @@ -150,8 +150,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsGreaterThanAndPositiveMaxScale( instance.setMaxPixels(maxPixels); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); } @Test @@ -161,8 +161,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsLessThanPositiveMaxScale() { instance.setMaxPixels(maxPixels); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); // TODO: fix + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); // TODO: fix } @Test @@ -170,8 +170,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsAndZeroMaxScale() { final long maxPixels = 100; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); } @Test @@ -180,9 +180,9 @@ void testNewImageInfoMaxAreaWithZeroMaxPixelsAndPositiveMaxScale() { instance.setMaxPixels(0); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals(Math.round(1500 * 1200 * maxScale), - imageInfo.get("maxArea")); + iiifInfo.get("maxArea")); } @Test @@ -191,8 +191,8 @@ void testNewImageInfoMaxAreaWithZeroMaxPixelsAndZeroMaxScale() { instance.setMaxScale(0); instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - assertFalse(imageInfo.containsKey("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertFalse(iiifInfo.containsKey("maxArea")); } @Test @@ -201,17 +201,17 @@ void testNewImageInfoMaxAreaWithAllowUpscalingDisabled() { instance.setMaxPixels(maxPixels); instance.setMaxScale(1); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals((long) Math.round(1500 * 1200), imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals((long) Math.round(1500 * 1200), iiifInfo.get("maxArea")); } @Test void testNewImageInfoSizes() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -228,11 +228,11 @@ void testNewImageInfoSizes() { @Test void testNewImageInfoSizesMinSize() { instance.setMinSize(500); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(2, sizes.size()); assertEquals(750, (int) sizes.get(0).width); assertEquals(600, (int) sizes.get(0).height); @@ -243,11 +243,11 @@ void testNewImageInfoSizesMinSize() { @Test void testNewImageInfoSizesMaxSize() { instance.setMaxPixels(10000); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(1, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -265,13 +265,13 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(75, (int) sizes.get(0).width); assertEquals(94, (int) sizes.get(0).height); @@ -291,13 +291,13 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1500, 1200) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(4, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -311,11 +311,11 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { @Test void testNewImageInfoTilesWithUntiledMonoResolutionImage() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -335,13 +335,13 @@ void testNewImageInfoTilesWithUntiledMultiResolutionImage() { .withSize(3000, 2000) .withNumResolutions(3) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -363,13 +363,13 @@ void testNewImageInfoMinTileSize() { .withTileSize(1000, 1000) .build(); instance.setMinTileSize(1000); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1000, (int) tiles.get(0).width); assertEquals(1000, (int) tiles.get(0).height); } @@ -387,13 +387,13 @@ public Orientation getOrientation() { }) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(56, (int) tiles.get(0).width); assertEquals(64, (int) tiles.get(0).height); } @@ -405,13 +405,13 @@ void testNewImageInfoTilesWithScaleConstrainedImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(32, (int) tiles.get(0).width); assertEquals(28, (int) tiles.get(0).height); } @@ -423,13 +423,13 @@ void testNewImageInfoTilesWithTiledImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(64, (int) tiles.get(0).width); assertEquals(56, (int) tiles.get(0).height); @@ -440,8 +440,8 @@ void testNewImageInfoTilesWithTiledImage() { @Test void testNewImageInfoExtraQualities() { - ImageInfo imageInfo = invokeNewImageInfo(); - List qualities = (List) imageInfo.get("extraQualities"); + Information iiifInfo = invokeNewImageInfo(); + List qualities = (List) iiifInfo.get("extraQualities"); assertEquals(3, qualities.size()); assertTrue(qualities.contains("color")); assertTrue(qualities.contains("gray")); @@ -450,25 +450,25 @@ void testNewImageInfoExtraQualities() { @Test void testNewImageInfoExtraFormats() { - ImageInfo imageInfo = invokeNewImageInfo(); - List formats = (List) imageInfo.get("extraFormats"); + Information iiifInfo = invokeNewImageInfo(); + List formats = (List) iiifInfo.get("extraFormats"); assertEquals(1, formats.size()); assertTrue(formats.contains("gif")); } @Test void testNewImageInfoExtraFeatures() { - ImageInfo imageInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + Information iiifInfo = invokeNewImageInfo(); + List features = (List) iiifInfo.get("extraFeatures"); assertEquals(17, features.size()); } @Test void testNewImageInfoExtraFeaturesOmitsSizeUpscalingWhenMaxScaleIsLessThanOrEqualTo1() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertEquals(16, features.size()); assertFalse(features.contains("sizeUpscaling")); } @@ -476,18 +476,18 @@ void testNewImageInfoExtraFeaturesOmitsSizeUpscalingWhenMaxScaleIsLessThanOrEqua @Test void testNewImageInfoExtraFeaturesWhenUpscalingIsAllowed() { instance.setMaxScale(9.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertTrue(features.contains("sizeUpscaling")); } @Test void testNewImageInfoExtraFeaturesWhenUpscalingIsDisallowed() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertFalse(features.contains("sizeUpscaling")); } @@ -495,11 +495,11 @@ void testNewImageInfoExtraFeaturesWhenUpscalingIsDisallowed() { void testNewImageInfoExtraFeaturesWithScaleConstraint() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 4)); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertFalse(features.contains("sizeUpscaling")); } @@ -508,12 +508,12 @@ void testNewImageInfoDelegateKeys() { DelegateProxy proxy = TestUtil.newDelegateProxy(); instance.setDelegateProxy(proxy); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals("Copyright My Great Organization. All rights reserved.", - imageInfo.get("attribution")); + iiifInfo.get("attribution")); assertEquals("http://example.org/license.html", - imageInfo.get("license")); + iiifInfo.get("license")); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java index e58afc2f6..38e55cec7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java @@ -399,7 +399,7 @@ void testGETScaleConstraintIsRespected() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals(32, info.get("width")); assertEquals(28, info.get("height")); } @@ -450,7 +450,7 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } @@ -465,7 +465,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } @@ -481,7 +481,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_3_PATH + path, info.get("id")); } @@ -497,7 +497,7 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + getHTTPPort() + Route.IIIF_3_PATH + path, info.get("id")); } @@ -515,7 +515,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_3_PATH + "/originalID", info.get("id")); } @@ -534,7 +534,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } @@ -548,7 +548,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(7, headers.size()); + assertEquals(8, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -559,6 +559,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary @@ -575,6 +577,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java index 7242b172f..a4b4c2590 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java @@ -882,7 +882,7 @@ void testInformationRequestCORSHeader() throws Exception { */ @Test void testInformationRequestJSON() { - // this is tested in ImageInfoFactoryTest + // this is tested in InformationFactoryTest } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java index 1096dacf0..c9084cc46 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java @@ -25,89 +25,105 @@ public void setUp() throws Exception { useBasicLookupStrategy(); } - /* checkAccess() */ + /* getFormatIterator() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentReadableImage() - throws Exception { + void testGetFormatIteratorConsecutiveInvocationsReturnSameInstance() { + Source instance = newInstance(); + var it = instance.getFormatIterator(); + assertSame(it, instance.getFormatIterator()); + } + + /* newStreamFactory() */ + + @Test + void testNewStreamFactoryInvokedMultipleTimes() throws Exception { + Source instance = newInstance(); try { initializeEndpoint(); - - newInstance().checkAccess(); + instance.newStreamFactory(); + instance.newStreamFactory(); + instance.newStreamFactory(); } finally { destroyEndpoint(); } } @Test - void testCheckAccessUsingBasicLookupStrategyWithMissingImage() + void testNewStreamFactoryReturnedInstanceIsReusable() throws Exception { + Source instance = newInstance(); try { initializeEndpoint(); + StreamFactory source = instance.newStreamFactory(); - Source instance = newInstance(); - instance.setIdentifier(new Identifier("bogus")); - assertThrows(NoSuchFileException.class, instance::checkAccess); + try (InputStream is = source.newInputStream(); + OutputStream os = OutputStream.nullOutputStream()) { + is.transferTo(os); + } + + try (InputStream is = source.newInputStream(); + OutputStream os = OutputStream.nullOutputStream()) { + is.transferTo(os); + } } finally { destroyEndpoint(); } } + /* stat() */ + @Test - void testCheckAccessInvokedMultipleTimes() throws Exception { + void testStatUsingBasicLookupStrategyWithPresentReadableImage() + throws Exception { try { initializeEndpoint(); - Source instance = newInstance(); - instance.checkAccess(); - instance.checkAccess(); - instance.checkAccess(); + newInstance().stat(); } finally { destroyEndpoint(); } } - /* getFormatIterator() */ - @Test - void testGetFormatIteratorConsecutiveInvocationsReturnSameInstance() { - Source instance = newInstance(); - var it = instance.getFormatIterator(); - assertSame(it, instance.getFormatIterator()); - } + void testStatUsingBasicLookupStrategyWithMissingImage() + throws Exception { + try { + initializeEndpoint(); - /* newStreamFactory() */ + Source instance = newInstance(); + instance.setIdentifier(new Identifier("bogus")); + assertThrows(NoSuchFileException.class, instance::stat); + } finally { + destroyEndpoint(); + } + } @Test - void testNewStreamFactoryInvokedMultipleTimes() throws Exception { - Source instance = newInstance(); + void testStatReturnsCorrectInstance() throws Exception { try { initializeEndpoint(); - instance.newStreamFactory(); - instance.newStreamFactory(); - instance.newStreamFactory(); + + StatResult result = newInstance().stat(); + assertNotNull(result.getLastModified()); } finally { destroyEndpoint(); } } + /** + * Tests that {@link Source#stat()} can be invoked multiple times without + * throwing an exception. + */ @Test - void testNewStreamFactoryReturnedInstanceIsReusable() - throws Exception { - Source instance = newInstance(); + void testStatInvokedMultipleTimes() throws Exception { try { initializeEndpoint(); - StreamFactory source = instance.newStreamFactory(); - try (InputStream is = source.newInputStream(); - OutputStream os = OutputStream.nullOutputStream()) { - is.transferTo(os); - } - - try (InputStream is = source.newInputStream(); - OutputStream os = OutputStream.nullOutputStream()) { - is.transferTo(os); - } + Source instance = newInstance(); + instance.stat(); + instance.stat(); + instance.stat(); } finally { destroyEndpoint(); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java index 02c017235..1acb57949 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java @@ -9,7 +9,7 @@ public class AccessDeniedSource extends AbstractSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { throw new AccessDeniedException(""); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java index fdff3bbc7..7f51989e2 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java @@ -169,7 +169,7 @@ void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); - instance.checkAccess(); + instance.stat(); } @Test @@ -188,14 +188,14 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { instance.setDelegateProxy(delegateProxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, instance::checkAccess); + assertThrows(NoSuchFileException.class, instance::stat); } @Test void testCheckAccessWithSAS() throws Exception { instance.setIdentifier(new Identifier(getSASURI())); clearConfig(); - instance.checkAccess(); + instance.stat(); } /* getFormatIterator() */ diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java index d3401fd7a..5db555695 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java @@ -77,7 +77,7 @@ void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableFile() Path path = instance.getFile(); try { assumeTrue(path.toFile().setReadable(false)); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } finally { path.toFile().setReadable(true); } @@ -95,7 +95,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentReadableFile() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test @@ -114,7 +114,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableFile() try { assumeTrue(path.toFile().setReadable(false)); Files.setPosixFilePermissions(path, Collections.emptySet()); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } finally { path.toFile().setReadable(true); } @@ -129,7 +129,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingFile() { proxy.getRequestContext().setIdentifier(identifier); instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, instance::checkAccess); + assertThrows(NoSuchFileException.class, instance::stat); } /* getFormatIterator() */ diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java index 05e93706b..20127c4cc 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java @@ -4,12 +4,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -public class HTTPRequestInfoTest extends BaseTest { +class HTTPRequestInfoTest extends BaseTest { private HTTPRequestInfo instance; @@ -17,22 +16,39 @@ public class HTTPRequestInfoTest extends BaseTest { public void setUp() throws Exception { super.setUp(); - Map headers = new HashMap<>(); - headers.put("X-Animal", "cats"); - - instance = new HTTPRequestInfo("http://example.org/cats", - "user", "secret", headers); + instance = new HTTPRequestInfo(); + instance.setURI("http://example.org/cats"); + instance.setUsername("user"); + instance.setSecret("secret"); + instance.setHeaders(Map.of("X-Animal", "cats")); + instance.setSendingHeadRequest(true); } @Test - void testGetBasicAuthTokenWithoutUserAndSecret() { - instance = new HTTPRequestInfo("http://example.org/cats"); + void getBasicAuthTokenWithoutUserAndSecret() { + instance = new HTTPRequestInfo(); + instance.setURI("http://example.org/cats"); assertNull(instance.getBasicAuthToken()); } @Test - void testGetBasicAuthTokenWithUserAndSecret() { + void getBasicAuthTokenWithUserAndSecret() { assertEquals("dXNlcjpzZWNyZXQ=", instance.getBasicAuthToken()); } + @Test + void setHeaders() { + assertEquals(1, instance.getHeaders().size()); + + instance.setHeaders(Map.of("X-Cats", "yes")); + assertEquals(2, instance.getHeaders().size()); + assertEquals("yes", instance.getHeaders().getFirstValue("X-Cats")); + } + + @Test + void setHeadersWithNullArgument() { + instance.setHeaders(null); + assertEquals(0, instance.getHeaders().size()); + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java index 6ef3954e8..7fa0205f3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java @@ -7,17 +7,18 @@ import edu.illinois.library.cantaloupe.source.stream.HTTPImageInputStream; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.WebServer; +import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.DefaultHandler; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import javax.imageio.stream.ImageInputStream; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.InputStream; -import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @@ -50,11 +51,11 @@ private HTTPStreamFactory newInstance() { } private HTTPStreamFactory newInstance(boolean serverAcceptsRanges) { - Map headers = new HashMap<>(); - headers.put("X-Custom", "yes"); - HTTPRequestInfo requestInfo = new HTTPRequestInfo( - server.getHTTPURI().resolve("/" + PRESENT_READABLE_IDENTIFIER).toString(), - null, null, headers); + Map headers = Map.of("X-Custom", "yes"); + HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI( + server.getHTTPURI().resolve("/" + PRESENT_READABLE_IDENTIFIER).toString()); + requestInfo.setHeaders(headers); return new HTTPStreamFactory( requestInfo, @@ -72,6 +73,28 @@ void isSeekingDirect() { assertTrue(instance.isSeekingDirect()); } + @Disabled + @Test + void newInputStreamWithProxy() throws Exception { + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO: write this + + // Set up HttpSource + final var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + int length = 0; + try (InputStream is = newInstance(true).newInputStream()) { + while (is.read() != -1) { + length++; + } + } + assertTrue(length > 1000); + } + @Test void newInputStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { @@ -139,6 +162,28 @@ void newSeekableStreamWhenChunkingIsDisabled() throws Exception { } } + @Disabled + @Test + void newSeekableStreamWithProxy() throws Exception { + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO: write this + + // Set up HttpSource + final var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + int length = 0; + try (ImageInputStream is = newInstance(true).newSeekableStream()) { + while (is.read() != -1) { + length++; + } + } + assertTrue(length > 1000); + } + @Test void newSeekableStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index fba94db2a..a30046c2a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -9,14 +9,16 @@ import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.test.WebServer; +import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.DefaultHandler; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -27,12 +29,36 @@ import java.util.HashMap; import java.util.Map; import java.util.NoSuchElementException; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; abstract class HttpSourceTest extends AbstractSourceTest { + private static class RequestCountingHandler extends DefaultHandler { + + private int numHEADRequests, numGETRequests; + + @Override + public void handle(String target, + Request baseRequest, + HttpServletRequest request, + HttpServletResponse response) { + switch (request.getMethod().toUpperCase()) { + case "HEAD": + numHEADRequests++; + break; + case "GET": + numGETRequests++; + break; + default: + throw new IllegalArgumentException( + "Unexpected method: " + request.getMethod()); + } + baseRequest.setHandled(true); + } + + } + private static final Identifier PRESENT_READABLE_IDENTIFIER = new Identifier("jpg-rgb-64x56x8-baseline.jpg"); @@ -93,16 +119,16 @@ void useScriptLookupStrategy() { "ScriptLookupStrategy"); } - /* checkAccess() */ + /* stat() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() + void testStatUsingBasicLookupStrategyWithPresentUnreadableImage() throws Exception { doTestCheckAccessWithPresentUnreadableImage(new Identifier("gif")); } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() + void testStatUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/" + @@ -111,7 +137,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingImage() + void testStatUsingScriptLookupStrategyWithMissingImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/bogus"); @@ -119,7 +145,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableImage() + void testStatUsingScriptLookupStrategyWithPresentUnreadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/gif"); @@ -135,7 +161,7 @@ private void doTestCheckAccessWithPresentReadableImage(Identifier identifier) instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } private void doTestCheckAccessWithPresentUnreadableImage(Identifier identifier) @@ -157,7 +183,7 @@ public void handle(String target, instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); instance.setIdentifier(identifier); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } private void doTestCheckAccessWithMissingImage(Identifier identifier) @@ -170,7 +196,7 @@ private void doTestCheckAccessWithMissingImage(Identifier identifier) instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (NoSuchFileException e) { // pass @@ -178,7 +204,7 @@ private void doTestCheckAccessWithMissingImage(Identifier identifier) } @Test - void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() + void testStatUsingScriptLookupStrategyWithValidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -192,11 +218,11 @@ void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test - void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() + void testStatUsingScriptLookupStrategyWithInvalidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -210,11 +236,11 @@ void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } @Test - void testCheckAccessWith403Response() throws Exception { + void testStatWith403Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -229,7 +255,7 @@ public void handle(String target, try { instance.setIdentifier(PRESENT_READABLE_IDENTIFIER); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (AccessDeniedException e) { assertTrue(e.getMessage().contains("403")); @@ -237,7 +263,7 @@ public void handle(String target, } @Test - void testCheckAccessWith500Response() throws Exception { + void testStatWith500Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -252,15 +278,34 @@ public void handle(String target, try { instance.setIdentifier(PRESENT_READABLE_IDENTIFIER); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (IOException e) { assertTrue(e.getMessage().contains("500")); } } + @Disabled @Test - void testCheckAccessSendsUserAgentHeader() throws Exception { + void testStatUsingProxy() throws Exception { + server.start(); + + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO; write this + + // Set up HttpSource + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + // Expect no exception + instance.stat(); + } + + @Test + void testStatSendsUserAgentHeader() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -281,11 +326,11 @@ public void handle(String target, }); server.start(); - instance.checkAccess(); + instance.stat(); } @Test - void testCheckAccessSendsCustomHeaders() throws Exception { + void testStatSendsCustomHeaders() throws Exception { useScriptLookupStrategy(); server.setHandler(new DefaultHandler() { @@ -307,26 +352,22 @@ public void handle(String target, instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test - void testCheckAccessWithMalformedURI() throws Exception { + void testStatWithMalformedURI() throws Exception { server.start(); Configuration config = Configuration.getInstance(); config.setProperty(Key.HTTPSOURCE_URL_PREFIX, ""); Identifier identifier = new Identifier( - getServerURI().toString().replace("://", "//") + "/" + PRESENT_READABLE_IDENTIFIER); + getServerURI().toString().replace("://", "//") + "/" + + PRESENT_READABLE_IDENTIFIER); instance.setIdentifier(identifier); - try { - instance.checkAccess(); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - // pass - } + assertThrows(IOException.class, () -> instance.stat()); } /* getFormatIterator() */ @@ -367,10 +408,10 @@ public void handle(String target, server.start(); HttpSource.FormatIterator it = instance.getFormatIterator(); - assertEquals(Format.get("png"), it.next()); // URI path extension - assertEquals(Format.get("png"), it.next()); // identifier extension - assertEquals(Format.UNKNOWN, it.next()); // Content-Type is null - assertEquals(Format.get("jpg"), it.next()); // magic bytes + assertEquals(Format.get("png"), it.next()); // URI path extension + assertEquals(Format.get("png"), it.next()); // identifier extension + assertEquals(Format.UNKNOWN, it.next()); // Content-Type is null + assertEquals(Format.get("jpg"), it.next()); // magic bytes assertThrows(NoSuchElementException.class, it::next); } @@ -478,6 +519,7 @@ void testGetRequestInfoUsingScriptLookupStrategyReturningHash() assertEquals("secret", actual.getSecret()); Headers headers = actual.getHeaders(); assertEquals("yes", headers.getFirstValue("X-Custom")); + assertTrue(actual.isSendingHeadRequest()); } @Test @@ -558,44 +600,51 @@ private void doTestNewStreamFactoryWithPresentReadableImage(Identifier identifie assertNotNull(instance.newStreamFactory()); } + /** + * Simulates a full usage cycle, checking that no unnecessary requests are + * made. + */ @Test - void testNoUnnecessaryRequests() throws Exception { - final AtomicInteger numHEADRequests = new AtomicInteger(0); - final AtomicInteger numGETRequests = new AtomicInteger(0); + void testNoUnnecessaryRequestsWithHEADRequestsEnabled() throws Exception { + final RequestCountingHandler handler = new RequestCountingHandler(); + server.setHandler(handler); + server.start(); - server.setHandler(new DefaultHandler() { - @Override - public void handle(String target, - Request baseRequest, - HttpServletRequest request, - HttpServletResponse response) { - switch (request.getMethod().toUpperCase()) { - case "HEAD": - numHEADRequests.incrementAndGet(); - break; - case "GET": - numGETRequests.incrementAndGet(); - break; - default: - throw new IllegalArgumentException( - "Unexpected method: " + request.getMethod()); - } - baseRequest.setHandled(true); - } - }); + instance.stat(); + instance.getFormatIterator().next(); + + StreamFactory source = instance.newStreamFactory(); + try (InputStream is = source.newInputStream()) { + is.readAllBytes(); + } + + assertEquals(1, handler.numHEADRequests); + assertEquals(1, handler.numGETRequests); + } + + /** + * Simulates a full usage cycle, checking that no unnecessary requests are + * made. + */ + @Test + void testNoUnnecessaryRequestsWithHEADRequestsDisabled() throws Exception { + var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_SEND_HEAD_REQUESTS, false); + + final RequestCountingHandler handler = new RequestCountingHandler(); + server.setHandler(handler); server.start(); - instance.checkAccess(); + instance.stat(); instance.getFormatIterator().next(); StreamFactory source = instance.newStreamFactory(); try (InputStream is = source.newInputStream()) { - //noinspection ResultOfMethodCallIgnored - is.read(); + is.readAllBytes(); } - assertEquals(1, numHEADRequests.get()); - assertEquals(1, numGETRequests.get()); + assertEquals(0, handler.numHEADRequests); + assertEquals(2, handler.numGETRequests); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java index 2e14cbcf9..78a8eddad 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java @@ -16,7 +16,9 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Timestamp; import java.sql.Types; +import java.time.Instant; import java.util.NoSuchElementException; import static org.junit.jupiter.api.Assertions.*; @@ -73,6 +75,7 @@ void initializeEndpoint() throws Exception { String sql = "CREATE TABLE IF NOT EXISTS items (" + "filename VARCHAR(255)," + "media_type VARCHAR(255)," + + "last_modified TIMESTAMP," + "image BLOB);"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.execute(); @@ -85,19 +88,21 @@ void initializeEndpoint() throws Exception { IMAGE_WITH_EXTENSION_WITHOUT_MEDIA_TYPE, IMAGE_WITH_INCORRECT_EXTENSION_WITHOUT_MEDIA_TYPE, IMAGE_WITHOUT_EXTENSION_OR_MEDIA_TYPE}) { - sql = "INSERT INTO items (filename, media_type, image) VALUES (?, ?, ?);"; + sql = "INSERT INTO items (filename, last_modified, media_type, image) " + + "VALUES (?, ?, ?, ?);"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, filename); + statement.setTimestamp(2, Timestamp.from(Instant.now())); if (IMAGE_WITHOUT_EXTENSION_OR_MEDIA_TYPE.equals(filename) || IMAGE_WITH_EXTENSION_WITHOUT_MEDIA_TYPE.equals(filename) || IMAGE_WITH_INCORRECT_EXTENSION_WITHOUT_MEDIA_TYPE.equals(filename)) { - statement.setNull(2, Types.VARCHAR); + statement.setNull(3, Types.VARCHAR); } else { - statement.setString(2, "image/jpeg"); + statement.setString(3, "image/jpeg"); } try (InputStream is = Files.newInputStream(TestUtil.getImage("jpg"))) { - statement.setBinaryStream(3, is); + statement.setBinaryStream(4, is); } statement.executeUpdate(); } @@ -126,20 +131,6 @@ void useScriptLookupStrategy() { // This source is always using ScriptLookupStrategy. } - /* checkAccess() */ - - @Override - @Test - void testCheckAccessUsingBasicLookupStrategyWithMissingImage() { - Identifier identifier = new Identifier("bogus"); - DelegateProxy proxy = TestUtil.newDelegateProxy(); - proxy.getRequestContext().setIdentifier(identifier); - instance.setDelegateProxy(proxy); - instance.setIdentifier(identifier); - - assertThrows(NoSuchFileException.class, instance::checkAccess); - } - /* getFormatIterator() */ @Test @@ -227,4 +218,18 @@ void testNewStreamFactoryWithPresentImage() throws Exception { assertNotNull(instance.newStreamFactory()); } + /* stat() */ + + @Override + @Test + void testStatUsingBasicLookupStrategyWithMissingImage() { + Identifier identifier = new Identifier("bogus"); + DelegateProxy proxy = TestUtil.newDelegateProxy(); + proxy.getRequestContext().setIdentifier(identifier); + instance.setDelegateProxy(proxy); + instance.setIdentifier(identifier); + + assertThrows(NoSuchFileException.class, instance::stat); + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java index cdb989640..b1d556497 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java @@ -11,7 +11,8 @@ public class MockFileSource extends AbstractSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { + return null; } @Override diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java index 402e8c0a5..5ecbfa6de 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java @@ -9,7 +9,8 @@ public class MockStreamSource extends AbstractSource implements Source { @Override - public void checkAccess() { + public StatResult stat() { + return null; } @Override diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java index be0707093..02a362a08 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java @@ -10,8 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; @@ -48,8 +48,8 @@ public void handle(String target, }); server.start(); - final HTTPRequestInfo requestInfo = - new HTTPRequestInfo(server.getHTTPURI().toString()); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI().toString()); final OkHttpHTTPImageInputStreamClient instance = new OkHttpHTTPImageInputStreamClient(requestInfo); @@ -71,8 +71,10 @@ public void handle(String target, }); server.start(); - final HTTPRequestInfo requestInfo = new HTTPRequestInfo( - server.getHTTPURI().toString(), "user", "secret"); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI().toString()); + requestInfo.setUsername("user"); + requestInfo.setSecret("secret"); final OkHttpHTTPImageInputStreamClient instance = new OkHttpHTTPImageInputStreamClient(requestInfo); @@ -93,8 +95,8 @@ public void handle(String target, }); server.start(); - final HTTPRequestInfo requestInfo = - new HTTPRequestInfo(server.getHTTPURI().toString()); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI().toString()); requestInfo.getHeaders().add("X-Cats", "yes"); final OkHttpHTTPImageInputStreamClient instance = @@ -107,8 +109,8 @@ public void handle(String target, void sendGETRequest() throws Exception { server.start(); - final HTTPRequestInfo requestInfo = - new HTTPRequestInfo(server.getHTTPURI() + "/jpg"); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI() + "/jpg"); final OkHttpHTTPImageInputStreamClient instance = new OkHttpHTTPImageInputStreamClient(requestInfo); @@ -134,8 +136,10 @@ public void handle(String target, }); server.start(); - final HTTPRequestInfo requestInfo = new HTTPRequestInfo( - server.getHTTPURI() + "/jpg", "user", "secret"); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI() + "/jpg"); + requestInfo.setUsername("user"); + requestInfo.setSecret("secret"); final OkHttpHTTPImageInputStreamClient instance = new OkHttpHTTPImageInputStreamClient(requestInfo); @@ -156,8 +160,8 @@ public void handle(String target, }); server.start(); - final HTTPRequestInfo requestInfo = - new HTTPRequestInfo(server.getHTTPURI() + "/jpg"); + final HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI(server.getHTTPURI() + "/jpg"); requestInfo.getHeaders().add("X-Cats", "yes"); final OkHttpHTTPImageInputStreamClient instance = new OkHttpHTTPImageInputStreamClient(requestInfo); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java index 2579b17ab..0df700cb3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java @@ -103,7 +103,9 @@ public void setUp() throws Exception { configureS3Source(); seedFixtures(); - S3ObjectInfo info = new S3ObjectInfo(FIXTURE_KEY, bucket()); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucket()); + info.setKey(FIXTURE_KEY); info.setLength(1584); instance = new S3HTTPImageInputStreamClient(info); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java index 63fc3b0cc..5a91fad41 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java @@ -202,27 +202,55 @@ void useScriptLookupStrategy() { } } + /* getClientInstance() */ + + @Test + void getClientInstanceReturnsDefaultClient() { + assertNotNull(S3Source.getClientInstance(new S3ObjectInfo())); + } + + @Test + void getClientInstanceReturnsUniqueClientsPerEndpoint() { + S3ObjectInfo info1 = new S3ObjectInfo(); + S3ObjectInfo info2 = new S3ObjectInfo(); + info2.setEndpoint("http://example.org/endpoint"); + S3Client client1 = S3Source.getClientInstance(info1); + S3Client client2 = S3Source.getClientInstance(info2); + assertNotSame(client1, client2); + } + + @Test + void getClientInstanceCachesReturnedClients() { + S3ObjectInfo info1 = new S3ObjectInfo(); + S3ObjectInfo info2 = new S3ObjectInfo(); + info1.setEndpoint("http://example.org/endpoint"); + info2.setEndpoint(info1.getEndpoint()); + S3Client client1 = S3Source.getClientInstance(info1); + S3Client client2 = S3Source.getClientInstance(info2); + assertSame(client1, client2); + } + /* checkAccess() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { + void checkAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { // TODO: write this } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() + void checkAccessUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); - instance.checkAccess(); + instance.stat(); } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableImage() { + void checkAccessUsingScriptLookupStrategyWithPresentUnreadableImage() { // TODO: write this } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { + void checkAccessUsingScriptLookupStrategyWithMissingImage() { useScriptLookupStrategy(); Identifier identifier = new Identifier("bogus"); @@ -231,12 +259,11 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, () -> instance.checkAccess()); + assertThrows(NoSuchFileException.class, () -> instance.stat()); } @Test - void testCheckAccessUsingScriptLookupStrategyReturningHash() - throws Exception { + void checkAccessUsingScriptLookupStrategyReturningHash() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier("bucket:" + getBucket() + @@ -247,11 +274,11 @@ void testCheckAccessUsingScriptLookupStrategyReturningHash() proxy.getRequestContext().setIdentifier(identifier); instance.setDelegateProxy(proxy); - instance.checkAccess(); + instance.stat(); } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { + void checkAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { useScriptLookupStrategy(); Identifier identifier = new Identifier("key:" + OBJECT_KEY_WITH_CONTENT_TYPE_AND_RECOGNIZED_EXTENSION); @@ -260,13 +287,13 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(IllegalArgumentException.class, () -> instance.checkAccess()); + assertThrows(IllegalArgumentException.class, () -> instance.stat()); } /* getFormatIterator() */ @Test - void testGetFormatIteratorHasNext() { + void getFormatIteratorHasNext() { S3Source source = newInstance(); source.setIdentifier(new Identifier(OBJECT_KEY_WITH_CONTENT_TYPE_AND_RECOGNIZED_EXTENSION)); S3Source.FormatIterator it = source.getFormatIterator(); @@ -283,7 +310,7 @@ void testGetFormatIteratorHasNext() { } @Test - void testGetFormatIteratorNext() { + void getFormatIteratorNext() { S3Source source = newInstance(); source.setIdentifier(new Identifier(OBJECT_KEY_WITH_NO_CONTENT_TYPE_AND_INCORRECT_EXTENSION)); @@ -298,12 +325,12 @@ void testGetFormatIteratorNext() { /* getObjectInfo() */ @Test - void testGetObjectInfo() throws Exception { + void getObjectInfo() throws Exception { assertNotNull(instance.getObjectInfo()); } @Test - void testGetObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() + void getObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() throws Exception { Configuration config = Configuration.getInstance(); config.setProperty(Key.S3SOURCE_PATH_PREFIX, "/prefix/"); @@ -314,7 +341,7 @@ void testGetObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() } @Test - void testGetObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() + void getObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() throws Exception { Configuration config = Configuration.getInstance(); config.setProperty(Key.S3SOURCE_PATH_PREFIX, ""); @@ -327,13 +354,12 @@ void testGetObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() /* newStreamFactory() */ @Test - void testNewStreamFactoryUsingBasicLookupStrategy() throws Exception { + void newStreamFactoryUsingBasicLookupStrategy() throws Exception { assertNotNull(instance.newStreamFactory()); } @Test - void testNewStreamFactoryUsingScriptLookupStrategy() - throws Exception { + void newStreamFactoryUsingScriptLookupStrategy() throws Exception { useScriptLookupStrategy(); assertNotNull(instance.newStreamFactory()); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java index b6527a90f..c1432a948 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java @@ -104,7 +104,9 @@ public void setUp() throws Exception { configureS3Source(); seedFixtures(); - S3ObjectInfo info = new S3ObjectInfo(FIXTURE_KEY, bucket()); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucket()); + info.setKey(FIXTURE_KEY); info.setLength(1584); instance = new S3StreamFactory(info); diff --git a/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java b/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java index e6b5f00b1..817c9041f 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java @@ -218,32 +218,6 @@ void testToByteSizeWithPB() { assertEquals(expected, StringUtils.toByteSize("25 pb")); } - @Test - void testTrimXMPWithTrimmableXMP() { - String xmp = "" + - "" + - "" + - "" + - ""; - String result = StringUtils.trimXMP(xmp); - assertTrue(result.startsWith("")); - } - - @Test - void testTrimXMPWithNonTrimmableXMP() { - String xmp = "" + - ""; - String result = StringUtils.trimXMP(xmp); - assertSame(xmp, result); - } - - @Test - void testTrimXMPWithNullArgument() { - assertThrows(NullPointerException.class, - () -> StringUtils.trimXMP(null)); - } - @Test void testWrap() { String str = "This is a very very very very very very very very long line."; diff --git a/src/test/resources/delegates.rb b/src/test/resources/delegates.rb index cf25f5c36..509ee0dda 100644 --- a/src/test/resources/delegates.rb +++ b/src/test/resources/delegates.rb @@ -182,14 +182,16 @@ def httpsource_resource_info(options = {}) 'uri' => 'http://example.org/bla/' + CGI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'https-jpg-rgb-64x56x8-baseline.jpg' return { 'uri' => 'https://example.org/bla/' + CGI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'http-jpg-rgb-64x56x8-plane.jpg' return { @@ -198,7 +200,8 @@ def httpsource_resource_info(options = {}) 'secret' => 'secret', 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'https-jpg-rgb-64x56x8-plane.jpg' return { @@ -207,7 +210,8 @@ def httpsource_resource_info(options = {}) 'secret' => 'secret', 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } end nil @@ -217,6 +221,10 @@ def jdbcsource_database_identifier(options = {}) context['identifier'] end + def jdbcsource_last_modified(options = {}) + 'SELECT last_modified FROM items WHERE filename = ?' + end + def jdbcsource_media_type(options = {}) 'SELECT media_type FROM items WHERE filename = ?' end diff --git a/src/test/resources/xmp/xmp.xmp b/src/test/resources/xmp/xmp.xmp new file mode 100644 index 000000000..4364c491c --- /dev/null +++ b/src/test/resources/xmp/xmp.xmp @@ -0,0 +1,27 @@ + + + + + + Illini Union Photographs Record Series 3707005 + + + + + University of Illinois Library + + + + + + Preservation Master creation + new subject code + + + + + No reproduction without prior permission + + + + \ No newline at end of file diff --git a/src/test/resources/xmp/xmp2.xmp b/src/test/resources/xmp/xmp2.xmp new file mode 100644 index 000000000..979ee87d6 --- /dev/null +++ b/src/test/resources/xmp/xmp2.xmp @@ -0,0 +1,121 @@ + + + + + 3.0 + CRW_1101.CRW + Custom + 5800 + +11 + +1.15 + 0 + 39 + +60 + 0 + 25 + 0 + 15 + 0 + 0 + 0 + 0 + -1 + +24 + -12 + -18 + +5 + -13 + Custom + + + 0, 0 + 34, 16 + 70, 53 + 121, 122 + 184, 214 + 255, 255 + + + ACR 2.4 + True + False + + + 1/100 + 6643856/1000000 + 8/1 + 6/1 + 2 + 2004-08-12T06:57:52-07:00 + 0/1 + 4/1 + 5 + 28/1 + + + 200 + + + + False + + + + 0660217086 + 18.0-55.0 mm + + + + + Canon + Canon EOS DIGITAL REBEL + 6 + 3072 + 2048 + 2 + 240/1 + 240/1 + 2 + + + 16 + 16 + 16 + + + 2004-08-12T06:57:52-07:00 + + + 2004-08-12T06:57:52-07:00 + 2005-03-25T16:27:25-08:00 + 0 + + + True + + + + + Bruce Fraser + + + + + ©2004 Bruce Fraser. All Rights Reserved + + + + + Glencoe + Scotland + Flowers + + + + + lichen and heather in Glencoe + + + + + \ No newline at end of file