Skip to content

Commit

Permalink
Create vivo/home on application start-up (#192.1)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wwelling
wwelling committed Feb 23, 2023
1 parent 15cb4d7 commit 4a9e598
Show file tree
Hide file tree
Showing 12 changed files with 409 additions and 98 deletions.
15 changes: 13 additions & 2 deletions api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/ApplicationSetup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
*/
public class ApplicationSetup implements ServletContextListener {
private static final String APPLICATION_SETUP_PATH = "config/applicationSetup.n3";
private static final String APPLICATION_SETUP_DEFAULT_PATH = "config/default.applicationSetup.n3";

private ServletContext ctx;
private StartupStatus ss;
Expand All @@ -45,6 +46,8 @@ public void contextInitialized(ServletContextEvent sce) {
this.vitroHomeDir = VitroHomeDirectory.find(ctx);
ss.info(this, vitroHomeDir.getDiscoveryMessage());

this.vitroHomeDir.populate();

locateApplicationConfigFile();
loadApplicationConfigFile();
createConfigurationBeanLoader();
Expand All @@ -63,11 +66,19 @@ public void contextInitialized(ServletContextEvent sce) {
private void locateApplicationConfigFile() {
Path path = this.vitroHomeDir.getPath().resolve(APPLICATION_SETUP_PATH);

if (!Files.exists(path) || !Files.isReadable(path)) {
path = this.vitroHomeDir.getPath().resolve(APPLICATION_SETUP_DEFAULT_PATH);
}

if (!Files.exists(path)) {
throw new IllegalStateException("'" + path + "' does not exist.");
throw new IllegalStateException("Neither '" + APPLICATION_SETUP_PATH + "' nor '" +
APPLICATION_SETUP_DEFAULT_PATH + "' were found in " +
this.vitroHomeDir.getPath());
}
if (!Files.isReadable(path)) {
throw new IllegalStateException("Can't read '" + path + "'");
throw new IllegalStateException("No readable '" + APPLICATION_SETUP_PATH + "' nor '" +
APPLICATION_SETUP_DEFAULT_PATH + "' files were found in " +
this.vitroHomeDir.getPath());
}
this.configFile = path;
}
Expand Down
261 changes: 242 additions & 19 deletions api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/VitroHomeDirectory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,46 @@

import static edu.cornell.mannlib.vitro.webapp.application.BuildProperties.WEBAPP_PATH_BUILD_PROPERTIES;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.naming.InitialContext;
import javax.servlet.ServletContext;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import edu.cornell.mannlib.vitro.webapp.config.ContextProperties;

/**
* Encapsulates some of the info relating to the Vitro home directory.
* Encapsulates some of the info relating to and initializes the Vitro home directory.
*/
public class VitroHomeDirectory {
private static final Log log = LogFactory.getLog(VitroHomeDirectory.class);

private static final String DIGEST_FILE_NAME = "digest.md5";

private static final Pattern CHECKSUM_PATTERN = Pattern.compile("^[a-f0-9]{32} \\*.+$");

public static VitroHomeDirectory find(ServletContext ctx) {
HomeDirectoryFinder finder = new HomeDirectoryFinder(ctx);
return new VitroHomeDirectory(ctx, finder.getPath(),
Expand Down Expand Up @@ -52,6 +73,219 @@ public String getDiscoveryMessage() {
return discoveryMessage;
}

/**
* Populates VIVO home directory with files required to run.
*
* NOTE: Will not overwrite any modified files on redeploy.
*/
public void populate() {
File vhdDir = getPath().toFile();

if (!vhdDir.isDirectory() || vhdDir.list() == null) {
throw new RuntimeException("Application home dir is not a directory! " + vhdDir);
}

Map<String, String> digest = untar(vhdDir);

writeDigest(digest);
}

/**
* A non-destructive untar process that returns checksum digest of tarred files.
*
* Checksum digest can be manually created with the following command.
*
* `find /vivo/home -type f | cut -c3- | grep -E '^bin/|^config/|^rdf/' | xargs md5sum > /vivo/home/digest.md5`
*
* @param destination VIVO home directory
* @return digest of each files checksum
*/
private Map<String, String> untar(File destination) {
log.info("Syncing VIVO home at: " + destination.getPath());

Map<String, String> digest = new HashMap<>();
Map<String, String> storedDigest = loadDigest();

TarArchiveEntry tarEntry;
try (
InputStream homeDirTar = getHomeDirTar();
TarArchiveInputStream tarInput = new TarArchiveInputStream(homeDirTar);
) {
while ((tarEntry = tarInput.getNextTarEntry()) != null) {

// Use the example configurations
String outFilename = tarEntry.getName().replace("example.", "");
File outFile = new File(destination, outFilename);

// Is the entry a directory?
if (tarEntry.isDirectory()) {
if (!outFile.exists()) {
outFile.mkdirs();
}
} else {
// Entry is a File
boolean write = true;

// reading bytes into memory to avoid having to unreliably reset stream
byte[] bytes = IOUtils.toByteArray(tarInput);
String newFileChecksum = checksum(bytes);
digest.put(outFilename, newFileChecksum);

// if file already exists and stored digest contains the file,
// check to determine if it has changed
if (outFile.exists() && storedDigest.containsKey(outFilename)) {
String existingFileChecksum = checksum(outFile);
// if file has not changed in home and is not the same as new file, overwrite
write = storedDigest.get(outFilename).equals(existingFileChecksum)
&& !existingFileChecksum.equals(newFileChecksum);
}

if (write) {
outFile.getParentFile().mkdirs();
try (
InputStream is = new ByteArrayInputStream(bytes);
FileOutputStream fos = new FileOutputStream(outFile);
) {
IOUtils.copy(is, fos);
log.info(outFile.getAbsolutePath() + " source has changed and has not been "
+ "edited in home, updated file has been copied to home directory.");
}
} else {
log.debug(outFile.getAbsolutePath() + " has been preserved.");
}
}
}
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException("Error creating home directory!", e);
}

return digest;
}

/**
* Load checksum digest of VIVO home directory.
*
* @return checksum digest
*/
private Map<String, String> loadDigest() {
File storedDigest = new File(getPath().toFile(), DIGEST_FILE_NAME);
if (storedDigest.exists() && storedDigest.isFile()) {
log.info("Reading VIVO home digest: " + storedDigest.getPath());
try {
return FileUtils
.readLines(storedDigest, StandardCharsets.UTF_8)
.stream()
.filter(CHECKSUM_PATTERN.asPredicate())
.map(this::split)
.collect(Collectors.toMap(this::checksumFile, this::checksumValue));
} catch (IOException e) {
throw new RuntimeException("Error reading VIVO home checksum digest!", e);
}
}
log.info("VIVO home digest not found: " + storedDigest.getPath());

return new HashMap<>();
}

/**
* Write VIVO home checksum digest following md5 format; `<checksum> *<file>`.
*
* @param digest checksum digest to write
*/
private void writeDigest(Map<String, String> digest) {
File storedDigest = new File(getPath().toFile(), DIGEST_FILE_NAME);
try (
FileOutputStream fos = new FileOutputStream(storedDigest);
OutputStreamWriter osw = new OutputStreamWriter(fos);
) {
for (Map.Entry<String, String> entry : digest.entrySet()) {
String filename = entry.getKey();
String checksum = entry.getValue();
osw.write(String.format("%s *%s\n", checksum, filename));
}
} catch (IOException e) {
throw new RuntimeException("Error writing home directory checksum digest!", e);
}
log.info("VIVO home digest created: " + storedDigest.getPath());
}

/**
* Split checksum.
*
* @param checksum checksum delimited by space and asterisks `<checksum> *<file>`
* @return split checksum
*/
private String[] split(String checksum) {
return checksum.split("\\s+");
}

/**
* Get value from split checksum.
*
* @param checksum split checksum
* @return checksum value
*/
private String checksumValue(String[] checksum) {
return checksum[0];
}

/**
* Return file from split checksum.
*
* @param checksum split checksum
* @return filename
*/
private String checksumFile(String[] checksum) {
return checksum[1].substring(1);
}

/**
* Get md5 checksum from file.
*
* @param file file
* @return md5 checksum as string
* @throws IOException
* @throws NoSuchAlgorithmException
*/
private String checksum(File file) throws IOException, NoSuchAlgorithmException {
return checksum(FileUtils.readFileToByteArray(file));
}

/**
* Get md5 checksum from bytes.
*
* @param bytes bytes from file
* @return md5 checksum as string
* @throws NoSuchAlgorithmException
*/
private String checksum(byte[] bytes) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(bytes);
// bytes to hex
StringBuilder result = new StringBuilder();
for (byte b : md.digest()) {
result.append(String.format("%02x", b));
}

return result.toString();
}

/**
* Get prepacked VIVO home tar file as input stream.
*
* @return input stream of VIVO home tar file
*/
private InputStream getHomeDirTar() {
String tarLocation = "/WEB-INF/resources/home-files/vivo-home.tar";
InputStream tar = ctx.getResourceAsStream(tarLocation);
if (tar == null) {
log.error("Application home tar not found in: " + tarLocation);
throw new RuntimeException("Application home tar not found in: " + tarLocation);
}

return tar;
}

/**
* Find something that specifies the location of the Vitro home directory.
* Look in the JDNI environment, the system properties, and the
Expand Down Expand Up @@ -92,23 +326,12 @@ public Path getPath() {
}

public void getVhdFromJndi() {
try {
String vhdPath = (String) new InitialContext()
.lookup(VHD_JNDI_PATH);
if (vhdPath == null) {
log.debug("Didn't find a JNDI value at '" + VHD_JNDI_PATH
+ "'.");
} else {
log.debug("'" + VHD_JNDI_PATH + "' as specified by JNDI: "
+ vhdPath);
String message = String.format(
"JNDI environment '%s' was set to '%s'",
VHD_JNDI_PATH, vhdPath);
foundLocations.add(new Found(Paths.get(vhdPath), message));
}
} catch (Exception e) {
log.debug("JNDI lookup failed. " + e);
}
String vhdPath = ContextProperties.findJndiProperty(VHD_JNDI_PATH);
log.debug("'" + VHD_JNDI_PATH + "' as specified by JNDI: " + vhdPath);
String message = String.format(
"JNDI environment '%s' was set to '%s'",
VHD_JNDI_PATH, vhdPath);
foundLocations.add(new Found(Paths.get(vhdPath), message));
}

private void getVhdFromSystemProperties() {
Expand Down
4 changes: 3 additions & 1 deletion api/src/main/java/edu/cornell/mannlib/vitro/webapp/config/ConfigurationPropertiesImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ public class ConfigurationPropertiesImpl extends ConfigurationProperties {

public ConfigurationPropertiesImpl(InputStream stream,
Map<String, String> preemptiveProperties,
Map<String, String> buildProperties) throws IOException {
Map<String, String> buildProperties,
Map<String, String> contextProperties) throws IOException {
Map<String, String> map = new HashMap<>(buildProperties);
map.putAll(contextProperties);

Properties props = loadFromPropertiesFile(stream);
for (String key: props.stringPropertyNames()) {
Expand Down

0 comments on commit 4a9e598

Please sign in to comment.