Initial commit of draft Chymistry web application, including reading …

…and caching P4 from Xubmit, conversion to P5, indexing in Solr, a simple search, and a stub of a conversion to HTML
IUBLibTech · Mar 19, 2019 · bc0c876 · bc0c876
commit bc0c876
Show file tree

Hide file tree

Showing 22 changed files with 3,571 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -0,0 +1,3 @@
+# Newton Chymistry website
+
+This codebase is a draft of a new version of 'The Chymistry of Isaac Newton', using XProc pipelines to generate a website based on  TEI XML encodings of Newton's alchemical manuscripts, and Apache Solr as a search engine.
diff --git a/search-fields.xml b/search-fields.xml
@@ -0,0 +1,20 @@
+<facets>
+	<facet>
+		<group></group>
+		<name>title</name>
+		<field>title</field>
+		<label>Title</label>
+	</facet>
+	<facet>
+		<group></group>
+		<name>description</name>
+		<field>description</field>
+		<label>Description</label>
+	</facet>
+	<facet>
+		<group></group>
+		<name>text</name>
+		<field>text</field>
+		<label>Text</label>
+	</facet>
+</facets>
diff --git a/xproc/administration.xpl b/xproc/administration.xpl
@@ -0,0 +1,42 @@
+<p:library version="1.0" 
+	xmlns:p="http://www.w3.org/ns/xproc" 
+	xmlns:c="http://www.w3.org/ns/xproc-step" 
+	xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" 
+	xmlns:chymistry="tag:conaltuohy.com,2018:chymistry"
+	xmlns:fn="http://www.w3.org/2005/xpath-functions">
+
+	<p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
+
+	<p:declare-step name="admin-form" type="chymistry:admin-form">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:identity>
+			<p:input port="source">
+				<p:inline>
+					<c:response status="200">
+						<c:body content-type="application/xhtml+xml">
+							<html xmlns="http://www.w3.org/1999/xhtml">
+								<head>
+									<title>Chymistry admin</title>
+								</head>
+								<body>
+									<h1>Chymistry admin</h1>
+									<p>Site updating</p>
+									<form method="post" action="p4/">
+										<button>Download P4 files from Xubmit</button>
+									</form>
+									<form method="post" action="p5/">
+										<button>Convert downloaded P4 files to P5</button>
+									</form>
+									<form method="post" action="reindex/">
+										<button>Rebuild Solr index from P5 files</button>
+									</form>
+								</body>
+							</html>
+						</c:body>
+					</c:response>
+				</p:inline>
+			</p:input>
+		</p:identity>
+	</p:declare-step>
+</p:library>
diff --git a/xproc/convert-to-p5.xpl b/xproc/convert-to-p5.xpl
@@ -0,0 +1,137 @@
+<p:library version="1.0"
+	xmlns:p="http://www.w3.org/ns/xproc" 
+	xmlns:c="http://www.w3.org/ns/xproc-step" 
+	xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" 
+	xmlns:fn="http://www.w3.org/2005/xpath-functions"
+	xmlns:chymistry="tag:conaltuohy.com,2018:chymistry"
+	xmlns:cx="http://xmlcalabash.com/ns/extensions">
+
+	<p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
+
+	<p:declare-step name="download-p4" type="chymistry:download-p4">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:variable name="xubmit-base-uri" select=" 'http://algernon.dlib.indiana.edu:8080/xubmit/rest/repository/newton/' "/>
+		<p:xslt name="manifest">
+			<p:with-param name="base-uri" select="$xubmit-base-uri"/>
+			<p:input port="stylesheet">
+				<p:inline>
+					<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="3.0">
+						<xsl:param name="base-uri"/>
+						<xsl:variable name="xubmit-manifest" select="json-doc(concat($base-uri, 'list?limit=9999'))"/>
+						<xsl:template match="/">
+							<collection>
+								<xsl:for-each select="$xubmit-manifest?results?*">
+									<text id="{.('@rdf:about')}" date="{.('cvs:date')}">
+										<c:request href="{concat($base-uri, .('@rdf:about'), '.xml')}" method="GET" detailed="true" override-content-type="application/octet-stream"/>
+									</text>
+								</xsl:for-each>
+							</collection>
+						</xsl:template>
+					</xsl:stylesheet>
+				</p:inline>
+			</p:input>
+		</p:xslt>
+		<p:for-each name="text-to-download">
+			<p:iteration-source select="/collection/text"/>
+			<p:variable name="id" select="/text/@id"/>
+			<p:http-request name="download">
+				<p:input port="source" select="/text/c:request">
+					<p:pipe step="text-to-download" port="current"/>
+				</p:input>
+			</p:http-request>
+			<p:for-each name="successful-download">
+				<p:iteration-source select="/c:response[@status='200']/c:body">
+					<p:pipe step="download" port="result"/>
+				</p:iteration-source>
+				<p:store method="text" cx:decode="true">
+					<p:with-option name="href" select="concat('../p4/', $id, '.xml')"/>
+				</p:store>
+			</p:for-each>
+			<p:for-each name="unsuccessful-download">
+				<p:iteration-source select="/c:response[@status!='200']/c:body">
+					<p:pipe step="download" port="result"/>
+				</p:iteration-source>
+				<p:store method="text" cx:decode="true">
+					<p:with-option name="href" select="concat('../p4/errors/', $id, '.json')"/>
+				</p:store>
+			</p:for-each>
+		</p:for-each>
+		<z:make-http-response content-type="application/xml">
+			<p:input port="source">
+				<p:pipe step="manifest" port="result"/>
+			</p:input>
+		</z:make-http-response>
+		</p:declare-step>
+	<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" name="convert-p4-to-p5" version="1.0" type="chymistry:convert-p4-to-p5"
+		xmlns:chymistry="tag:conaltuohy.com,2018:chymistry"
+		xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" 
+		xmlns:c="http://www.w3.org/ns/xproc-step"
+		xmlns:cx="http://xmlcalabash.com/ns/extensions">
+		<p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
+		<p:import href="xproc-z-library.xpl"/>	
+		<p:input port="source" primary="true"/>
+		<p:output port="result"/>
+		<p:directory-list name="list-p4-files" path="../p4/"/>
+		<p:add-xml-base relative="false" all="true"/>
+		<p:for-each>
+			<p:iteration-source select="
+				//c:file
+					[ends-with(@name, '.xml')]
+					[not(starts-with(@name, 'iu.'))]
+					[not(@name='schemas.xml')]
+			"/>
+			<p:variable name="file-name" select="/c:file/@name"/>
+			<p:variable name="file-uri" select="encode-for-uri($file-name)"/>
+			<p:variable name="input-file" select="resolve-uri($file-uri, /c:file/@xml:base)"/>
+			<p:variable name="output-file" select="concat('../p5/', $file-uri)"/>
+			<cx:message>
+				<p:with-option name="message" select="$file-name"/>
+			</cx:message>
+			<p:load name="read-p4" dtd-validate="true">
+				<p:with-option name="href" select="$input-file"/>
+			</p:load>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/links.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/remove-dubious-default.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/purge-foreign.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/regularize-dates.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/p4_to_p5_newton.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:xslt>
+				<p:input port="parameters"><p:empty/></p:input>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/convert-to-p5/select-non-emoji-glyphs.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:store name="save-p5-file">
+				<p:with-option name="href" select="$output-file"/>
+			</p:store>
+		</p:for-each>
+		<p:directory-list name="list-p5-files" path="../p5/"/>
+		<z:make-http-response content-type="application/xml"/>
+	</p:declare-step>
+</p:library>
diff --git a/xproc/p5-processing.xpl b/xproc/p5-processing.xpl
@@ -0,0 +1,68 @@
+<p:library version="1.0" 
+	xmlns:p="http://www.w3.org/ns/xproc" 
+	xmlns:c="http://www.w3.org/ns/xproc-step" 
+	xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" 
+	xmlns:fn="http://www.w3.org/2005/xpath-functions"
+	xmlns:chymistry="tag:conaltuohy.com,2018:chymistry"
+	xmlns:cx="http://xmlcalabash.com/ns/extensions">
+
+	<p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
+
+	<p:declare-step name="reindex" type="chymistry:reindex">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:directory-list name="list-p5-files" path="../p5/"/>
+		<p:add-xml-base relative="false" all="true"/>
+		<p:for-each>
+			<p:iteration-source select="//c:file"/>
+			<p:variable name="file-name" select="/c:file/@name"/>
+			<p:variable name="file-id" select="substring-before($file-name, '.xml')"/>
+			<p:variable name="file-uri" select="encode-for-uri($file-name)"/>
+			<p:variable name="input-file" select="resolve-uri($file-uri, /c:file/@xml:base)"/>
+			<p:variable name="output-file" select="concat('../p5/', $file-uri)"/>
+			<cx:message>
+				<p:with-option name="message" select="$file-name"/>
+			</cx:message>
+			<p:load name="read-p5">
+				<p:with-option name="href" select="$input-file"/>
+			</p:load>
+			<p:xslt>
+				<p:with-param name="id" select="$file-id"/>
+				<p:input port="stylesheet">
+					<p:document href="../xslt/p5-to-solr-index-request.xsl"/>
+				</p:input>
+			</p:xslt>
+			<p:http-request/>
+		</p:for-each>
+		<p:wrap-sequence wrapper="solr-index-responses"/>
+		<z:make-http-response/>
+	</p:declare-step>
+
+
+	<p:declare-step name="p5-as-html" type="chymistry:p5-as-html">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:variable name="text" select="substring-before(substring-after(/c:request/@href, 'xproc-z/text/'), '/')"/>
+		<p:load name="text">
+			<p:with-option name="href" select="concat('../p5/', $text, '.xml')"/>
+		</p:load>
+		<p:xslt>
+			<p:input port="parameters"><p:empty/></p:input>
+			<p:input port="stylesheet">
+				<p:document href="../xslt/p5-to-html.xsl"/>
+			</p:input>
+		</p:xslt>
+		<z:make-http-response content-type="application/xhtml+xml"/>
+	</p:declare-step>
+
+	<p:declare-step name="p5-as-xml" type="chymistry:p5-as-xml">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:variable name="text" select="substring-before(substring-after(/c:request/@href, 'xproc-z/p5/'), '/')"/>
+		<p:load name="text">
+			<p:with-option name="href" select="concat('../p5/', $text, '.xml')"/>
+		</p:load>
+		<z:make-http-response content-type="application/xml"/>
+	</p:declare-step>
+
+</p:library>
diff --git a/xproc/search.xpl b/xproc/search.xpl
@@ -0,0 +1,97 @@
+<p:library version="1.0" 
+	xmlns:p="http://www.w3.org/ns/xproc" 
+	xmlns:c="http://www.w3.org/ns/xproc-step" 
+	xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" 
+	xmlns:chymistry="tag:conaltuohy.com,2018:chymistry"
+	xmlns:fn="http://www.w3.org/2005/xpath-functions">
+
+	<p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
+
+
+
+	<p:declare-step name="search" type="chymistry:search">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:choose>
+			<p:when test="substring-after(/c:request/@href, '?')">
+				<chymistry:search-results/>
+			</p:when>
+			<p:otherwise>
+				<!-- ... otherwise display a search form -->
+				<chymistry:search-form/>
+			</p:otherwise>
+		</p:choose>
+	</p:declare-step>
+
+
+
+	<p:declare-step name="search-results" type="chymistry:search-results">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<!-- TODO, if there are URL parameters, perform a search and display results -->
+		<p:www-form-urldecode name="fields">
+			<p:with-option name="value" select="substring-after(/c:request/@href, '?')"/>
+		</p:www-form-urldecode>
+		<p:load name="facets" href="../search-fields.xml"/>
+		<p:wrap-sequence name="facets-and-fields" wrapper="facets-and-fields">
+			<p:input port="source">
+				<p:pipe step="facets" port="result"/>
+				<p:pipe step="fields" port="result"/>
+			</p:input>
+		</p:wrap-sequence>
+		<p:xslt name="prepare-solr-request">
+			<p:input port="parameters"><p:empty/></p:input>
+			<p:input port="stylesheet"><p:document href="../xslt/search-parameters-to-solr-request.xsl"/></p:input>
+		</p:xslt>
+		<p:xslt name="convert-xml-to-json">
+			<p:input port="parameters"><p:empty/></p:input>
+			<p:input port="stylesheet"><p:document href="../xslt/convert-between-xml-and-json.xsl"/></p:input>
+		</p:xslt>
+		<p:http-request/>
+		<p:xslt name="convert-json-to-xml">
+			<p:input port="parameters"><p:empty/></p:input>
+			<p:input port="stylesheet"><p:document href="../xslt/convert-between-xml-and-json.xsl"/></p:input>
+		</p:xslt>
+		<p:wrap-sequence name="request-and-response" wrapper="request-and-reponse">
+			<p:input port="source">
+				<p:pipe step="fields" port="result"/>
+				<p:pipe step="convert-json-to-xml" port="result"/>
+				<p:pipe step="facets" port="result"/>
+			</p:input>
+		</p:wrap-sequence>
+		<p:xslt name="render-solr-response">
+			<p:input port="parameters"><p:empty/></p:input>
+			<p:input port="stylesheet"><p:document href="../xslt/solr-response-to-html.xsl"/></p:input>
+		</p:xslt>
+		<z:make-http-response content-type="application/xhtml+xml"/>
+		<!-- TODO query solr, transform results to html -->
+		<!--<z:make-http-response/>--><!-- for now, just dump search request -->
+	</p:declare-step>
+
+	<p:declare-step name="search-form" type="chymistry:search-form">
+		<p:input port="source"/>
+		<p:output port="result"/>
+		<p:identity>
+			<p:input port="source">
+				<p:inline>
+					<c:response status="200">
+						<c:body content-type="application/xhtml+xml">
+							<html xmlns="http://www.w3.org/1999/xhtml">
+								<head>
+									<title>Chymistry search</title>
+								</head>
+								<body>
+									<h1>Chymistry search</h1>
+									<form method="get" action="">
+										<input type="text" name="text"/>
+										<button>search</button>
+									</form>
+								</body>
+							</html>
+						</c:body>
+					</c:response>
+				</p:inline>
+			</p:input>
+		</p:identity>
+	</p:declare-step>
+</p:library>
diff --git a/xproc/temp-decode-p4-docs.xpl b/xproc/temp-decode-p4-docs.xpl
@@ -0,0 +1,21 @@
+
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" xmlns:c="http://www.w3.org/ns/xproc-step" xmlns:z="https://github.com/Conal-Tuohy/XProc-Z" xmlns:chymistry="tag:conaltuohy.com,2018:chymistry" xmlns:cx="http://xmlcalabash.com/ns/extensions" version="1.0" name="main">
+
+	<p:directory-list path="../p4" include-filter="^.*.xml$"/>
+	<p:for-each name="item">
+		<p:iteration-source select="
+			//c:file
+				[ends-with(@name, '.xml')]
+				[not(starts-with(@name, 'iu.'))]
+				[not(@name='schemas.xml')]
+		"/>
+		<p:variable name="name" select="/c:file/@name"/>
+		<p:load>
+			<p:with-option name="href" select="concat('../p4/', $name)"/>
+		</p:load>
+		<p:store method="text">
+			<p:with-option name="href" select="concat('../p5/', $name)"/>
+		</p:store>
+	</p:for-each>
+
+</p:declare-step>