Merge pull request #810 from FreeAndFair/auditcenter

rla_export command, module to export data in json, csv spreadsheet formats

Dockerfile

@@ -1,4 +1,4 @@
-FROM centos
+FROM centos:7
 MAINTAINER Neal McBurnett <[email protected]>
 ENV container docker
 RUN yum -y update; yum clean all
@@ -26,8 +26,9 @@ RUN yum -y install ./epel-release-latest-*.noarch.rpm
 RUN yum -y update
 RUN yum -y install python-pip
 RUN yum -y install tmux
+RUN yum -y install python-psycopg2.x86_64
 
-EXPOSE 8888 80 443
+EXPOSE 80 443
 # TODO PostgreSQL port for debugging only
 EXPOSE 5432
 

docs/Makefile

@@ -11,7 +11,7 @@ MD_DOCS := 01_book.md 05_license.md 15_installation.md 25_developer.md \
            50_requirements.md 55_specification.md 65_deployment.md 70_team.md \
            89_glossary.md 99_bibliography.md
 
-all: book.pdf book.html county_runbook.docx sos_runbook.docx user_manual.docx 
+all: book.pdf book.html county_runbook.docx sos_runbook.docx user_manual.docx export_manual.docx export_manual.html
 
 book.pdf: $(MD_DOCS)
 	pandoc $(MD_DOCS) --toc --variable linkcolor -f markdown -t latex -o book.pdf
@@ -28,5 +28,11 @@ sos_runbook.docx: 24_sos_runbook.md
 user_manual.docx: 20_user_manual.md
 	pandoc 20_user_manual.md 89_glossary.md --variable linkcolor --table-of-contents --reference-docx=reference_fandf_pages.docx -f markdown -t docx -o user_manual.docx
 
+export_manual.docx: 26_export_manual.md
+	pandoc 26_export_manual.md --variable linkcolor --table-of-contents --reference-docx=reference_fandf_pages.docx -f markdown -t docx -o export_manual.docx 
+
+export_manual.html: 26_export_manual.md
+	pandoc 26_export_manual.md --variable linkcolor --table-of-contents -f markdown -o export_manual.html
+
 clean:
 	rm -f book.pdf book.html runbook.docx county_runbook.docx sos_runbook.docx user_manual.docx

docs/book.html

@@ -450,7 +450,10 @@ <h1 id="glossary">Glossary</h1>
 </ul>
 
 <h1 id="bibliography-1">Bibliography</h1>
-<p><em>Introduction to be written. We will be creating a new repository for the bibliography. The contents will be copied from the <a href="https://github.com/FreeAndFair/OpenRLA">OpenRLA repository</a> and organized here.</em></p>
+<ul>
+<li>See our <a href="https://github.com/FreeAndFair/Bibliography">Bibliography</a> repository for materials related to Free &amp; Fair's R&amp;D.</li>
+</ul>
+<p>Introduction <em>to be written</em>.</p>
 <h2 id="risk-limiting-audits">Risk-Limiting Audits</h2>
 <p><em>To be written.</em></p>
 <h2 id="jurisdiction-sources">Jurisdiction Sources</h2>

server/eclipse-project/script/build-release

@@ -27,7 +27,7 @@ cp $jar $reldir/corla-server.jar
 
 cp ../deploy/corla.conf $reldir
 
-echo 'Checkout an alternate branch now if desired'
+echo 'Checkout an alternate branch now if desired. Hit return when ready.'
 read
 echo Proceeding
 

server/eclipse-project/script/rla_export/.gitignore

@@ -0,0 +1,9 @@
+# Compiled python modules.
+*.pyc
+
+# Setuptools distribution folder.
+/dist/
+/build/
+
+# Python egg metadata, regenerated from source files by setuptools.
+/*.egg-info

server/eclipse-project/script/rla_export/README.rst

@@ -0,0 +1,115 @@
+Documentation for ``rla_export``
+================================
+
+The ``rla_export`` command exports ColoradoRLA data for publication
+on the Audit Center web site, and for other monitoring and diagnostic
+purposes.
+
+Install the pip packagemanager
+------------------------------
+
+If some version of Python's package manager pip is already installed, skip to next section.
+
+Pip installation depends on the operating system and version.
+
+For RHEL 7, as a one-time step on each RLA Tool server, run these commands to install the
+"pip" command for installing packages and the necessary dependencies.
+
+::
+
+    yum install epel-release
+    curl https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm > epel-release-latest-7.noarch.rpm
+    yum install ./epel-release-latest-*.noarch.rpm
+    yum install python-pip
+    sudo pip install --upgrade pip
+
+
+Install rla_export
+------------------
+
+Once pip is installed, you use it to install rla_export and all the packages it depends on.
+Locate the ``rla_export*.tar.gz`` file provided in the deliverable, and
+run a command like this on it.
+Note that the version number may be different than that shown and you'll need to specify a
+properly qualified path name if it isn't in the current directory.
+
+  ``sudo pip install rla_export-1.1.0.tar.gz --upgrade``
+
+Setup procedure for rla_export
+------------------------------
+
+Make a copy of the supplied ``corla-template.ini`` file and modify it, following
+the directions in that file.
+
+In it you should specify the path to the properties file for database configration,
+as well as connection and authentication information for your application server.
+
+If you are using DatabaseAuthentication, you can specify both a state administrator user and password,
+and put any value in for the three grid values so all the authentication prompts will be skipped.
+
+You use the ``-C`` option of rla_export to specify your configuration file.
+
+Running the Export Command
+--------------------------
+
+By default, the command will run queries using
+all the standard ``.sql`` files provided in the package, and
+put the resulting exported data in files in the current directory.
+
+  ``rla_export -C myconfig``
+
+The output can also optionally be put in a different output directory,
+which will be created if necessary,
+using a ``-e export_directory`` argument.
+
+  ``rla_export -C myconfig -e /tmp/new-exports``
+
+If the ``-r`` (``--reports``) option is used, it will also download any reports
+that are available, and a csv-format list of ballot cards for auditing for all
+rounds defined to date.
+
+If you wish to export the results of a custom database query in json and csv format,
+put the query into ``file.sql`` and add the file name as an argument:
+
+  ``rla_export file.sql``
+
+The export from this command will have just a json and a csv file for the single SQL command in ``file.sql``.
+
+To get a full command line usage synopsis, run:
+
+  ``rla_export -h``
+
+Recommended rla_export usage:
+-----------------------------
+
+When the counties have uploaded CVR and manifest files,
+run ``rla_export`` with the ``-f`` option to capture all file uploads to date. They don't change
+after the audit is started, so it isn't necessary to use the ``-f`` option later on.
+
+For example:
+
+  ``rla_export -C myconfig.ini -f -e /srv/full-upload-archive``
+
+Run `rla_export` without the ``-f`` or ``-r`` options at any time to export just the database queries.
+
+If you are running without access to the database, you can use the ``--no-db-query`` option to turn
+those queries off, and the -r and/or -f options to export files from the application server.
+
+For example:
+
+  ``rla_export -C myconfig.ini -r --no-db-query -e $HOME/reports-export``
+
+
+Exported files
+--------------
+
+Some export files are the same as the files available via the GUI interface,
+for example ``state_report.xlsx``.
+
+Other export files are generated based on ``sql`` query files.
+These are always produced in two formats: json and csv.
+The basename of each resulting file is the same as the basename of the query file.
+Thus, given the query file ``seed.sql``, the files ``seed.json`` and ``seed.csv``
+will be produced.
+
+Refer to the supplied Export Manual for documentation on the exported data files and columns.

server/eclipse-project/script/rla_export/build_deliverable.bash

@@ -0,0 +1,19 @@
+# Build a deliverable for rla_export
+
+relname=rla_export-1.1.0
+deliverable=deliverable-$relname
+mkdir -p $deliverable
+
+# These should be the same
+diff rla_export/default.properties /srv/s/electionaudits/ColoradoRLA/server/eclipse-project/src/main/resources/us/freeandfair/corla/default.properties
+
+# First run make in the docs directory
+# and runsetup sdist in this directory
+
+cp -p dist/$relname.tar.gz $deliverable
+cp -p rla_export/corla.ini $deliverable/corla-template.ini
+cp -p ../../../../docs/export_manual.docx $deliverable
+cp -p ../../../../docs/export_manual.html $deliverable
+pandoc -t html README.rst > $deliverable/README.html
+
+zip -r $deliverable.zip $deliverable

server/eclipse-project/script/rla_export/requirements.txt

@@ -0,0 +1,2 @@
+requests>=2.12.1
+psycopg2>=2.7.3.1

server/eclipse-project/script/rla_export/rla_export/__init__.py

@@ -0,0 +1,31 @@
+"""rla_export: Export data from ColoradoRLA to allow public verification of the audit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Examples
+--------
+
+With no options, the command will run queries using
+all the standard .sql files provided in the package, and
+put the resulting exported data in files in the current directory.
+
+  ``rla_export``
+
+The optional -p argument specifies connection information via
+a database properties file, which should be the same file used
+for the ``java jar`` command line. The output can also optionally be put
+in a different output directory using the -e argument.
+
+  ``rla_export [-p properties_file] [-e export_directory]``
+
+Export a query in json and csv format for selected sql files:
+
+  ``rla_export file.sql ...``
+
+Full command line usage synopsis:
+
+  ``rla_export -h``
+
+See README.rst for documentation.
+"""
+
+from __main__ import main