First public release

Author: oskar456

.gitignore

@@ -55,3 +55,5 @@ docs/_build/
 
 # PyBuilder
 target/
+
+lddata/*.json

README.md

@@ -1,2 +1,14 @@
 # ldform
 CGI form handler used for LinuxDays web forms
+
+This is an quick and dirty hack to get the form data stored to JSON files.
+Every submission has to contain key `regid` linking to an existing directory in
+the data directory. All data from each submission is then saved into a JSON file
+named YYYYMMDD-HHMMSS-nnn.json where nnn is random nonce.
+
+Request handling can be extended by creating a custom handler with some special
+features like sending confirmation e-mail or mailing list subscription.
+
+For extracting data, there is a convinience script `export.py` which exports
+given form fields as a CSV file. Utility `losuj.py` can be used to draw a
+submission by random.

cgi-bin/ldform.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import os.path
+BASE_PATH = "/srv/www/"
+DATA_PATH = os.path.join(BASE_PATH, "lddata")
+
+import sys
+sys.path[0] = BASE_PATH
+
+import ldform
+ldform.cgi_main(DATA_PATH, traceback=True)

ldform.html

@@ -0,0 +1,6 @@
+<form action="/cgi-bin/ldform.py" method="post">
+<input type="text" name="formid" value="cfp2015">
+<input type="text" name="email">
+<input type="text" name="name">
+<input type="submit">
+</form>

ldform/__init__.py

@@ -0,0 +1 @@
+from .cgi import cgi_main

ldform/cgi.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+import cgi
+import cgitb
+
+import os
+import datetime
+import random
+import string
+import json
+import importlib
+
+# Python 3.2 compatibility
+if __package__ is None:
+    __package__ = "ldform"
+
+# Base path global variable
+BASE_PATH = ""
+
+
+class MyDict(dict):
+    """
+    Dictionary with few convinience functions and
+    a default value instead of KeyError
+    """
+    def getfirst(self, key, default=None):
+        return self.get(key, [default])[0]
+
+    def getlist(self, key, default=''):
+        return ", ".join(self.get(key, [default]))
+
+    def __missing__(self, key):
+        return [""]
+
+
+def _randstr(length=3):
+    return ''.join(random.SystemRandom().choice(string.ascii_lowercase)
+                   for i in range(length))
+
+
+def save(data):
+    global BASE_PATH
+    outfname = "{}.json".format(data['regid'])
+    with open(os.path.join(BASE_PATH, data['formid'], outfname),
+              "w", encoding="utf-8") as outf:
+        json.dump(data, outf, indent=4, sort_keys=True, ensure_ascii=False)
+        outf.write('\n')
+
+
+def cgi_main(base_path, traceback=False):
+    """
+    Serve CGI query
+    There has to be a formid value with a corresponding data directory.
+    A JSON file is produced in target directory with all user submitted
+    values plus remote IP address and a nonce, which can be used to
+    authenticate the owner of such request.
+    Finally a notify and respond functions of module named same as formid
+    is called. The intended purpuse is to send e-mail notification and
+    create a HTML response output.
+
+    @param base_path path where to store collected form JSONs
+                        (subdirectory per formid should exist)
+    @param traceback display fancy tracebacks on an exception
+    """
+    global BASE_PATH
+    BASE_PATH = base_path
+    if traceback:
+        cgitb.enable()
+
+    try:
+        form = cgi.FieldStorage()
+        formid = os.path.basename(form.getfirst('formid', 'default'))
+        outpath = os.path.join(BASE_PATH, formid)
+        if not os.path.isdir(outpath) or not os.access(outpath, os.W_OK):
+            raise RuntimeError("No valid formid found.")
+
+        data = MyDict()
+        for key in form.keys():
+            data[key] = form.getlist(key)
+
+        regid = datetime.datetime.now().strftime("%Y%m%d-%H%M%S"
+                                                 "-{}".format(_randstr(3)))
+
+        # Make sure these are added after user-supplied input
+        # so they can be trusted
+        data['formid'] = formid
+        data['regid'] = regid
+        data['nonce'] = _randstr(10)
+        data['ip'] = os.environ["REMOTE_ADDR"]
+
+        try:
+            handler = importlib.import_module('.handlers.{}'.format(formid),
+                                              __package__)
+        except ImportError:
+            handler = importlib.import_module('.handlers.generic',
+                                              __package__)
+
+        handler.respond(data)
+        save(data)
+
+    except RuntimeError as e:
+        print("Content-type: text/html; charset=UTF-8")
+        print("""
+<html>
+<body>
+<h1>Something went wrong</h1>
+<p>We were unable to process your request…</p>
+<pre>{}</pre>
+</body>
+</html>
+    """.format(e))

ldform/emailer.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import email.mime.text
+import email.utils
+import email.header
+import smtplib
+import copy
+import sys
+import re
+import argparse
+
+
+def sendemail(textpart, subject, recipients,
+              sender=("LinuxDays", "[email protected]"),
+              server="localhost"):
+    """
+    Send textual MIME e-mail
+    @param textpart Textual part of e-mail
+    @param subject Message subject
+    @param recipients List of tuples of ("User Name", "E-mail address")
+    @param sender Tuple of ("User Name", "E-mail address")
+    @param server SMTP server
+    """
+
+    msgtpl = email.mime.text.MIMEText(textpart, _charset="utf-8")
+    msgtpl['Subject'] = subject
+    msgtpl['From'] = email.header.make_header(
+        ((sender[0], None),
+         ("<{}>".format(sender[1]), None))
+    ).encode()
+    msgtpl['Precedence'] = 'bulk'
+    msgtpl['Date'] = email.utils.formatdate(localtime=True)
+
+    smtp = smtplib.SMTP(server)
+    for name, emailaddr in recipients:
+        try:
+            msg = copy.deepcopy(msgtpl)
+            msg['Message-id'] = email.utils.make_msgid('linuxdays')
+            msg['To'] = email.header.make_header(
+                ((name, None),
+                 ("<{}>".format(emailaddr), None))
+            ).encode()
+            # print(msg.as_string())
+            smtp.send_message(msg)
+        except:
+            print("Unexpected error:", sys.exc_info()[0])
+    smtp.quit()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='LinuxDays spammer')
+    parser.add_argument('--subject', "-s", help='message Subject',
+                        default='LinuxDays informace')
+    parser.add_argument('--fromname', help='sender name', default='LinuxDays')
+    parser.add_argument('--fromemail', '-f', help='sender e-mail',
+                        default='[email protected]')
+    parser.add_argument('recipients', help='file with recipient list, one'
+                        'on line, formatted Jmeno Prijmeni <adresa>',)
+    parser.add_argument('text', help='file with message body',)
+    parser.add_argument('--notest', help='actually send the messages',
+                        action='store_true')
+
+    args = parser.parse_args()
+
+    recipients = list()
+    with open(args.recipients) as recfile:
+        for line in recfile:
+            m = re.match('(.*) <([^>]*)>$', line)
+            if m:
+                recipients.append(m.groups())
+
+    print("{} recipients added...".format(len(recipients)))
+    with open(args.text) as bodyfile:
+        textpart = bodyfile.read()
+
+    if args.notest:
+        print("Sending...")
+        sendemail(textpart, args.subject, recipients,
+                  (args.fromname, args.fromemail))
+    else:
+        print(
+            "About to send text:\n{}\nfrom:{} <{}>\n"
+            "with subject: {}\nto addresses:{}".format(
+                textpart, args.fromname, args.fromemail,
+                args.subject, recipients
+            )
+        )

ldform/export.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+import os
+import json
+import argparse
+import sys
+import csv
+
+BASE_PATH = '/srv/www/lddata'
+
+
+def export_data(formid, *args):
+    """ Generate list of with values for *args. """
+    global BASE_PATH
+    path = os.path.join(BASE_PATH, formid)
+    for f in sorted(os.listdir(path)):
+        if not f.endswith('.json'):
+            continue
+        with open(os.path.join(path, f), 'r', encoding='utf-8') as inf:
+            data = json.load(inf)
+        r = []
+        for field in args:
+            d = data.get(field, [""])
+            if type(d) is list:
+                r.append(", ".join(d))
+            elif type(d) is str:
+                r.append(d)
+            else:
+                r.append("")
+        yield r
+
+
+def write_csv(outf, data, dialect='excel-tab', heading=None):
+    writer = csv.writer(outf, dialect=dialect)
+    if heading:
+        writer.writerow(heading)
+    writer.writerows(data)
+
+def main():
+    parser = argparse.ArgumentParser(description='Export form data'
+                                                 'to a CSV file')
+    parser.add_argument('-o', '--outfile',
+                        help='output to a CSV file instead of stdout')
+    parser.add_argument('-n', '--noheader', action='store_true',
+                        help='Do not output field names on the first line')
+    parser.add_argument('-d', '--dialect', choices=csv.list_dialects(),
+                        default='excel-tab', help="CSV dialect to output "
+                        "(default %(default)s)")
+    parser.add_argument('formid',
+                        help="id of form whose data should be extracted")
+    parser.add_argument('field', nargs="+",
+                        help="list of fields that should be exported")
+    args = parser.parse_args()
+
+    data = export_data(args.formid, *args.field)
+    heading = None if args.noheader else args.field
+    if args.outfile:
+        outf = open(args.outfile, 'w', newline='', encoding='utf-8')
+    else:
+        outf = sys.stdout
+    with outf:
+        write_csv(outf, data, args.dialect, heading)
+
+
+if __name__ == '__main__':
+    main()

ldform/handlers/__init__.py

@@ -0,0 +1,51 @@
+import os
+import json
+import re
+from ..cgi import BASE_PATH
+from ..cgi import MyDict
+
+confirmationtext = """Content-type: text/html; charset=UTF-8
+
+<html>
+<body>
+<h1>Nalezení registrace</h1>
+<form action="/cgi-bin/ldform.py" method="get">
+<input type="hidden" name="formid" value="findreg">
+<input type="text" name="q" placeholder="Hledané jméno" value="{q}">
+<input type="submit">
+</form>
+<table>
+<tr><th>#</th><th>Jméno</th><th>regid</th></tr>
+{table}
+</table>
+</body>
+</html>
+"""
+
+
+
+def respond(data):
+    """Find registration data and delete them."""
+    global BASE_PATH
+    query = data.getfirst('q', '')
+    path = os.path.join(BASE_PATH, 'reg2015')
+    order = 0
+    results = []
+    if len(query) > 2:
+        for f in sorted(os.listdir(path)):
+            if not f.endswith('.json'):
+                continue
+            with open(os.path.join(path, f), 'r', encoding='utf-8') as inf:
+                data = MyDict(json.load(inf))
+            order += 1
+            name = data.getfirst('name')
+            if not name:
+                continue
+            regid = data.get('regid')
+            if re.search(query, name, re.I):
+                results.append((order, name, regid))
+
+    rows = ("<tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(o, n, r) 
+             for o, n, r in results)
+    table = "\n".join(rows)
+    print(confirmationtext.format(q=query, table=table))