Python3 and Django1.8 support - part 1

Vagrantfile

@@ -0,0 +1,148 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Enable vagrant-env if installed
+  if Vagrant.has_plugin?('vagrant-env')
+    config.env.enable
+  end
+
+  # Every Vagrant virtual environment requires a box to build off of.
+  config.vm.box = "bento/ubuntu-20.04"
+  config.vm.network "private_network", ip: "192.168.33.10"
+
+  config.vm.provision :shell, path: "bootstrap.sh"
+  config.vm.network :forwarded_port, host: 8081, guest: 8000
+
+  # celery flower monitoring running on guest port 5555 => 5556
+  # # by running ngrok http 5556, we can tunnel the output
+  #config.vm.network :forwarded_port, host: 5556, guest: 5555
+
+  # The url from where the 'config.vm.box' box will be fetched if it
+  # doesn't already exist on the user's system.
+  # config.vm.box_url = "http://domain.com/path/to/above.box"
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network :forwarded_port, guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network :private_network, ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network :public_network
+
+  # If true, then any SSH connections made will enable agent forwarding.
+  # Default value: false
+  # config.ssh.forward_agent = true
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+
+  if ENV['SMITH_VAGRANT_NFS']
+    # Use NFS server for Mac
+    config.vm.synced_folder ".", "/vagrant", mount_options: ["nolock"], type: "nfs"
+  end
+
+  config.vm.provider :virtualbox do |vb|
+    # Use VBoxManage to customize the VM. For example to change memory:
+    if ENV['SMITH_VM_MEMORY']
+      memory = ENV['SMITH_VM_MEMORY']
+    else
+      memory = "2048"  # default memory
+    end
+    vb.customize ["modifyvm", :id, "--memory", memory]
+
+    if ENV['SMITH_VAGRANT_HTTPS_OPTIMIZATIONS']
+      # Nat fixes for http
+      vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+      vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+      # Network
+      vb.customize ["modifyvm", :id, "--nictype1", "virtio"]
+    end
+
+  end
+  #
+  # View the documentation for the provider you're using for more
+  # information on available options.
+
+  # Enable provisioning with Puppet stand alone.  Puppet manifests
+  # are contained in a directory path relative to this Vagrantfile.
+  # You will need to create the manifests directory and a manifest in
+  # the file base.pp in the manifests_path directory.
+  #
+  # An example Puppet manifest to provision the message of the day:
+  #
+  # # group { "puppet":
+  # #   ensure => "present",
+  # # }
+  # #
+  # # File { owner => 0, group => 0, mode => 0644 }
+  # #
+  # # file { '/etc/motd':
+  # #   content => "Welcome to your Vagrant-built virtual machine!
+  # #               Managed by Puppet.\n"
+  # # }
+  #
+  # config.vm.provision :puppet do |puppet|
+  #   puppet.manifests_path = "manifests"
+  #   puppet.manifest_file  = "site.pp"
+  # end
+
+  # Enable provisioning with chef solo, specifying a cookbooks path, roles
+  # path, and data_bags path (all relative to this Vagrantfile), and adding
+  # some recipes and/or roles.
+  #
+  # config.vm.provision :chef_solo do |chef|
+  #   chef.cookbooks_path = "../my-recipes/cookbooks"
+  #   chef.roles_path = "../my-recipes/roles"
+  #   chef.data_bags_path = "../my-recipes/data_bags"
+  #   chef.add_recipe "mysql"
+  #   chef.add_role "web"
+  #
+  #   # You may also specify custom JSON attributes:
+  #   chef.json = { :mysql_password => "foo" }
+  # end
+
+  # Enable provisioning with chef server, specifying the chef server URL,
+  # and the path to the validation key (relative to this Vagrantfile).
+  #
+  # The Opscode Platform uses HTTPS. Substitute your organization for
+  # ORGNAME in the URL and validation key.
+  #
+  # If you have your own Chef Server, use the appropriate URL, which may be
+  # HTTP instead of HTTPS depending on your configuration. Also change the
+  # validation key to validation.pem.
+  #
+  # config.vm.provision :chef_client do |chef|
+  #   chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME"
+  #   chef.validation_key_path = "ORGNAME-validator.pem"
+  # end
+  #
+  # If you're using the Opscode platform, your validator client is
+  # ORGNAME-validator, replacing ORGNAME with your organization name.
+  #
+  # If you have your own Chef Server, the default validation client name is
+  # chef-validator, unless you changed the configuration.
+  #
+  #   chef.validation_client_name = "ORGNAME-validator"
+end

bootstrap.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# This script run under sudo by default so no need to
+# put sudo statements here.
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+
+# Avoid annoying Grub prompt
+# See https://askubuntu.com/questions/146921/how-do-i-apt-get-y-dist-upgrade-without-a-grub-config-prompt
+apt-get -y -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" upgrade
+
+apt-get -y install libpq-dev libxml2-dev libxslt-dev
+apt-get -y install libffi-dev libssl-dev
+apt-get -y install libjpeg-dev
+apt-get -y install gettext
+apt-get -y install memcached
+apt-get -y install python3
+apt-get -y install libmysqlclient-dev
+apt-get -y install python3-pip
+
+# Postgresql 10
+echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+apt-get update
+apt-get -y install postgresql-10
+
+echo "local all postgres peer" > /etc/postgresql/10/main/pg_hba.conf
+echo "local met met md5" >> /etc/postgresql/10/main/pg_hba.conf
+echo "local test_met met md5" >> /etc/postgresql/10/main/pg_hba.conf
+echo "local all all peer" >> /etc/postgresql/10/main/pg_hba.conf
+echo "host all all 127.0.0.1/32 md5" >> /etc/postgresql/10/main/pg_hba.conf
+echo "host all all ::1/128 md5" >> /etc/postgresql/10/main/pg_hba.conf
+
+service postgresql restart
+
+sudo -u postgres bash -c "psql -c \"CREATE USER met WITH SUPERUSER PASSWORD 'met';\""
+sudo -u postgres createdb --owner=met --encoding=UTF8 met
+
+su vagrant << EOF
+cd /vagrant
+EOF
+
+echo "cd /vagrant/" >> /home/vagrant/.profile
+echo "export LC_ALL='en_US.UTF-8'" >> /home/vagrant/.profile
+echo "export LC_CTYPE='en_US.UTF-8'" >> /home/vagrant/.profile

doc/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Saml2 Metadata Explorer Tool documentation build configuration file, created by
 # sphinx-quickstart on Thu Oct  4 17:54:30 2012.
@@ -40,8 +39,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'Saml2 Metadata Explorer Tool'
-copyright = u'2012, Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL'
+project = 'Saml2 Metadata Explorer Tool'
+copyright = '2012, Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -183,8 +182,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'Saml2MetadataExplorerTool.tex', u'Saml2 Metadata Explorer Tool Documentation',
-   u'Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL', 'manual'),
+  ('index', 'Saml2MetadataExplorerTool.tex', 'Saml2 Metadata Explorer Tool Documentation',
+   'Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -213,8 +212,8 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'saml2metadataexplorertool', u'Saml2 Metadata Explorer Tool Documentation',
-     [u'Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL'], 1)
+    ('index', 'saml2metadataexplorertool', 'Saml2 Metadata Explorer Tool Documentation',
+     ['Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -227,8 +226,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Saml2MetadataExplorerTool', u'Saml2 Metadata Explorer Tool Documentation',
-   u'Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL', 'Saml2MetadataExplorerTool', 'One line description of project.',
+  ('index', 'Saml2MetadataExplorerTool', 'Saml2 Metadata Explorer Tool Documentation',
+   'Antonio Pérez-Aranda Alcaide at Yaco Sistemas SL', 'Saml2MetadataExplorerTool', 'One line description of project.',
    'Miscellaneous'),
 ]
 

doc/source/install.rst

@@ -3,30 +3,32 @@
 Metadata Explorer Tool Install
 ==============================
 
+Install Python 3.7
+******************
+
+
+.. code-block:: bash
+
+  # apt-get update
+  # apt-get install python3
+  # apt-get install python3-pip
 
 Install requirements
 ********************
 
-System packages (ubuntu-1204)
-
-* python-setuptools
-* python-dev
-* python-virtualenv
-* python-imaging
-* libjpeg-dev
-* libpng-dev
-* postgresql
-* libapache2-mod-wsgi
-* build-essential
-* libxml2-dev
-* libxslt-dev
-* libpq-dev
-* xmlsec1
-* memcached
-* libffi-dev
-* django_chartit
-* dateutils
+.. code-block:: bash
+
+  # pip3 install -r requirements.txt
 
+Make sure locales are correctly installed
+*****************************************
+
+.. code-block:: bash
+
+  echo "LC_ALL=en_US.UTF-8" >> /etc/environment
+  echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
+  echo "LANG=en_US.UTF-8" > /etc/locale.conf
+  locale-gen en_US.UTF-8
 
 Create database
 ***************
@@ -72,34 +74,28 @@ Project deployment
 
   .. code-block:: bash
 
-      git clone git://github.com/Yaco-Sistemas/met.git
+      git clone git://github.com/GEANT/met.git
 
 * Deploy met egg:
 
   .. code-block:: bash
 
       cd met
-      python setup.py develop
+      python3.7 setup.py develop
 
 * Configure ``local_settings`` and initialize met database (create models):
 
   .. code-block:: bash
 
-      cp local_settings.example.py local_settings.py
-      python manage.py syncdb
+      cp local_settings.py.example local_settings.py
+      python3.7 manage.py migrate
 
 * To initialize static files for admin page of Django execute:
 
   .. code-block:: bash
 
       python manage.py collectstatic
 
-* Patch the pyFF modyle with files in pyff-patch
-
-  .. code-block:: bash
-
-      cp pyff-patch/* met-venv/lib/python2.7/site-packages/pyff/
-
 
 Apache configuration
 ********************
@@ -129,7 +125,7 @@ http://httpd.apache.org/docs/2.2/mod/core.html#allowencodedslashes
 
     WSGIDaemonProcess <server name> home=/home/met
     WSGIProcessGroup <server name>
-    
+
     WSGIScriptAlias / /home/met/met/met-wsgi.py
 
     <Directory /home/met/met/met-wsgi.py>
@@ -187,7 +183,7 @@ Automatic refresh of federations' metadata
 ******************************************
 
 Metadata of configured federations can be refreshed automatically. To achieve this
-you just need to configure a cronjob on your server such as: 
+you just need to configure a cronjob on your server such as:
 
 .. code-block:: bash
 

local_settings.py.example

@@ -13,25 +13,10 @@
 import os
 import saml2
 
-HOSTNAME = 'https://met-hostname.example.com'
-BASEURL = '/'
+HOSTNAME = os.environ.get('HOSTNAME')
+BASEURL = os.environ.get('BASEURL')
 BASEDIR = os.path.abspath(os.path.dirname(__file__))
 
-DEBUG = False
-PROFILE = DEBUG
-TEMPLATE_DEBUG = DEBUG
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-        'NAME': 'met',                      # Or path to database file if using sqlite3.
-        'USER': 'met',                      # Not used with sqlite3.
-        'PASSWORD': 'met',                  # Not used with sqlite3.
-        'HOST': 'localhost',                      # Set to empty string for localhost. Not used with sqlite3.
-        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
-    }
-}
-
 MYSQLPOOL_BACKEND = 'QueuePool'
 
 MYSQLPOOL_ARGUMENTS = {

manage.py

@@ -15,7 +15,10 @@
 import os
 import sys
 
+import dotenv
+
 if __name__ == "__main__":
+    dotenv.read_dotenv()
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "met.settings")
 
     from django.core.management import execute_from_command_line