[Python3 and Django2.2 upgrade] part 1 - Remove unused code and make tests pass again

.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*.{es6,js,coffee,scss,css}]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{html}]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true

.gitignore

@@ -57,3 +57,5 @@ custom_template
 .vagrant
 Vagrantfile
 ansible
+
+/src

README.md

@@ -18,9 +18,18 @@ please refer to
 As features are migrated, we will list them here along with the last
 release where each was present:
 
-  - REST Services - an improved version [has been added to
-    KPI](https://github.com/kobotoolbox/kpi/pull/1864). Last KoBoCAT
+  - To ensure security and stability, many endpoints that were already
+    available in KPI, long-unsupported, or underutilized have been removed in
     release
+    [2.020.40](https://github.com/kobotoolbox/kobocat/releases/tag/2.020.40).
+    These were related to charts and stats, form cloning, form sharing, user
+    profiles, organizations / projects / teams, bamboo, and ziggy. For a full
+    list, please see [REMOVALS.md](REMOVALS.md). These endpoints were last
+    available in the release
+    [2.020.39](https://github.com/kobotoolbox/kobocat/releases/tag/2.020.39).
+  - REST Services - an improved version [has been added to
+    KPI](https://github.com/kobotoolbox/kpi/pull/1864). The last KoBoCAT
+    release to contain legacy REST services is
     [2.019.39](https://github.com/kobotoolbox/kobocat/releases/tag/2.019.39).
 
 ## About

REMOVALS.md

@@ -0,0 +1,106 @@
+# KoBoCAT endpoint removals as of release [2.020.40](https://github.com/kobotoolbox/kobocat/releases/tag/2.020.40)
+
+The last release to contain any of the endpoints listed below was https://github.com/kobotoolbox/kobocat/releases/tag/2.020.39.
+
+## Already available in KPI
+
+### Charts and Stats
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/<username>/forms/<id_string>/stats` | `onadata.apps.viewer.views.charts` | `form-stats`
+`/<username>/forms/<id_string>/tables` | `onadata.apps.viewer.views.stats_tables` |  
+`/api/v1/charts` | `onadata.apps.api.viewsets.charts_viewset.ChartsViewSet` | `chart-list`
+`/api/v1/charts.<format>/` | `onadata.apps.api.viewsets.charts_viewset.ChartsViewSet` | `chart-list`
+`/api/v1/charts/<pk>` | `onadata.apps.api.viewsets.charts_viewset.ChartsViewSet` | `chart-detail`
+`/api/v1/charts/<pk>.<format>/` | `onadata.apps.api.viewsets.charts_viewset.ChartsViewSet` | `chart-detail`
+`/api/v1/stats` | `onadata.apps.api.viewsets.stats_viewset.StatsViewSet` | `stats-list`
+`/api/v1/stats.<format>/` | `onadata.apps.api.viewsets.stats_viewset.StatsViewSet` | `stats-list`
+`/api/v1/stats/<pk>` | `onadata.apps.api.viewsets.stats_viewset.StatsViewSet` | `stats-detail`
+`/api/v1/stats/<pk>.<format>/` | `onadata.apps.api.viewsets.stats_viewset.StatsViewSet` | `stats-detail`
+`/api/v1/stats/submissions` | `onadata.apps.api.viewsets.submissionstats_viewset.SubmissionStatsViewSet` | `submissionstats-list`
+`/api/v1/stats/submissions.<format>/` | `onadata.apps.api.viewsets.submissionstats_viewset.SubmissionStatsViewSet` | `submissionstats-list`
+`/api/v1/stats/submissions/<pk>` | `onadata.apps.api.viewsets.submissionstats_viewset.SubmissionStatsViewSet` | `submissionstats-detail`
+`/api/v1/stats/submissions/<pk>.<format>/` | `onadata.apps.api.viewsets.submissionstats_viewset.SubmissionStatsViewSet` | `submissionstats-detail`
+`/stats/` | `onadata.apps.stats.views.stats` | `form-stats`
+`/stats/submissions/` | `onadata.apps.stats.views.submissions` |
+
+### Form Cloning
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/api/v1/forms/<pk>/clone` | `onadata.apps.api.viewsets.xform_viewset.XFormViewSet` | `xform-clone`
+`/api/v1/forms/<pk>/clone.<format>/` | `onadata.apps.api.viewsets.xform_viewset.XFormViewSet` | `xform-clone`
+
+### Form Sharing
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/api/v1/forms/<pk>/share` | `onadata.apps.api.viewsets.xform_viewset.XFormViewSet` | `xform-share`
+`/api/v1/forms/<pk>/share.<format>/` | `onadata.apps.api.viewsets.xform_viewset.XFormViewSet` | `xform-share`
+
+### User Profiles
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/api/v1/profiles` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-list`
+`/api/v1/profiles.<format>/` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-list`
+`/api/v1/profiles/<user>` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-detail`
+`/api/v1/profiles/<user>.<format>/` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-detail`
+`/api/v1/profiles/<user>/change_password` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-change-password`
+`/api/v1/profiles/<user>/change_password.<format>/` | `onadata.apps.api.viewsets.user_profile_viewset.UserProfileViewSet` | `userprofile-change-password`
+`/api/v1/user/reset` | `onadata.apps.api.viewsets.connect_viewset.ConnectViewSet` | `userprofile-reset`
+`/api/v1/user/reset.<format>/` | `onadata.apps.api.viewsets.connect_viewset.ConnectViewSet` | `userprofile-reset`
+`/api/v1/users` | `onadata.apps.api.viewsets.user_viewset.UserViewSet` | `user-list`
+`/api/v1/users.<format>/` | `onadata.apps.api.viewsets.user_viewset.UserViewSet` | `user-list`
+`/api/v1/users/<username>` | `onadata.apps.api.viewsets.user_viewset.UserViewSet` | `user-detail`
+`/api/v1/users/<username>.<format>/` | `onadata.apps.api.viewsets.user_viewset.UserViewSet` | `user-detail`
+
+## Discontinued
+
+These endpoints existed but were neither maintained nor tested, and their features were never available in the UI. They might be added to KPI at an indeterminate future time given interest and funding.
+
+### Organizations, Projects, and Teams
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/api/v1/orgs` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-list`
+`/api/v1/orgs.<format>/` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-list`
+`/api/v1/orgs/<user>` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-detail`
+`/api/v1/orgs/<user>.<format>/` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-detail`
+`/api/v1/orgs/<user>/members` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-members`
+`/api/v1/orgs/<user>/members.<format>/` | `onadata.apps.api.viewsets.organization_profile_viewset.OrganizationProfileViewSet` | `organizationprofile-members`
+`/api/v1/projects` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-list`
+`/api/v1/projects.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-list`
+`/api/v1/projects/<pk>` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-detail`
+`/api/v1/projects/<pk>.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-detail`
+`/api/v1/projects/<pk>/forms` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-forms`
+`/api/v1/projects/<pk>/forms.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-forms`
+`/api/v1/projects/<pk>/labels` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-labels`
+`/api/v1/projects/<pk>/labels.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-labels`
+`/api/v1/projects/<pk>/share` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-share`
+`/api/v1/projects/<pk>/share.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-share`
+`/api/v1/projects/<pk>/star` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-star`
+`/api/v1/projects/<pk>/star.<format>/` | `onadata.apps.api.viewsets.project_viewset.ProjectViewSet` | `project-star`
+`/api/v1/teams` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-list`
+`/api/v1/teams.<format>/` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-list`
+`/api/v1/teams/<pk>` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-detail`
+`/api/v1/teams/<pk>.<format>/` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-detail`
+`/api/v1/teams/<pk>/members` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-members`
+`/api/v1/teams/<pk>/members.<format>/` | `onadata.apps.api.viewsets.team_viewset.TeamViewSet` | `team-members`
+
+### User Profiles
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/api/v1/user/<user>/starred` | `onadata.apps.api.viewsets.connect_viewset.ConnectViewSet` | `userprofile-starred`
+`/api/v1/user/<user>/starred.<format>/` | `onadata.apps.api.viewsets.connect_viewset.ConnectViewSet` | `userprofile-starred`
+
+## Discontinued permanently
+
+### Bamboo and Ziggy
+
+URL Pattern | View Class or Function | View Name
+-- | -- | --
+`/<username>/forms/<id_string>/bamboo` | `onadata.apps.main.views.link_to_bamboo` |
+`/<username>/form-submissions` | `onadata.apps.logger.views.ziggy_submissions` |

conftest.py

@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals, print_function, division, absolute_import
+import os
+import pytest
+import sys
+
+from django.conf import settings
+
+from onadata.libs.utils.storage import delete_user_storage, user_storage_exists
+
+TEST_USERNAMES = [
+    'alice',
+    'auser',
+    'bob',
+    'carl',
+    'deno'
+    'harry',
+    'jo',
+    'lilly',
+    'peter',
+]
+
+try:
+    INTERRUPTED = pytest.ExitCode.INTERRUPTED  # pytest 5
+except AttributeError:
+    INTERRUPTED = 2
+
+
+def stderr_prompt(message):
+    sys.stderr.write(message)
+    # FIXME: Python 3 compatibility
+    return raw_input().strip()
+
+
+def toggle_capturing(capture_manager, stop):
+    if stop:
+        capture_manager.suspend_global_capture()
+        capture_manager.stop_global_capturing()
+    else:
+        capture_manager.start_global_capturing()
+        capture_manager.resume_global_capture()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup(request):
+    # We need to disable global capturing in case `-s` is not passed to `pytest`
+    # by the users to force print the safeguard messages about data loss.
+    capture_manager = request.config.pluginmanager.getplugin("capturemanager")
+    is_global_capturing = capture_manager.is_globally_capturing()
+
+    if is_global_capturing:
+        toggle_capturing(capture_manager, stop=True)
+
+    for username in TEST_USERNAMES:
+        if user_storage_exists(username):
+            response = stderr_prompt(
+                '\n\n'
+                'WARNING - DATA LOSS! A storage directory already exists for '
+                'user {}, but it will be DELETED if you continue with these '
+                'tests!\nPlease type "yes" to proceed anyway, or "no" to '
+                'cancel: '.format(username)
+            )
+            if response.lower() != 'yes':
+                if is_global_capturing:
+                    toggle_capturing(capture_manager, stop=False)
+                pytest.exit('User interrupted tests', INTERRUPTED)
+
+    if 'instances' in settings.MONGO_DB.collection_names():
+        response = stderr_prompt(
+            '\n\n'
+            'WARNING: the MongoDB collection {}.instances already exists!\n'
+            "Type 'yes' if you would like to delete it, or 'no' to "
+            'cancel: '.format(settings.MONGO_DB.name)
+        )
+        if response.lower() == 'yes':
+            settings.MONGO_DB.instances.drop()
+        else:
+            if is_global_capturing:
+                toggle_capturing(capture_manager, stop=False)
+            pytest.exit('User interrupted tests', INTERRUPTED)
+
+    if is_global_capturing:
+        toggle_capturing(capture_manager, stop=False)
+
+    request.addfinalizer(_tear_down)
+
+
+def _tear_down():
+    print("\nCleaning testing environment...")
+    print('Removing MongoDB...')
+    settings.MONGO_DB.instances.drop()
+
+    root_path = os.path.dirname(os.path.realpath(__file__))
+    if os.path.exists(os.path.join(root_path, 'db.sqlite3')):
+        print('Removing SQL DB...')
+        os.remove(os.path.join(root_path, 'db.sqlite3'))
+
+    print("Removing users' storage...")
+    for username in TEST_USERNAMES:
+        delete_user_storage(username)

kobocat-template/templates/base.html

@@ -72,6 +72,8 @@
 		{% include "topbar.html"%}
 
   {% endif %}
+
+    {% block before-content %}{% endblock %}
 
     <div class="container-fluid container__wide main {{ request.resolver_match.url_name }}">
       {% block message %}

kobocat-template/templates/data_view.html

@@ -8,7 +8,9 @@
 <link rel="stylesheet" href="{% static "bower_components/backgrid-filter/backgrid-filter.min.css" %}" />
 <link rel="stylesheet" href="{% static "bower_components/backgrid-paginator/backgrid-paginator.min.css" %}" />
 <link href="{% static "css/data_view.css" %}" rel="stylesheet">
+{% endblock %}
 
+{% block before-content %}
 <div class="sub-header-bar">
   <div class="container__wide">
     <a class="sub-header__back" href="{% url "onadata.apps.main.views.show" xform.user.username xform.id_string %}"><i class="fa fa-chevron-left"></i> {% trans "Return to" %} {{ xform.title }}</a>

kobocat-template/templates/export_list.html

@@ -2,7 +2,7 @@
 {% load static %}
 
 
-{% block additional-headers %}
+{% block before-content %}
 {% load i18n %}
 
 <div class="sub-header-bar">
@@ -46,16 +46,6 @@ <h1>{{ export_type_name|upper }} {% blocktrans %}Exports{% endblocktrans %}</h1>
           <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
           <h3>{% trans "Advanced Export" %}</h3>
         </div>
-        {% if export_type == 'external' %}
-            <div class="modal-body">
-              <label>{% trans "Template to use for the export" %}</label>
-              <select id="external_export_metadata" name="meta" class="span2">
-                {% for meta in metas %}
-                <option value="{{ meta.id }}">{% trans meta.data_value %}</option>
-                {% endfor %}
-              </select>
-        </div>
-        {% else %}
         <div class="modal-body">
           <label>{% trans "Delimiter to use to separate group names from field names" %}</label>
           <select id="options-group_delimiter" name="options[group_delimiter]" class="span2">
@@ -67,7 +57,6 @@ <h3>{% trans "Advanced Export" %}</h3>
             {% trans "DONT split select multiple choice answers into separate columns" %}
           </label>
         </div>
-        {% endif %}
         <div class="modal-footer">
           <button class="btn" data-dismiss="modal" aria-hidden="true">{% trans "Cancel" %}</button>
           <input type="submit" class="btn large btn-primary" value="{% trans "Create Export" %}" />

kobocat-template/templates/instance.html

@@ -2,7 +2,7 @@
 {% load i18n %}
 {% load static %}
 
-{% block additional-headers %}
+{% block before-content %}
 {% load i18n %}
 
 <div class="sub-header-bar">