make user-defined network from db

implements custom filtering for user service

customized auth to work with phoenix

fixes bug in non-incognito browsers

creates user in h at moment of auth if it does not exist

canonical determination of authority

Create .gitlab-ci.yml

Add proxy prefix

Add hypothesis base url to assets

fetches user based on default authority, not hard-coded authority

hack to authenticate user a second time when building session

replaces request.authenticated_userid

client expects userid, not username

resolves root session issue in security policy, removes hack solution in session model

dynamically manages tosdr url via store for dev env (to-do : staging, prod)

adds tosdr base domain to client settings

removes print statement

fixes error that is interpreting every h request as api request

Update .gitlab-ci.yml

exposes es with a network

queries annotations in es based on strict uri

determines asset path based on env

forces username compliance for tosdr users created in h

Build stable image

relies on phoenix docker-compose for db, es

.gitlab-ci.yml

@@ -0,0 +1,36 @@
+# You can override the included template(s) by including variable overrides
+# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
+# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
+# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
+# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
+# Note that environment variables can be set in several places
+# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+
+stages:
+- publish
+
+
+docker-nightly:
+  stage: publish
+  image:
+    name: gcr.io/kaniko-project/executor:v1.9.0-debug
+    entrypoint:
+    - ''
+  script:
+  - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile"
+    --destination "${CI_REGISTRY_IMAGE}:nightly"
+  rules:
+  - if: $CI_COMMIT_BRANCH == "phoenix-integration"
+
+
+docker-prod:
+  stage: publish
+  image:
+    name: gcr.io/kaniko-project/executor:v1.9.0-debug
+    entrypoint:
+    - ''
+  script:
+  - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile"
+    --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" --destination "${CI_REGISTRY_IMAGE}:stable"
+  rules:
+  - if: $CI_COMMIT_TAG

conf/app.ini

@@ -8,6 +8,8 @@ use: call:h.app:create_app
 
 [filter:proxy-prefix]
 use: egg:PasteDeploy#prefix
+prefix = /hypothesis
+
 
 [server:main]
 use: egg:gunicorn#main

conf/development-app.ini

@@ -14,6 +14,7 @@ h.bouncer_url: http://localhost:8000
 h.client_rpc_allowed_origins: http://localhost:8001 https://localhost:48001
 h.client_url: {current_scheme}://{current_host}:3001/hypothesis
 h.websocket_url: ws://localhost:5001/ws
+h.tosdr: http://localhost:9090
 
 h.debug: True
 h.reload_assets: True

docker-compose.yml

@@ -1,20 +1,5 @@
 version: '3'
 services:
-  postgres:
-    image: postgres:11.5-alpine
-    ports:
-      - '127.0.0.1:5432:5432'
-    healthcheck:
-        test: ["CMD", "pg_isready", "-U", "postgres"]
-        interval: 1s
-    networks:
-      - dbs
-  elasticsearch:
-    image: hypothesis/elasticsearch:latest
-    ports:
-      - '127.0.0.1:9200:9200'
-    environment:
-      - discovery.type=single-node
   rabbit:
     image: rabbitmq:3.6-management-alpine
     ports:
@@ -26,4 +11,9 @@ networks:
   # To avoid having unnecessary dependencies between the projects
   # the network is created with `docker network crate dbs` in each project's Makefile (make services)
   dbs:
+    name: dbs
+    external: true
+  elasticsearch:
+    name: elasticsearch
     external: true
+

h/assets.py

@@ -1,16 +1,22 @@
 """View for serving static assets under `/assets`."""
 
+import os
 import importlib_resources
 from h_assets import Environment, assets_view
 from pyramid.settings import asbool
 
 
+
 def includeme(config):
     auto_reload = asbool(config.registry.settings.get("h.reload_assets", False))
     h_files = importlib_resources.files("h")
 
+    asset_path = "/hypothesis/assets"
+    if "ASSET_PATH" in os.environ:
+        asset_path = os.environ["ASSET_PATH"]
+
     assets_env = Environment(
-        assets_base_url="/assets",
+        assets_base_url=asset_path,
         bundle_config_path=h_files / "assets.ini",
         manifest_path=h_files / "../build/manifest.json",
         auto_reload=auto_reload,

h/config.py

@@ -28,6 +28,7 @@ def configure(environ=None, settings=None):  # pylint: disable=too-many-statemen
         environ = os.environ
     if settings is None:
         settings = {}
+
     settings_manager = SettingsManager(settings, environ)
     # Configuration for external components
     settings_manager.set("broker_url", "BROKER_URL")
@@ -87,6 +88,9 @@ def configure(environ=None, settings=None):  # pylint: disable=too-many-statemen
     # secret.
     settings_manager.set("h.client_oauth_id", "CLIENT_OAUTH_ID")
 
+    # Base domain for tosdr
+    settings_manager.set("h.tosdr", "TOSDR_URL")
+
     # The list of origins that the client will respond to cross-origin RPC
     # requests from.
     settings_manager.set(

h/search/query.py

@@ -272,14 +272,16 @@ def __call__(self, search, params):
             [u for u in wildcard_uris if wildcard_uri_is_valid(u)],
             normalize_method=self._wildcard_uri_normalized,
         )
+        strict_uri = uris[0]
         uris = self._normalize_uris(uris)
 
         queries = []
         if wildcard_uris:
             queries = [Q("wildcard", **{"target.scope": u}) for u in wildcard_uris]
         if uris:
             queries.append(Q("terms", **{"target.scope": uris}))
-        return search.query("bool", should=queries)
+        # TOSDR
+        return search.query(Q("bool", must=[Q("match", uri=strict_uri)]))
 
     def _normalize_uris(self, query_uris, normalize_method=uri.normalize):
         uris = set()

h/security/policy/combined.py

@@ -74,6 +74,7 @@ def _call_sub_policies(self, method, request, *args, **kwargs):
             return getattr(self._ui_policy, method)(request, *args, **kwargs)
 
         # Then we try the bearer header (or `access_token` GET param)
+
         result = getattr(self._bearer_token_policy, method)(request, *args, **kwargs)
 
         if not result and self._http_basic_auth_policy.handles(request):
@@ -87,7 +88,9 @@ def _call_sub_policies(self, method, request, *args, **kwargs):
 
     @staticmethod
     def _is_api_request(request):
-        return request.path.startswith("/api") and request.path not in [
+        return (request.path.startswith("/api") or request.path.startswith("/hypothesis/api")) and request.path not in [
             "/api/token",
             "/api/badge",
+            "/hypothesis/api/token",
+            "/hypothesis/api/badge"
         ]

h/services/user.py

@@ -1,4 +1,5 @@
 import sqlalchemy as sa
+from sqlalchemy.sql import text
 
 from h.models import User, UserIdentity
 from h.util.db import on_transaction_end
@@ -50,6 +51,7 @@ def fetch(self, userid_or_username, authority=None):
         :rtype: h.models.User or None
 
         """
+
         if authority is not None:
             username = userid_or_username
         else:
@@ -65,12 +67,19 @@ def fetch(self, userid_or_username, authority=None):
             self._cache[cache_key] = (
                 self.session.query(User)
                 .filter_by(username=username)
-                .filter_by(authority=authority)
+                # comment for tosdr
+                # .filter_by(authority=authority)
                 .one_or_none()
             )
 
         return self._cache[cache_key]
 
+    def fetch_from_tosdr(self, h_key):
+        statement = text("SELECT * FROM users WHERE h_key =:x")
+        statement = statement.bindparams(x=h_key)
+        user_tosdr = self.session.execute(statement).one_or_none()
+        return user_tosdr
+
     def fetch_all(self, userids):
         """
         Fetch a list of users by their userids.

h/session.py

@@ -29,10 +29,15 @@ def profile(request, authority=None):
         authority = user.authority
     else:
         authority = authority or request.default_authority
+
+    userid = request.authenticated_userid
+
+    if not userid:
+        userid = user.userid
 
     return dict(
         {
-            "userid": request.authenticated_userid,
+            "userid": userid,
             "authority": authority,
             "groups": _current_groups(request, authority),
             "features": request.feature.all(),

h/views/api/auth.py

@@ -1,5 +1,9 @@
 import json
 import logging
+import random
+import re
+import string
+from datetime import datetime
 from functools import wraps
 from urllib.parse import parse_qs, urlparse
 
@@ -8,7 +12,9 @@
 from pyramid.view import view_config, view_defaults
 
 from h import models
+from h.models import User
 from h.services.oauth import DEFAULT_SCOPES
+
 from h.util.datetime import utc_iso8601
 from h.views.api.config import api_config
 from h.views.api.exceptions import OAuthAuthorizeError, OAuthTokenError
@@ -33,18 +39,20 @@ def inner(*args, **kwargs):
 
     return inner
 
-
 @view_defaults(route_name="oauth_authorize")
 class OAuthAuthorizeController:
     def __init__(self, context, request):
         self.context = context
         self.request = request
-
+        self.session = request.db
         self.user_svc = self.request.find_service(name="user")
         self.oauth = self.request.find_service(name="oauth_provider")
 
     @view_config(
-        request_method="GET", renderer="h:templates/oauth/authorize.html.jinja2"
+        request_method="GET",
+        # for tosdr 
+        # renderer=None
+        renderer="h:templates/oauth/authorize.html.jinja2"
     )
     def get(self):
         """
@@ -87,7 +95,7 @@ def get_web_message(self):
 
     @view_config(
         request_method="POST",
-        is_authenticated=True,
+        # is_authenticated=True,
         renderer="json",
     )
     def post(self):
@@ -102,8 +110,8 @@ def post(self):
     @view_config(
         request_method="POST",
         request_param="response_mode=web_message",
-        is_authenticated=True,
-        renderer="h:templates/oauth/authorize_web_message.html.jinja2",
+        # is_authenticated=True,
+        renderer="json",
     )
     def post_web_message(self):
         """
@@ -115,8 +123,7 @@ def post_web_message(self):
 
         .. _draft-sakimura-oauth: https://tools.ietf.org/html/draft-sakimura-oauth-wmrm-00
         """
-        found = self._authorized_response()
-        return self._render_web_message_response(found.location)
+        return self._authorized_response()
 
     def _authorize(self):
         try:
@@ -126,12 +133,6 @@ def _authorize(self):
                 err.description or f"Error: {self.context.error}"
             ) from err
 
-        if self.request.authenticated_userid is None:
-            raise HTTPFound(
-                self.request.route_url(
-                    "login", _query={"next": self.request.url, "for_oauth": True}
-                )
-            )
 
         client_id = credentials.get("client_id")
         client = self.request.db.query(models.AuthClient).get(client_id)
@@ -142,7 +143,7 @@ def _authorize(self):
         # logged-in user.
         if client.trusted:
             return self._authorized_response()
-
+        
         state = credentials.get("state")
         user = self.user_svc.fetch(self.request.authenticated_userid)
         response_mode = credentials.get("request").response_mode
@@ -156,20 +157,31 @@ def _authorize(self):
             "state": state,
         }
 
-    @handles_oauth_errors
+    # @handles_oauth_errors
     def _authorized_response(self):
         # We don't support scopes at the moment, but oauthlib does need a scope,
         # so we're explicitly overwriting whatever the client provides.
         scopes = DEFAULT_SCOPES
-        user = self.user_svc.fetch(self.request.authenticated_userid)
+        # TOSDR : find tosdr user based on h_key cookie
+        h_key = self.request.cookies.get('h_key')
+        user_tosdr = self.user_svc.fetch_from_tosdr(h_key)     
+        username = user_tosdr.username
+        user = self.user_svc.fetch(username, authority=self.request.default_authority)
+        # TOSDR : create user in h if it does not exist
+        if h_key and not user:
+            clean_username = re.sub('[^a-zA-Z0-9\_\.]', '', username)
+            password = ''.join(random.choice(string.printable) for i in range(12))
+            user = User(username=clean_username, email=user_tosdr.email, privacy_accepted=datetime.now(), comms_opt_in=False, password=password, authority=self.request.default_authority)
+            self.session.add(user)
+
         credentials = {"user": user}
-
         headers, _, _ = self.oauth.create_authorization_response(
             self.request.url, scopes=scopes, credentials=credentials
         )
 
         try:
-            return HTTPFound(location=headers["Location"])
+            found = HTTPFound(location=headers["Location"])
+            return self._render_web_message_response(found.location)
         except KeyError as err:
             client_id = self.request.params.get("client_id")
             raise RuntimeError(
@@ -207,7 +219,8 @@ def post(self):
         )
 
         if status == 200:
-            return json.loads(body)
+            response = json.loads(body)
+            return response
 
         raise exception_response(status, detail=body)
 

h/views/api/links.py

@@ -1,3 +1,4 @@
+import os
 from h.views.api.config import api_config
 from h.views.api.helpers.angular import AngularRouteTemplater
 
@@ -18,6 +19,9 @@ def links(_context, request):
     tag_search_url = request.route_url("activity.search", _query={"q": 'tag:"__tag__"'})
     tag_search_url = tag_search_url.replace("__tag__", ":tag")
 
+    settings = request.registry.settings
+    tosdr_url = settings.get("h.tosdr")
+
     oauth_authorize_url = request.route_url("oauth_authorize")
     oauth_revoke_url = request.route_url("oauth_revoke")
 
@@ -34,4 +38,5 @@ def links(_context, request):
         "signup": request.route_url("signup"),
         "user": templater.route_template("stream.user_query"),
         "websocket": websocket_url,
+        "tosdr": tosdr_url
     }

h/views/client.py

@@ -63,6 +63,7 @@ def sidebar_app(request, extra=None):
         # The list of origins that the client will respond to cross-origin RPC
         # requests from.
         "rpcAllowedOrigins": settings.get("h.client_rpc_allowed_origins"),
+        "tosdr": settings.get("h.tosdr")
     }
 
     if sentry_public_dsn: