diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..32b7f0122bd --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,36 @@ +# You can override the included template(s) by including variable overrides +# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings +# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings +# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings +# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# Note that environment variables can be set in several places +# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence + +stages: +- publish + + +docker-nightly: + stage: publish + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: + - '' + script: + - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:nightly" + rules: + - if: $CI_COMMIT_BRANCH == "phoenix-integration" + + +docker-prod: + stage: publish + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: + - '' + script: + - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" --destination "${CI_REGISTRY_IMAGE}:stable" + rules: + - if: $CI_COMMIT_TAG diff --git a/conf/development.ini b/conf/development.ini index 76f88f5172b..5a782b914ef 100644 --- a/conf/development.ini +++ b/conf/development.ini @@ -13,6 +13,7 @@ h.bouncer_url: http://localhost:8000 h.client_rpc_allowed_origins: http://localhost:8001 https://localhost:48001 h.client_url: {current_scheme}://{current_host}:3001/hypothesis h.websocket_url: ws://localhost:5001/ws +h.tosdr: http://localhost:9090 h.debug: True h.reload_assets: True diff --git a/conf/production.ini b/conf/production.ini index bb45d3e8f02..d17706c03f2 100644 --- a/conf/production.ini +++ b/conf/production.ini @@ -8,6 +8,8 @@ use: call:h.app:create_app [filter:proxy-prefix] use: egg:PasteDeploy#prefix +prefix = /hypothesis + [loggers] keys = root, alembic, gunicorn.error, h diff --git a/docker-compose.yml b/docker-compose.yml index 510a61ddee6..0c3ef06dafe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,4 @@ services: - postgres: - image: postgres:15.6-alpine - ports: - - '127.0.0.1:5432:5432' - healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres"] - interval: 1s - environment: - POSTGRES_HOST_AUTH_METHOD: trust - networks: - - dbs - elasticsearch: - image: hypothesis/elasticsearch:latest - ports: - - '127.0.0.1:9200:9200' - healthcheck: - test: curl --fail --silent http://localhost:9200 >/dev/null - interval: 3s - start_period: 1m - environment: - - discovery.type=single-node rabbit: image: rabbitmq:3.12-management-alpine ports: @@ -31,4 +10,9 @@ networks: # To avoid having unnecessary dependencies between the projects # the network is created with `docker network create dbs` in each project's Makefile (make services) dbs: + name: dbs + external: true + elasticsearch: + name: elasticsearch external: true + diff --git a/h/assets.py b/h/assets.py index fe50a1d0f64..73d5d59386b 100644 --- a/h/assets.py +++ b/h/assets.py @@ -1,5 +1,6 @@ """View for serving static assets under `/assets`.""" +import os import importlib_resources from h_assets import Environment, assets_view from pyramid.settings import asbool @@ -9,8 +10,12 @@ def includeme(config): # pragma: no cover auto_reload = asbool(config.registry.settings.get("h.reload_assets", False)) h_files = importlib_resources.files("h") + asset_path = "/hypothesis/assets" + if "ASSET_PATH" in os.environ: + asset_path = os.environ["ASSET_PATH"] + assets_env = Environment( - assets_base_url="/assets", + assets_base_url=asset_path, bundle_config_path=h_files / "assets.ini", manifest_path=h_files / "../build/manifest.json", auto_reload=auto_reload, diff --git a/h/config.py b/h/config.py index 208d7c0d88a..a3b02835149 100644 --- a/h/config.py +++ b/h/config.py @@ -28,6 +28,7 @@ def configure(environ=None, settings=None): # pylint: disable=too-many-statemen environ = os.environ if settings is None: # pragma: no cover settings = {} + settings_manager = SettingsManager(settings, environ) # Configuration for external components settings_manager.set("broker_url", "BROKER_URL") @@ -88,6 +89,9 @@ def configure(environ=None, settings=None): # pylint: disable=too-many-statemen # secret. settings_manager.set("h.client_oauth_id", "CLIENT_OAUTH_ID") + # Base domain for tosdr + settings_manager.set("h.tosdr", "TOSDR_URL") + # The list of origins that the client will respond to cross-origin RPC # requests from. settings_manager.set( diff --git a/h/search/query.py b/h/search/query.py index 206b5187bc9..a2791b9c3a3 100644 --- a/h/search/query.py +++ b/h/search/query.py @@ -268,6 +268,7 @@ def __call__(self, search, params): [u for u in wildcard_uris if wildcard_uri_is_valid(u)], normalize_method=self._wildcard_uri_normalized, ) + strict_uri = uris[0] uris = self._normalize_uris(uris) queries = [] @@ -275,7 +276,8 @@ def __call__(self, search, params): queries = [Q("wildcard", **{"target.scope": u}) for u in wildcard_uris] if uris: queries.append(Q("terms", **{"target.scope": uris})) - return search.query("bool", should=queries) + # TOSDR + return search.query(Q("bool", must=[Q("match", uri=strict_uri)])) def _normalize_uris(self, query_uris, normalize_method=uri.normalize): uris = set() diff --git a/h/security/policy/combined.py b/h/security/policy/combined.py index abc6af57040..737dce5e25a 100644 --- a/h/security/policy/combined.py +++ b/h/security/policy/combined.py @@ -74,6 +74,7 @@ def _call_sub_policies(self, method, request, *args, **kwargs): return getattr(self._ui_policy, method)(request, *args, **kwargs) # Then we try the bearer header (or `access_token` GET param) + result = getattr(self._bearer_token_policy, method)(request, *args, **kwargs) if not result and self._http_basic_auth_policy.handles(request): @@ -87,7 +88,9 @@ def _call_sub_policies(self, method, request, *args, **kwargs): @staticmethod def _is_api_request(request): - return request.path.startswith("/api") and request.path not in [ + return (request.path.startswith("/api") or request.path.startswith("/hypothesis/api")) and request.path not in [ "/api/token", "/api/badge", + "/hypothesis/api/token", + "/hypothesis/api/badge" ] diff --git a/h/services/user.py b/h/services/user.py index a33bce54a16..d6a436d6f7f 100644 --- a/h/services/user.py +++ b/h/services/user.py @@ -1,4 +1,5 @@ import sqlalchemy as sa +from sqlalchemy.sql import text from h.models import User, UserIdentity from h.util.db import on_transaction_end @@ -50,6 +51,7 @@ def fetch(self, userid_or_username, authority=None): :rtype: h.models.User or None """ + if authority is not None: username = userid_or_username else: @@ -65,12 +67,19 @@ def fetch(self, userid_or_username, authority=None): self._cache[cache_key] = ( self.session.query(User) .filter_by(username=username) - .filter_by(authority=authority) + # comment for tosdr + # .filter_by(authority=authority) .one_or_none() ) return self._cache[cache_key] + def fetch_from_tosdr(self, h_key): + statement = text("SELECT * FROM users WHERE h_key =:x") + statement = statement.bindparams(x=h_key) + user_tosdr = self.session.execute(statement).one_or_none() + return user_tosdr + def fetch_all(self, userids): """ Fetch a list of users by their userids. diff --git a/h/session.py b/h/session.py index 9b86a67be61..0358938bf03 100644 --- a/h/session.py +++ b/h/session.py @@ -31,10 +31,15 @@ def profile(request, authority=None): authority = user.authority else: authority = authority or request.default_authority + + userid = request.authenticated_userid + + if not userid: + userid = user.userid return dict( { - "userid": request.authenticated_userid, + "userid": userid, "authority": authority, "groups": _current_groups(request, authority), "features": request.feature.all(), diff --git a/h/views/api/auth.py b/h/views/api/auth.py index 3b1233cea84..a2df1f81eb5 100644 --- a/h/views/api/auth.py +++ b/h/views/api/auth.py @@ -1,5 +1,9 @@ import json import logging +import random +import re +import string +from datetime import datetime from functools import wraps from urllib.parse import parse_qs, urlparse @@ -8,7 +12,9 @@ from pyramid.view import view_config, view_defaults from h import models +from h.models import User from h.services.oauth import DEFAULT_SCOPES + from h.util.datetime import utc_iso8601 from h.views.api.config import api_config from h.views.api.exceptions import OAuthAuthorizeError, OAuthTokenError @@ -33,18 +39,20 @@ def inner(*args, **kwargs): return inner - @view_defaults(route_name="oauth_authorize") class OAuthAuthorizeController: def __init__(self, context, request): self.context = context self.request = request - + self.session = request.db self.user_svc = self.request.find_service(name="user") self.oauth = self.request.find_service(name="oauth_provider") @view_config( - request_method="GET", renderer="h:templates/oauth/authorize.html.jinja2" + request_method="GET", + # for tosdr + # renderer=None + renderer="h:templates/oauth/authorize.html.jinja2" ) def get(self): """ @@ -87,7 +95,7 @@ def get_web_message(self): @view_config( request_method="POST", - is_authenticated=True, + # is_authenticated=True, renderer="json", ) def post(self): @@ -102,8 +110,8 @@ def post(self): @view_config( request_method="POST", request_param="response_mode=web_message", - is_authenticated=True, - renderer="h:templates/oauth/authorize_web_message.html.jinja2", + # is_authenticated=True, + renderer="json", ) def post_web_message(self): """ @@ -115,8 +123,7 @@ def post_web_message(self): .. _draft-sakimura-oauth: https://tools.ietf.org/html/draft-sakimura-oauth-wmrm-00 """ - found = self._authorized_response() - return self._render_web_message_response(found.location) + return self._authorized_response() def _authorize(self): try: @@ -126,12 +133,6 @@ def _authorize(self): err.description or f"Error: {self.context.error}" ) from err - if self.request.authenticated_userid is None: - raise HTTPFound( - self.request.route_url( - "login", _query={"next": self.request.url, "for_oauth": True} - ) - ) client_id = credentials.get("client_id") client = self.request.db.get(models.AuthClient, client_id) @@ -142,7 +143,7 @@ def _authorize(self): # logged-in user. if client.trusted: return self._authorized_response() - + state = credentials.get("state") user = self.user_svc.fetch(self.request.authenticated_userid) response_mode = credentials.get("request").response_mode @@ -156,20 +157,31 @@ def _authorize(self): "state": state, } - @handles_oauth_errors + # @handles_oauth_errors def _authorized_response(self): # We don't support scopes at the moment, but oauthlib does need a scope, # so we're explicitly overwriting whatever the client provides. scopes = DEFAULT_SCOPES - user = self.user_svc.fetch(self.request.authenticated_userid) + # TOSDR : find tosdr user based on h_key cookie + h_key = self.request.cookies.get('h_key') + user_tosdr = self.user_svc.fetch_from_tosdr(h_key) + username = user_tosdr.username + user = self.user_svc.fetch(username, authority=self.request.default_authority) + # TOSDR : create user in h if it does not exist + if h_key and not user: + clean_username = re.sub('[^a-zA-Z0-9\_\.]', '', username) + password = ''.join(random.choice(string.printable) for i in range(12)) + user = User(username=clean_username, email=user_tosdr.email, privacy_accepted=datetime.now(), comms_opt_in=False, password=password, authority=self.request.default_authority) + self.session.add(user) + credentials = {"user": user} - headers, _, _ = self.oauth.create_authorization_response( self.request.url, scopes=scopes, credentials=credentials ) try: - return HTTPFound(location=headers["Location"]) + found = HTTPFound(location=headers["Location"]) + return self._render_web_message_response(found.location) except KeyError as err: # pragma: no cover client_id = self.request.params.get("client_id") raise RuntimeError( @@ -207,7 +219,8 @@ def post(self): ) if status == 200: - return json.loads(body) + response = json.loads(body) + return response raise exception_response(status, detail=body) diff --git a/h/views/api/links.py b/h/views/api/links.py index 1448495ea1d..e37c68cf8b4 100644 --- a/h/views/api/links.py +++ b/h/views/api/links.py @@ -1,3 +1,4 @@ +import os from h.views.api.config import api_config from h.views.api.helpers.angular import AngularRouteTemplater @@ -18,6 +19,9 @@ def links(_context, request): tag_search_url = request.route_url("activity.search", _query={"q": 'tag:"__tag__"'}) tag_search_url = tag_search_url.replace("__tag__", ":tag") + settings = request.registry.settings + tosdr_url = settings.get("h.tosdr") + oauth_authorize_url = request.route_url("oauth_authorize") oauth_revoke_url = request.route_url("oauth_revoke") @@ -34,4 +38,5 @@ def links(_context, request): "signup": request.route_url("signup"), "user": templater.route_template("stream.user_query"), "websocket": websocket_url, + "tosdr": tosdr_url } diff --git a/h/views/client.py b/h/views/client.py index 16a8eb4ac38..7910e0acd06 100644 --- a/h/views/client.py +++ b/h/views/client.py @@ -63,6 +63,7 @@ def sidebar_app(request, extra=None): # The list of origins that the client will respond to cross-origin RPC # requests from. "rpcAllowedOrigins": settings.get("h.client_rpc_allowed_origins"), + "tosdr": settings.get("h.tosdr") } if sentry_public_dsn: # pragma: no cover diff --git a/tox.ini b/tox.ini index 1cf2af1c207..a0eaa502bcc 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ passenv = dev: SENTRY_DSN_CLIENT dev: SENTRY_DSN_FRONTEND dev: SENTRY_ENVIRONMENT + dev: TOSDR_URL dev: USE_HTTPS dev: NEW_RELIC_LICENSE_KEY dev: NEW_RELIC_APP_NAME @@ -92,6 +93,7 @@ setenv = dev: HTTP_HOST = {env:HTTP_HOST:localhost:5000} dev: PYTHONPATH = . dev: APP_URL = {env:APP_URL:http://localhost:5000} + dev: ASSET_PATH = {env:ASSET_PATH:/assets} dev: WEBSOCKET_URL = {env:WEBSOCKET_URL:ws://localhost:5001/ws} dev: ENABLE_WEB = {env:ENABLE_WEB:true} dev: ENABLE_WEBSOCKET = {env:ENABLE_WEBSOCKET:true}