diff --git a/.dockerignore b/.dockerignore index cf76ed57..67ef5242 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ # Git .git .gitignore +.github create_permissions.log # Logs log/* @@ -9,3 +10,8 @@ tmp/* # Editor temp files *.swp *.swo +coverage +# Ignore generated test data +test/data/uploaded_ontologies/**/* +test/data/ontology_files/repo/**/* +test/log diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..212d5dcd --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,85 @@ +# Workflow for deploying ontologies_api to stage/prod systems via capistrano. +# This workflow runs after a successeful execution of the unit test workflow and it +# can also be triggered manually. +# +# Required github secrets: +# +# CONFIG_REPO - github repo containing config and customizations for the API. Format 'author/private_config_repo' +# it is used for getting capistrano deployment configuration for stages on the github actions runner and +# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the remote servers for pulling configs. +# +# GH_PAT - github Personal Access Token for accessing PRIVATE_CONFIG_REPO +# +# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if app servers are hosted on private network. +# +# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/deploy_id_rsa_enc (see miloserdow/capistrano-deploy) +# this SSH key is used for accessing jump host, UI nodes, and private github repo. + +name: Capistrano Deployment +# Controls when the action will run. +on: + # Trigger deployment to staging after unit test action completes + workflow_run: + workflows: ["Ruby Unit Tests"] + types: + - completed + branches: [master, develop] + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + branches: [master, develop] + inputs: + BRANCH: + description: 'Branch/tag to deploy' + default: develop + required: true + environment: + description: 'target environment to deploy to' + type: choice + options: + - staging + - production + default: staging + +jobs: + deploy: + runs-on: ubuntu-latest + # run deployment only if "Ruby Unit Tests" workflow completes sucessefully or when manually triggered + if: ${{ (github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch') }} + env: + BUNDLE_WITHOUT: default #install gems required primarily for the deployment in order to speed this workflow + PRIVATE_CONFIG_REPO: ${{ format('git@github.com:{0}.git', secrets.CONFIG_REPO) }} + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: set branch/tag and environment to deploy from inputs + run: | + # workflow_dispatch default input doesn't get set on push so we need to set defaults + # via shell parameter expansion + # https://dev.to/mrmike/github-action-handling-input-default-value-5f2g + USER_INPUT_BRANCH=${{ inputs.branch }} + echo "BRANCH=${USER_INPUT_BRANCH:-develop}" >> $GITHUB_ENV + USER_INPUT_ENVIRONMENT=${{ inputs.environment }} + echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7.6 # Not needed with a .ruby-version file + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: get-deployment-config + uses: actions/checkout@v3 + with: + repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings + token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT + path: deploy_config + - name: copy-deployment-config + run: cp -r deploy_config/ontologies_api/* . + # add ssh hostkey so that capistrano doesn't complain + - name: Add jumphost's hostkey to Known Hosts + run: | + mkdir -p ~/.ssh + ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts + shell: bash + - uses: miloserdow/capistrano-deploy@master + with: + target: ${{ env.TARGET }} # which environment to deploy + deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000..a3f95c93 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,42 @@ +name: Docker Image CI + +on: + release: + types: [published] + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: bioportal/ontologies_api + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + build-args: | + RUBY_VERSION=2.7 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index ce35f77c..13dd5c3f 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -7,11 +7,12 @@ on: jobs: test: strategy: + fail-fast: false matrix: backend: ['api', 'api-agraph'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build docker-compose run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step - name: Run unit tests @@ -19,7 +20,7 @@ jobs: # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v' + docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.gitignore b/.gitignore index 68d9c3f8..4045ab4b 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ test/test_run.log test/data/ontology_files/catalog-v001.xml create_permissions.log + +ontologies_api.iml diff --git a/Capfile b/Capfile index f86314f3..7ecc995c 100644 --- a/Capfile +++ b/Capfile @@ -22,6 +22,6 @@ require 'capistrano/bundler' # require 'capistrano/rails/assets' # require 'capistrano/rails/migrations' require 'capistrano/locally' - +require 'new_relic/recipes' # announce deployments in NewRelic # Loads custom tasks from `lib/capistrano/tasks' if you have any defined. Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r } diff --git a/Dockerfile b/Dockerfile index 3e65fe4a..a7adf16c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ FROM ruby:$RUBY_VERSION-$DISTRO_NAME RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ openjdk-11-jre-headless \ raptor2-utils \ - wait-for-it \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /srv/ontoportal/ontologies_api @@ -15,12 +14,16 @@ COPY Gemfile* /srv/ontoportal/ontologies_api/ WORKDIR /srv/ontoportal/ontologies_api -RUN gem update --system +# set rubygem and bundler to the last version supported by ruby 2.7 +# remove version after ruby v3 upgrade +RUN gem update --system '3.4.22' +RUN gem install bundler -v 2.4.22 RUN gem install bundler ENV BUNDLE_PATH=/srv/ontoportal/bundle RUN bundle install COPY . /srv/ontoportal/ontologies_api +RUN cp /srv/ontoportal/ontologies_api/config/environments/config.rb.sample /srv/ontoportal/ontologies_api/config/environments/development.rb EXPOSE 9393 CMD ["bundle", "exec", "rackup", "-p", "9393", "--host", "0.0.0.0"] diff --git a/Gemfile b/Gemfile index fdc0aac6..ec1f76f3 100644 --- a/Gemfile +++ b/Gemfile @@ -1,12 +1,11 @@ source 'https://rubygems.org' -gem 'activesupport', '~> 3.0' +gem 'activesupport', '~> 3.2' # see https://github.com/ncbo/ontologies_api/issues/69 gem 'bigdecimal', '1.4.2' -gem 'faraday', '~> 1.9' gem 'json-schema', '~> 2.0' gem 'multi_json', '~> 1.0' -gem 'oj', '~> 2.0' +gem 'oj', '~> 3.0' gem 'parseconfig' gem 'rack' gem 'rake', '~> 10.0' @@ -18,7 +17,7 @@ gem 'sinatra-contrib', '~> 1.0' gem 'ffi' gem 'rack-accept', '~> 0.4' gem 'rack-attack', '~> 6.6.1', require: 'rack/attack' -gem 'rack-cache', '~> 1.0' +gem 'rack-cache', '~> 1.13.0' # see https://github.com/ncbo/ontologies_api/issues/118 gem 'rack-cors', require: 'rack/cors' # GitHub dependency can be removed when https://github.com/niko/rack-post-body-to-params/pull/6 is merged and released gem 'rack-post-body-to-params', github: 'palexander/rack-post-body-to-params', branch: 'multipart_support' @@ -27,18 +26,18 @@ gem 'redis-rack-cache', '~> 2.0' # Data access (caching) gem 'redis' -gem 'redis-activesupport' +gem 'redis-store', '~>1.10' # Monitoring gem 'cube-ruby', require: 'cube' -gem 'newrelic_rpm' +gem 'newrelic_rpm', group: [:default, :deployment] # HTTP server gem 'unicorn' gem 'unicorn-worker-killer' # Templating -gem 'haml' +gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO @@ -50,12 +49,19 @@ gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'ma gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' group :development do + # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 + gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' + gem 'rubocop' +end + +group :deployment do + # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 + gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false gem 'capistrano', '~> 3', require: false gem 'capistrano-bundler', require: false gem 'capistrano-locally', require: false gem 'capistrano-rbenv', require: false - gem 'pry' - gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' + gem 'ed25519', '>= 1.2', '< 2.0', require: false end group :profiling do diff --git a/Gemfile.lock b/Gemfile.lock index 2bea0ca4..f56d0232 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 562826ba21f7da641159071531375776a1414207 + revision: 75436fe8e387febc53e34ee31ff0e6dd837a9d3f branch: master specs: goo (0.0.2) @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 71d41e3afb35dafe29abfb6d9becaadc725bad36 + revision: 63c986880aa88c9384043e6611a682434a14aba7 branch: master specs: ncbo_annotator (0.0.1) @@ -26,13 +26,13 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 5683c127113cf8eee68445413443ba3e164fbed7 + revision: b01a9046c4c110f00e832e8c16073a74558fdba5 branch: master specs: ncbo_cron (0.0.1) dante goo - google-apis-analytics_v3 + google-analytics-data mlanett-redis-lock multi_json ncbo_annotator @@ -42,7 +42,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 6010ff60b99dc1282822b8a1fb59bd59f453755f + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 8196bf34b45c75f8104bb76dfcba1db0f2c048e4 + revision: ee0013f0ee23876076bff9d9258b46371ec3b248 branch: master specs: ontologies_linked_data (0.0.1) @@ -73,7 +73,7 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9 + revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 branch: master specs: sparql-client (1.0.1) @@ -103,15 +103,18 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - airbrussh (1.4.1) + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) + airbrussh (1.5.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.23.0) - bcrypt (3.1.18) + ast (2.4.2) + backports (3.24.1) + base64 (0.2.0) + bcrypt (3.1.20) + bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.18.0) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -124,107 +127,141 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) + connection_pool (2.4.1) cube-ruby (0.0.3) dante (0.2.0) - declarative (0.0.20) + date (3.3.4) docile (1.4.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) - faraday (1.10.1) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + domain_name (0.6.20240107) + ed25519 (1.3.0) + faraday (2.8.1) + base64 + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - ffi (1.15.5) + faraday-net_http (3.0.2) + faraday-retry (2.2.0) + faraday (~> 2.0) + ffi (1.16.3) + gapic-common (0.21.1) + faraday (>= 1.9, < 3.a) + faraday-retry (>= 1.0, < 3.a) + google-protobuf (~> 3.18) + googleapis-common-protos (>= 1.4.0, < 2.a) + googleapis-common-protos-types (>= 1.11.0, < 2.a) + googleauth (~> 1.9) + grpc (~> 1.59) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.10.0) - google-apis-core (>= 0.7, < 2.a) - google-apis-core (0.7.0) - addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.16.2, < 2.a) - httpclient (>= 2.8.1, < 3.a) - mini_mime (~> 1.0) - representable (~> 3.0) - retriable (>= 2.0, < 4.a) - rexml - webrick - googleauth (1.2.0) - faraday (>= 0.17.3, < 3.a) + google-analytics-data (0.4.0) + google-analytics-data-v1beta (>= 0.7, < 2.a) + google-cloud-core (~> 1.6) + google-analytics-data-v1beta (0.11.1) + gapic-common (>= 0.21.1, < 2.a) + google-cloud-errors (~> 1.0) + google-cloud-core (1.6.1) + google-cloud-env (>= 1.0, < 3.a) + google-cloud-errors (~> 1.0) + google-cloud-env (2.1.0) + faraday (>= 1.0, < 3.a) + google-cloud-errors (1.3.1) + google-protobuf (3.25.2-aarch64-linux) + google-protobuf (3.25.2-arm64-darwin) + google-protobuf (3.25.2-x86_64-darwin) + google-protobuf (3.25.2-x86_64-linux) + googleapis-common-protos (1.4.0) + google-protobuf (~> 3.14) + googleapis-common-protos-types (~> 1.2) + grpc (~> 1.27) + googleapis-common-protos-types (1.11.0) + google-protobuf (~> 3.18) + googleauth (1.9.1) + faraday (>= 1.0, < 3.a) + google-cloud-env (~> 2.1) jwt (>= 1.4, < 3.0) - memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) + grpc (1.60.0-aarch64-linux) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) + grpc (1.60.0-arm64-darwin) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) + grpc (1.60.0-x86_64-darwin) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) + grpc (1.60.0-x86_64-linux) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) - httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.6.2) + json (2.7.1) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.6.2) - jwt (2.4.1) + json_pure (2.7.1) + jwt (2.7.1) kgio (2.11.4) - libxml-ruby (3.2.3) - logger (1.5.1) + language_server-protocol (3.17.0.3) + libxml-ruby (5.0.2) + logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) - memoist (0.16.2) + net-imap + net-pop + net-smtp method_source (1.0.0) - mime-types (3.4.1) + mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) - mini_mime (1.1.2) + mime-types-data (3.2023.1205) + mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - multipart-post (2.2.3) + mutex_m (0.2.0) net-http-persistent (2.9.4) - net-scp (1.2.1) - net-ssh (>= 2.6.5) - net-ssh (7.0.1) + net-imap (0.4.9.1) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) + net-sftp (4.0.0) + net-ssh (>= 5.0.0, < 8.0.0) + net-smtp (0.4.0.1) + net-protocol + net-ssh (7.2.1) netrc (0.11.0) - newrelic_rpm (8.9.0) - oj (2.18.5) + newrelic_rpm (9.7.0) + oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) + parallel (1.24.0) parseconfig (1.1.2) + parser (3.3.0.3) + ast (~> 2.4.1) + racc pony (1.13.1) mail (>= 2.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (4.0.7) + public_suffix (5.0.4) + racc (1.7.3) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -234,52 +271,63 @@ GEM rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.0.0) + rack-mini-profiler (3.3.0) rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.0.2) + rack-test (2.1.0) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + rainbow (3.1.1) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) - redcarpet (3.5.1) - redis (4.7.1) - redis-activesupport (5.3.0) - activesupport (>= 3, < 8) - redis-store (>= 1.3, < 2) + redcarpet (3.6.0) + redis (5.0.8) + redis-client (>= 0.17.0) + redis-client (0.19.1) + connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) - representable (3.2.0) - declarative (< 0.1.0) - trailblazer-option (>= 0.1.1, < 0.2.0) - uber (< 0.2.0) + redis-store (1.10.0) + redis (>= 4, < 6) + regexp_parser (2.9.0) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - retriable (3.1.2) - rexml (3.2.5) + rexml (3.2.6) rsolr (2.5.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) rubyzip (2.3.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) - signet (0.17.0) + signet (0.18.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) @@ -301,19 +349,18 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.2) + sshkit (1.22.0) + mutex_m net-scp (>= 1.1.2) + net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.8.2) - tilt (2.0.11) - trailblazer-option (0.1.2) - tzinfo (2.0.5) + temple (0.10.3) + tilt (2.3.0) + timeout (0.4.1) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) - uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) + unicode-display_width (2.5.0) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) @@ -322,23 +369,27 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) - webrick (1.7.0) PLATFORMS + aarch64-linux + arm64-darwin-22 + arm64-darwin-23 + x86_64-darwin-23 x86_64-linux DEPENDENCIES - activesupport (~> 3.0) + activesupport (~> 3.2) + bcrypt_pbkdf (>= 1.0, < 2.0) bigdecimal (= 1.4.2) capistrano (~> 3) capistrano-bundler capistrano-locally capistrano-rbenv cube-ruby - faraday (~> 1.9) + ed25519 (>= 1.2, < 2.0) ffi goo! - haml + haml (~> 5.2.2) json-schema (~> 2.0) minitest (~> 4.0) minitest-stub_any_instance @@ -347,14 +398,13 @@ DEPENDENCIES ncbo_cron! ncbo_ontology_recommender! newrelic_rpm - oj (~> 2.0) + oj (~> 3.0) ontologies_linked_data! parseconfig - pry rack rack-accept (~> 0.4) rack-attack (~> 6.6.1) - rack-cache (~> 1.0) + rack-cache (~> 1.13.0) rack-cors rack-mini-profiler rack-post-body-to-params! @@ -363,8 +413,9 @@ DEPENDENCIES rake (~> 10.0) redcarpet redis - redis-activesupport redis-rack-cache (~> 2.0) + redis-store (~> 1.10) + rubocop shotgun! simplecov simplecov-cobertura @@ -376,4 +427,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.20 + 2.4.22 diff --git a/app.rb b/app.rb index 5360ae4b..41ad56ac 100644 --- a/app.rb +++ b/app.rb @@ -11,8 +11,6 @@ require 'oj' require 'multi_json' require 'cgi' -require 'google/apis/analytics_v3' -require 'google/api_client/auth/key_utils' # NCBO dependencies require 'ontologies_linked_data' diff --git a/bin/owlapi_wrapper.jar b/bin/owlapi_wrapper.jar deleted file mode 100755 index 85bd53e4..00000000 --- a/bin/owlapi_wrapper.jar +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env ruby -# -# This file was generated by Bundler. -# -# The application 'owlapi_wrapper.jar' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -require 'pathname' -ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -require 'rubygems' -require 'bundler/setup' - -load Gem.bin_path('ontologies_linked_data', 'owlapi_wrapper.jar') diff --git a/config/deploy.rb b/config/deploy.rb index 441be857..d4f0903b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,9 +1,10 @@ # config valid only for Capistrano 3 -APP_PATH = '/srv/ncbo' +APP_PATH = '/srv/ontoportal' +set :author, 'ncbo' set :application, 'ontologies_api' -set :repo_url, "https://github.com/ncbo/#{fetch(:application)}.git" +set :repo_url, "https://github.com/#{fetch(:author)}/#{fetch(:application)}.git" set :deploy_via, :remote_cache @@ -77,7 +78,7 @@ namespace :deploy do - desc 'Incorporate the bioportal_conf private repository content' + desc 'Incorporate the private repository content' # Get cofiguration from repo if PRIVATE_CONFIG_REPO env var is set # or get config from local directory if LOCAL_CONFIG_PATH env var is set task :get_config do @@ -108,6 +109,7 @@ after :publishing, :get_config after :get_config, :restart # after :deploy, :smoke_test + after :restart, "newrelic:notice_deployment" after :restart, :clear_cache do on roles(:web), in: :groups, limit: 3, wait: 10 do diff --git a/config/deploy/production.rb b/config/deploy/production.rb index c84d24ea..1c61d0f7 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -4,9 +4,7 @@ # server in each group is considered to be the first # unless any hosts have the primary property set. # Don't declare `role :all`, it's a meta role -role :app, %w{deploy@example.com} -role :web, %w{deploy@example.com} -role :db, %w{deploy@example.com} +ole :app, %w{api1.prd.ontoportal.org api2.prd.ontoportal.org/} # Extended Server Syntax # ====================== @@ -14,7 +12,6 @@ # definition into the server list. The second argument # something that quacks like a hash can be used to set # extended properties on the server. -server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value # you can set custom ssh options # it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options @@ -37,3 +34,19 @@ # # password: 'please use keys' # } # setting per server overrides global ssh_options +set :ssh_options, { + user: 'deployer', + forward_agent: 'true', + keys: %w(config/deploy_id_rsa), + auth_methods: %w(publickey), + # use ssh proxy if UI servers are on a private network + proxy: Net::SSH::Proxy::Command.new('ssh deployer@sshproxy.ontoportal.org -W %h:%p') +} + +# set git branch to deploy from. Default to master in produciton and develop in staging +# this can be overwritten with a different branch or tag, i.e v6.9.0 by setting env variable BRANCH +BRANCH = ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master' +set :branch, BRANCH + +# private git repo for configuraiton +PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'git@github.com:author/private_config_repo.git' diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb new file mode 100644 index 00000000..8b1afa7b --- /dev/null +++ b/config/deploy/staging.rb @@ -0,0 +1,52 @@ +# Simple Role Syntax +# ================== +# Supports bulk-adding hosts to roles, the primary +# server in each group is considered to be the first +# unless any hosts have the primary property set. +# Don't declare `role :all`, it's a meta role +ole :app, %w{api1.stg.ontoportal.org api2.stg.ontoportal.org} + +# Extended Server Syntax +# ====================== +# This can be used to drop a more detailed server +# definition into the server list. The second argument +# something that quacks like a hash can be used to set +# extended properties on the server. + +# you can set custom ssh options +# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options +# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start) +# set it globally +# set :ssh_options, { +# keys: %w(/home/rlisowski/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(password) +# } +# and/or per server +# server 'example.com', +# user: 'user_name', +# roles: %w{web app}, +# ssh_options: { +# user: 'user_name', # overrides user setting above +# keys: %w(/home/user_name/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: 'please use keys' +# } +# setting per server overrides global ssh_options +set :ssh_options, { + user: 'deployer', + forward_agent: 'true', + keys: %w(config/deploy_id_rsa), + auth_methods: %w(publickey), + # use ssh proxy if UI servers are on a private network + proxy: Net::SSH::Proxy::Command.new('ssh deployer@sshproxy.ontoportal.org -W %h:%p') +} + +# set git branch to deploy from. Default to master in produciton and develop in staging +# this can be overwritten with a different branch or tag, i.e v6.9.0 by setting env variable BRANCH +BRANCH = ENV.include?('BRANCH') ? ENV['BRANCH'] : 'develop' +set :branch, BRANCH + +# private git repo for configuraiton +PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'git@github.com:author/private_config_repo.git' diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 3d185b43..e8667dc4 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -3,36 +3,61 @@ # development.rb # test.rb -LinkedData.config do |config| - config.repository_folder = File.expand_path('../../../test/data/uploaded_ontologies', __FILE__) - config.goo_host = "localhost" - config.goo_port = 9000 - config.search_server_url = "http://localhost:8983/solr/term_search_core1" - config.property_search_server_url = "http://localhost:8983/solr/prop_search_core1" - config.rest_url_prefix = "http://data.bioontology.org/" - config.enable_security = false - - #caches - config.http_redis_host = "localhost" - config.http_redis_port = 6379 - config.goo_redis_host = "localhost" - config.goo_redis_port = 6379 +GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" +GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" +GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" +GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" +GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 +MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost" +MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555 +MGREP_DICTIONARY_FILE = ENV.include?("MGREP_DICTIONARY_FILE") ? ENV["MGREP_DICTIONARY_FILE"] : "./test/data/dictionary.txt" +REDIS_GOO_CACHE_HOST = ENV.include?("REDIS_GOO_CACHE_HOST") ? ENV["REDIS_GOO_CACHE_HOST"] : "localhost" +REDIS_HTTP_CACHE_HOST = ENV.include?("REDIS_HTTP_CACHE_HOST") ? ENV["REDIS_HTTP_CACHE_HOST"] : "localhost" +REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSISTENT_HOST"] : "localhost" +REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 +REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" +REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" +REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" - #Ontology Analytics Redis - config.ontology_analytics_redis_host = "localhost" - config.ontology_analytics_redis_port = 6379 +LinkedData.config do |config| + config.goo_backend_name = GOO_BACKEND_NAME.to_s + config.goo_host = GOO_HOST.to_s + config.goo_port = GOO_PORT.to_i + config.goo_path_query = GOO_PATH_QUERY.to_s + config.goo_path_data = GOO_PATH_DATA.to_s + config.goo_path_update = GOO_PATH_UPDATE.to_s + config.goo_redis_host = REDIS_GOO_CACHE_HOST.to_s + config.goo_redis_port = REDIS_PORT.to_i + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i + config.ontology_analytics_redis_host = REDIS_PERSISTENT_HOST.to_s + config.ontology_analytics_redis_port = REDIS_PORT.to_i + config.search_server_url = SOLR_TERM_SEARCH_URL.to_s + config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s + config.replace_url_prefix = true + config.rest_url_prefix = REST_URL_PREFIX.to_s +# config.enable_notifications = false end Annotator.config do |config| - config.mgrep_dictionary_file ||= "./test/tmp/dict" - config.stop_words_default_file ||= "./config/default_stop_words.txt" - config.mgrep_host ||= "localhost" - config.mgrep_port ||= 55555 - config.annotator_redis_host = "localhost" - config.annotator_redis_port = 6379 - config.enable_recognizer_param = false + config.annotator_redis_host = REDIS_PERSISTENT_HOST.to_s + config.annotator_redis_port = REDIS_PORT.to_i + config.mgrep_host = MGREP_HOST.to_s + config.mgrep_port = MGREP_PORT.to_i + config.mgrep_dictionary_file = MGREP_DICTIONARY_FILE.to_s end LinkedData::OntologiesAPI.config do |config| - config.restrict_download = ["ACR0", "ACR1", "ACR2"] + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i +# config.restrict_download = ["ACR0", "ACR1", "ACR2"] +end + +NcboCron.config do |config| + config.redis_host = REDIS_PERSISTENT_HOST.to_s + config.redis_port = REDIS_PORT.to_i + config.ontology_report_path = REPORT_PATH end diff --git a/config/environments/test.rb b/config/environments/test.rb index acd5fd5a..c18cbcf4 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,18 +1,36 @@ -# This file is designed to be used for unit testing with docker-compose +# conig file for unit tests + +# GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "AG" +# GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" +# GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/repositories/bioportal" +# GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/repositories/bioportal/statements" +# GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/repositories/bioportal/statements" +# GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 10035 GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" -GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" +GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" -GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 -GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" -REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : "localhost" -REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 -SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" -SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 8080 + +MGREP_DICTIONARY_FILE = ENV.include?("MGREP_DICTIONARY_FILE") ? ENV["MGREP_DICTIONARY_FILE"] : "./test/data/dictionary.txt" MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost" + + +# MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55556 MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555 + +REDIS_GOO_CACHE_HOST = ENV.include?("REDIS_GOO_CACHE_HOST") ? ENV["REDIS_GOO_CACHE_HOST"] : "localhost" +REDIS_HTTP_CACHE_HOST = ENV.include?("REDIS_HTTP_CACHE_HOST") ? ENV["REDIS_HTTP_CACHE_HOST"] : "localhost" +REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSISTENT_HOST"] : "localhost" +REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 +REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" +REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" + LinkedData.config do |config| config.goo_backend_name = GOO_BACKEND_NAME.to_s config.goo_host = GOO_HOST.to_s @@ -20,34 +38,32 @@ config.goo_path_query = GOO_PATH_QUERY.to_s config.goo_path_data = GOO_PATH_DATA.to_s config.goo_path_update = GOO_PATH_UPDATE.to_s - config.goo_redis_host = REDIS_HOST.to_s + config.goo_redis_host = REDIS_GOO_CACHE_HOST.to_s config.goo_redis_port = REDIS_PORT.to_i - config.http_redis_host = REDIS_HOST.to_s + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s config.http_redis_port = REDIS_PORT.to_i - config.ontology_analytics_redis_host = REDIS_HOST.to_s + config.ontology_analytics_redis_host = REDIS_PERSISTENT_HOST.to_s config.ontology_analytics_redis_port = REDIS_PORT.to_i config.search_server_url = SOLR_TERM_SEARCH_URL.to_s config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s -# config.enable_notifications = false + #config.enable_notifications = false end Annotator.config do |config| - config.annotator_redis_host = REDIS_HOST.to_s - config.annotator_redis_port = REDIS_PORT.to_i - config.mgrep_host = MGREP_HOST.to_s - config.mgrep_port = MGREP_PORT.to_i - config.mgrep_dictionary_file = "./test/data/dictionary.txt" -end - -OntologyRecommender.config do |config| + config.annotator_redis_host = REDIS_PERSISTENT_HOST.to_s + config.annotator_redis_port = REDIS_PORT.to_i + config.mgrep_host = MGREP_HOST.to_s + config.mgrep_port = MGREP_PORT.to_i + config.mgrep_dictionary_file = MGREP_DICTIONARY_FILE.to_s end LinkedData::OntologiesAPI.config do |config| - config.http_redis_host = REDIS_HOST.to_s + config.http_redis_host = REDIS_HTTP_CACHE_HOST.to_s config.http_redis_port = REDIS_PORT.to_i end NcboCron.config do |config| - config.redis_host = REDIS_HOST.to_s + config.redis_host = REDIS_PERSISTENT_HOST.to_s config.redis_port = REDIS_PORT.to_i +# config.ontology_report_path = REPORT_PATH end diff --git a/config/rack_attack.rb b/config/rack_attack.rb index 7256f1c7..4612b493 100644 --- a/config/rack_attack.rb +++ b/config/rack_attack.rb @@ -1,17 +1,18 @@ # frozen_string_literal: true -puts "(API) >> Throttling enabled at #{LinkedData::OntologiesAPI.settings.req_per_second_per_ip} req/sec" +limit_req_ip = LinkedData::OntologiesAPI.settings.req_per_second_per_ip +limit_req_ip_heavy = limit_req_ip / 5 +puts "(API) >> Throttling enabled at #{limit_req_ip} req/sec" require 'rack/attack' -require 'redis-activesupport' use Rack::Attack attack_redis_host_port = { host: LinkedData::OntologiesAPI.settings.http_redis_host, - port: LinkedData::OntologiesAPI.settings.http_redis_port + port: LinkedData::OntologiesAPI.settings.http_redis_port, + db: 1 } -attack_store = ActiveSupport::Cache::RedisStore.new(attack_redis_host_port) -Rack::Attack.cache.store = attack_store +Rack::Attack.cache.store = Redis.new(attack_redis_host_port) safe_ips = LinkedData::OntologiesAPI.settings.safe_ips ||= Set.new safe_ips.each do |safe_ip| @@ -20,24 +21,33 @@ safe_accounts = LinkedData::OntologiesAPI.settings.safe_accounts ||= Set.new(%w[ncbobioportal ontoportal_ui biomixer]) -Rack::Attack.safelist('mark safe accounts such as ontoportal_ui and biomixer as safe') do |req| - req.env['REMOTE_USER'] && safe_accounts.include?(req.env['REMOTE_USER'].username) +Rack::Attack.safelist('mark safe accounts such as ontoportal_ui and biomixer as safe') do |request| + request.env['REMOTE_USER'] && safe_accounts.include?(request.env['REMOTE_USER'].username) end -Rack::Attack.safelist('mark administrators as safe') do |req| - req.env['REMOTE_USER']&.admin? +Rack::Attack.safelist('mark administrators as safe') do |request| + request.env['REMOTE_USER']&.admin? end -Rack::Attack.throttle('req/ip', limit: LinkedData::OntologiesAPI.settings.req_per_second_per_ip, - period: 1.second, &:ip) +Rack::Attack.throttle('req/ip/heavy', limit: limit_req_ip_heavy, period: 1.second) do |req| + req.ip if req.path.include?('/recommender') || req.path.include?('/annotator') +end + +Rack::Attack.throttle('req/ip', limit: limit_req_ip, period: 1.second) do |req| + req.ip +end + +Rack::Attack.throttled_responder = lambda do |request| + match_data = request.env['rack.attack.match_data'] -Rack::Attack.throttled_response = lambda do |env| - match_data = env['rack.attack.match_data'] headers = { 'RateLimit-Limit' => match_data[:limit].to_s, 'RateLimit-Remaining' => '0', 'RateLimit-Reset' => match_data[:period].to_s } - body = "You have made #{match_data[:count]} requests in the last #{match_data[:period]} seconds. For user #{env['REMOTE_USER']}, we limit API Keys to #{match_data[:limit]} requests every #{match_data[:period]} seconds" + + body = "You have made #{match_data[:count]} requests in the last #{match_data[:period]} seconds. + For user #{request.env['REMOTE_USER']}, we limit API Keys to #{match_data[:limit]} requests every #{match_data[:period]} seconds\n" + [429, headers, [body]] end diff --git a/controllers/metrics_controller.rb b/controllers/metrics_controller.rb index 4b034de4..cfc476bf 100644 --- a/controllers/metrics_controller.rb +++ b/controllers/metrics_controller.rb @@ -85,7 +85,6 @@ class MetricsController < ApplicationController get "/ontologies/:ontology/metrics" do check_last_modified_collection(LinkedData::Models::Metric) ont, sub = get_ontology_and_submission - ont = Ontology.find(params["ontology"]).first error 404, "Ontology #{params['ontology']} not found" unless ont sub.bring(ontology: [:acronym], metrics: LinkedData::Models::Metric.goo_attrs_to_load(includes_param)) reply sub.metrics || {} @@ -106,10 +105,9 @@ class MetricsController < ApplicationController # reply {} end - get "/ontologies/:ontology/submissions/:submissionId/metrics" do + get "/ontologies/:ontology/submissions/:ontology_submission_id/metrics" do check_last_modified_collection(LinkedData::Models::Metric) ont, sub = get_ontology_and_submission - ont = Ontology.find(params["ontology"]).first error 404, "Ontology #{params['ontology']} not found" unless ont sub.bring(ontology: [:acronym], metrics: LinkedData::Models::Metric.goo_attrs_to_load(includes_param)) reply sub.metrics || {} diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index fc15d567..006f982c 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -43,6 +43,25 @@ class OntologiesController < ApplicationController reply(latest || {}) end + # on demand ontology pull + post "/:acronym/pull" do + LOGGER.info "Forcing the pull and processing of ontology #{params['acronym']}" + actions = NcboCron::Models::OntologySubmissionParser::ACTIONS.dup + actions[:remote_pull] = true + ont = Ontology.find(params["acronym"]).first + error 404, "You must provide a valid `acronym` to retrieve an ontology" if ont.nil? + ont.bring(:acronym, :submissions, :viewingRestriction, :administeredBy) + check_write_access(ont) + latest = ont.latest_submission(status: :any) + error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil? + check_last_modified(latest) + latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + error 404, "Ontology #{params["acronym"]} is not configured to be remotely pulled" unless latest.remote_pulled? + NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions) + LOGGER.info "Ontology #{params['acronym']} has been queued for pull and processing" + halt 204 + end + ## # Create an ontology post do @@ -87,8 +106,8 @@ class OntologiesController < ApplicationController get '/:acronym/download' do acronym = params["acronym"] ont = Ontology.find(acronym).include(Ontology.goo_attrs_to_load).first - ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) error 422, "You must provide an existing `acronym` to download" if ont.nil? + ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) check_access(ont) restricted_download = LinkedData::OntologiesAPI.settings.restrict_download.include?(acronym) error 403, "License restrictions on download for #{acronym}" if restricted_download && !current_user.admin? diff --git a/controllers/ontology_analytics_controller.rb b/controllers/ontology_analytics_controller.rb index 5113c926..829708c1 100644 --- a/controllers/ontology_analytics_controller.rb +++ b/controllers/ontology_analytics_controller.rb @@ -7,6 +7,7 @@ class OntologyAnalyticsController < ApplicationController namespace "/analytics" do get do + expires 86400, :public year = year_param(params) error 400, "The year you supplied is invalid. Valid years start with 2 and contain 4 digits." if params["year"] && !year month = month_param(params) @@ -24,9 +25,14 @@ class OntologyAnalyticsController < ApplicationController namespace "/ontologies/:acronym/analytics" do get do + expires 86400, :public ont = Ontology.find(params["acronym"]).first error 404, "No ontology exists with the acronym: #{params["acronym"]}" if ont.nil? - analytics = ont.analytics + year = year_param(params) + error 400, "The year you supplied is invalid. Valid years start with 2 and contain 4 digits." if params["year"] && !year + month = month_param(params) + error 400, "The month you supplied is invalid. Valid months are 1-12." if params["month"] && !month + analytics = ont.analytics(year, month) if params["format"].to_s.downcase.eql?("csv") tf = Tempfile.new("analytics-#{params['acronym']}") @@ -37,7 +43,7 @@ class OntologyAnalyticsController < ApplicationController years.each do |year| months = analytics[params["acronym"]][year].keys.sort months.each do |month| - next if now.year == year && now.month <= month || (year == 2013 && month < 10) # we don't have good data going back past Oct 2013 + next if year && month && (now.year == year.to_i && now.month <= month.to_i || year.to_i == 2013 && month.to_i < 10) # we don't have good data going back past Oct 2013 visits = analytics[params["acronym"]][year][month] month = DateTime.parse("#{year}/#{month}").strftime("%b %Y") csv << [month, visits] diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index daa8bb66..3d1050af 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -19,9 +19,10 @@ class OntologySubmissionsController < ApplicationController ## # Display all submissions of an ontology get do - ont = Ontology.find(params["acronym"]).include(:acronym).first + ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) + check_access(ont) ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param)) reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId end @@ -38,6 +39,7 @@ class OntologySubmissionsController < ApplicationController # Display a submission get '/:ontology_submission_id' do ont = Ontology.find(params["acronym"]).include(:acronym).first + error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) ont.bring(:submissions) ont_submission = ont.submission(params["ontology_submission_id"]) @@ -93,15 +95,16 @@ class OntologySubmissionsController < ApplicationController submission_attributes = [:submissionId, :submissionStatus, :uploadFilePath, :pullLocation] included = Ontology.goo_attrs_to_load.concat([submissions: submission_attributes]) ont = Ontology.find(acronym).include(included).first - ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) error 422, "You must provide an existing `acronym` to download" if ont.nil? + ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) check_access(ont) ont_restrict_downloads = LinkedData::OntologiesAPI.settings.restrict_download error 403, "License restrictions on download for #{acronym}" if ont_restrict_downloads.include? acronym submission = ont.submission(params['ontology_submission_id'].to_i) error 404, "There is no such submission for download" if submission.nil? file_path = submission.uploadFilePath - + # handle edge case where uploadFilePath is not set + error 422, "Upload File Path is not set for this submission" if file_path.to_s.empty? download_format = params["download_format"].to_s.downcase allowed_formats = ["csv", "rdf"] if download_format.empty? diff --git a/controllers/slices_controller.rb b/controllers/slices_controller.rb index bc55eab1..04e0d53e 100644 --- a/controllers/slices_controller.rb +++ b/controllers/slices_controller.rb @@ -6,5 +6,65 @@ class SlicesController < ApplicationController includes = LinkedData::Models::Slice.goo_attrs_to_load(includes_param) reply LinkedData::Models::Slice.where.include(includes).all end + + get '/:slice_id' do + slice = LinkedData::Models::Slice.where(acronym: params["slice_id"]).first + error 404, "Slice #{params['slice_id']} not found" if slice.nil? + check_last_modified(slice) + slice.bring(*LinkedData::Models::Slice.goo_attrs_to_load(includes_param)) + reply slice + end + ## + # Create a new slice + post do + error 403, "Access denied" unless current_user && current_user.admin? + create_slice + end + + # Delete a slice + delete '/:slice' do + error 403, "Access denied" unless current_user && current_user.admin? + LinkedData::Models::Slice.find(params[:slice]).first.delete + halt 204 + end + + # Update an existing slice + patch '/:slice' do + error 403, "Access denied" unless current_user && current_user.admin? + slice = LinkedData::Models::Slice.find(params[:slice]).include(LinkedData::Models::Slice.attributes(:all)).first + populate_from_params(slice, params) + if slice.valid? + slice.save + else + error 422, slice.errors + end + halt 204 + end + + private + + def create_slice + params ||= @params + ontologies = [] + + params["ontologies"].each do |ont| + ontologies.push(LinkedData::Models::Ontology.find(ont).first) + end + + slice = LinkedData::Models::Slice.new({ + acronym: params["acronym"], + name: params["name"], + description: params["description"], + ontologies: ontologies + }) + + if slice.valid? + slice.save + else + error 400, slice.errors + end + reply 201, slice + end + end end \ No newline at end of file diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index b401ea8f..1a02190f 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -81,6 +81,7 @@ class UsersController < ApplicationController # Update an existing submission of an user patch '/:username' do user = User.find(params[:username]).include(User.attributes).first + params.delete("role") unless current_user.admin? populate_from_params(user, params) if user.valid? user.save @@ -92,6 +93,7 @@ class UsersController < ApplicationController # Delete a user delete '/:username' do + error 403, "Access denied" unless current_user.admin? User.find(params[:username]).first.delete halt 204 end @@ -109,6 +111,7 @@ def create_user params ||= @params user = User.find(params["username"]).first error 409, "User with username `#{params["username"]}` already exists" unless user.nil? + params.delete("role") unless current_user.admin? user = instance_from_params(User, params) if user.valid? user.save diff --git a/dip.yml b/dip.yml index c4c12684..c829068f 100644 --- a/dip.yml +++ b/dip.yml @@ -7,7 +7,7 @@ version: '7.1' compose: files: - - .docker-compose.yml + - docker-compose.yml project_name: ontologies_api interaction: @@ -34,14 +34,14 @@ interaction: # A shortcut to run unit tests test: - description: Run minitest unit tests + description: Run unit tests with 4store triplestore service: api - command: bundle exec rake test + command: bundle exec rake test TESTOPTS='-v' test-ag: - description: Run minitest unit tests + description: Run unit tests with AllegroGraph triplestore service: api-agraph - command: bundle exec rake test + command: bundle exec rake test TESTOPTS='-v' rackup: description: Run onotlogies_api service via rack up commands diff --git a/docker-compose.yml b/docker-compose.yml index de084081..2e552b91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,44 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_api:0.0.1 - environment: &env - BUNDLE_PATH: /srv/ontoportal/bundle - # default bundle config resolves to /usr/local/bundle/config inside of the container - # we are setting it to local app directory if we need to use 'bundle config local' - BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle - COVERAGE: 'true' - GOO_REDIS_HOST: redis-ut - REDIS_HOST: redis-ut - REDIS_PORT: 6379 - SOLR_HOST: solr-ut - SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 - MGREP_HOST: mgrep-ut - MGREP_PORT: 55555 - stdin_open: true - tty: true - command: "bundle exec rackup -o 0.0.0.0 --port 9393" - ports: - - 9393:9393 - volumes: - # bundle volume for hosting gems installed by bundle; it helps in local development with gem udpates - - bundle:/srv/ontoportal/bundle - # api code - - .:/srv/ontoportal/ontologies_api - # mount directory containing development version of the gems if you need to use 'bundle config local' - #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal - depends_on: - - solr-ut - - redis-ut - - mgrep-ut + build: + context: . + args: + RUBY_VERSION: '2.7' + # Increase the version number in the image tag every time Dockerfile or its arguments is changed + image: ontologies_api:0.0.4 + environment: &env + BUNDLE_PATH: /srv/ontoportal/bundle + # default bundle config resolves to /usr/local/bundle/config inside of the container + # we are setting it to local app directory if we need to use 'bundle config local' + BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle + COVERAGE: 'true' + GOO_REDIS_HOST: redis-ut + REDIS_GOO_CACHE_HOST: redis-ut + REDIS_HTTP_CACHE_HOST: redis-ut + REDIS_PERSISTENT_HOST: redis-ut + REDIS_PORT: 6379 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 + MGREP_HOST: mgrep-ut + MGREP_PORT: 55556 + stdin_open: true + tty: true + command: "bundle exec rackup -o 0.0.0.0 --port 9393" + ports: + - 9393:9393 + volumes: + # bundle volume for hosting gems installed by bundle; it helps in local development with gem udpates + - bundle:/srv/ontoportal/bundle + # api code + - .:/srv/ontoportal/ontologies_api + # mount directory containing development version of the gems if you need to use 'bundle config local' + #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal + depends_on: &depends_on + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + mgrep-ut: + condition: service_healthy services: api: @@ -50,10 +54,9 @@ services: profiles: - 4store depends_on: - - solr-ut - - redis-ut - - mgrep-ut - - 4store-ut + <<: *depends_on + 4store-ut: + condition: service_started api-agraph: <<: *app @@ -68,16 +71,21 @@ services: profiles: - agraph depends_on: - - solr-ut - - redis-ut - - mgrep-ut - - agraph-ut + <<: *depends_on + agraph-ut: + condition: service_healthy redis-ut: image: redis + healthcheck: + test: redis-cli ping + interval: 10s + timeout: 3s + retries: 10 4store-ut: image: bde2020/4store + platform: linux/amd64 #volume: fourstore:/var/lib/4store command: > bash -c "4s-backend-setup --segments 4 ontoportal_kb @@ -86,15 +94,28 @@ services: profiles: - 4store - solr-ut: - image: ontoportal/solr-ut:0.1 + image: ontoportal/solr-ut:0.0.2 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 mgrep-ut: - image: ontoportal/mgrep-ncbo:0.1 + image: ontoportal/mgrep:0.0.2 + platform: linux/amd64 + healthcheck: + test: ["CMD", "nc", "-z", "-v", "localhost", "55556"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 agraph-ut: - image: franzinc/agraph:v7.3.0 + image: franzinc/agraph:v8.0.0 + platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test - AGRAPH_SUPER_PASSWORD=xyzzy @@ -107,9 +128,14 @@ services: ; agtool users add anonymous ; agtool users grant anonymous root:bioportal_test:rw ; tail -f /agraph/data/agraph.log" + healthcheck: + test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"] + start_period: 10s + interval: 60s + timeout: 5s + retries: 3 profiles: - agraph volumes: bundle: - #fourstore: diff --git a/fix_purls.rb b/fix_purls.rb deleted file mode 100755 index 2f8f7a98..00000000 --- a/fix_purls.rb +++ /dev/null @@ -1,136 +0,0 @@ -require 'ontologies_linked_data' -require 'ncbo_annotator' -require_relative 'config/config' -require_relative 'config/environments/production' -# require_relative 'config/environments/stage' - - -purl_client = LinkedData::Purl::Client.new - - -# acronyms = ["AAO", "ABA-AMB", "ACGT-MO", "AI-RHEUM", "APAONTO", "ATC", "AURA", "BCGO", "BCTEO", "BHN", "BILA", -# "CANONT", "CCO", "CCON", "CHD", "CHEMBIO", "CLIN-EVAL", "CNO", "CO", "CO-WHEAT", "CPRO", "CPTH", "CRISP", -# "CTONT", "DC-CL", "DIAGONT", "DIKB", "DOCCC", "DSO", "DWC-TEST", "ELIG", "EPILONT", "EPSO", "ERO", "FB-SP", -# "FIRE", "FLO", "FLU", "GCC", "GENE-CDS", "GENETRIAL", "GEOSPECIES", "GEXO", "GLYCANONT", "GLYCOPROT", "GMO", -# "GPI", "GRO-CPD", "GRO-CPGA", "HCPCS-HIMC", "HIMC-CPT", "HINO", "HIV", "HOM-TEST", "I2B2-LOINC", "ICD0", -# "ICD09", "ICD11-BODYSYSTEM", "IDOMAL", "IEV", "IMMDIS", "IMR", "LHN", "LSM", "MCCL", "MIXSCV", "MPO", "MS", -# "MSTDE", "MSTDE-FRE", "NBO", "NCC", "NCCO", "NEOMARK3", "NEOMARK4", "NHSQI2009", "NIC", "NONRCTO", "OBOREL", -# "OBR-Scolio", "OFP", "OGSF", "OMIT", "ONTF", "ONTOAD", "ONTODM-CORE", "OntoOrpha", "ORTHO", "PHARMGKB", -# "PHENOMEBLAST", "PHENOSCAPE-EXT", "PHYFIELD", "PIERO", "PLATSTG", "PPIO", "PRO-ONT", "PROVO", "PSDS", -# "PSIMOD", "PTRANS", "RBSCI", "RCTONT", "RETO", "REXO", "RGMU", "RGMU2", "RNIMU", "SBRO", "SENTI", "STY", -# "TAXRANK", "TEST-PROD", "TEST1015", "TESTONTO", "TESTONTO2", "TESTONTOO", "TST-PAUL2", "TTTTHHHH", "TYPON", -# "UDEF", "UNITSONT", "VBCV", "WB-BT", "WB-LS", "WB-PHENOTYPE", "WH", "WSIO", "XEO", "ZIP3", "ZIP5"] - - -# acronyms = ["HOM-PROCS2.owl", "HOM-DXVCODES2.owl", "HOM-ICD9.owl", "HOM-PROCS_OSHPD.owl", "HOM-ICD9-PROCS_OSHPD.owl", -# "HOM-ICD9-ECODES_OSHPDowl", "HOM-ICD9_DXandVCODES_OSHPD.owl", "HOM-MDCDRG_OSHPD.owl", -# "HOM-ICD9-ECODES_OSHPD.owl", "HOM-PROCS2", "HOM-PROCS2", "HOM-DXVCODES2", "FBbt", "FBdv", "FAO", "EMAP", -# "ECO", "EV", "CL", "CHEBI", "DDANAT", "DOID", "MA", "BTO", "ProPreO", "MOD", "MI", "PW", "MPATH", "TO", -# "EO", "MFO", "MAO", "TGMA", "EHDA", "EHDAA", "MP", "FBbi", "CARO", "FBsp", "SOPHARM", "PRO", "SNPO", -# "ClinicalTrialOntology", "basic-vertebrate-gross-anatomy", "RID", "ZFA", "amino-acid", "WBbt", "WBls", -# "SEP", "SBO", "OBO_REL", "REX", "SPD", "birnlex", "MHC", "nif", "GRO", "TTO", "BSPO", "MIRO", "OCRe", "GO", -# "ENVO", "WBPhenotype", "TADS", "BOOTStrep", "PATO", "SO", "TAO", "UO", "YPO", "IDO", "TRANS", "XAO", "ATMO", -# "OGI", "ICD-9", "BRO", "DC_CL", "EP", "BPMetadata", "ECG", "RS", "DermLex", "VO", "MAT", "SPO", "BHO", "HP", -# "OBI", "MO", "FHHO", "OPB", "EFO", "NEMO"] - -# acronyms = ["HOM-PROCS2", "TADS", "HOM", "ABA", "CLKB", "CST", "ICPC", "BFO", "PEO", "SYMP", "SitBAC", "PLO", "APO", -# "MeGO", "gsontology", "NIF_Dysfunction", "ATO", "Field", "LDA", "GAZ", "SSO", "IAO", "LNC", "PDQ", "OMIM", -# "MEDLINEPLUS", "WHO", "SNOMEDCT", "NDFRT", "MSH", "PKO", "ICF", "KiSAO", "ICNP", "POL", "k2p", "NIF_Cell", -# "TOK", "RXNORM", "MDR", "OGMS", "CTCAE", "ICPC2P", "AIR", "BTC", "NDDF", "ICD10PCS", "MDDB", "RCD", "GFO", -# "GFO-Bio", "CHEMINF", "bioportal", "bioportal", "npo", "npo.owl", "bioportal.owl", "bro.owl", "BRO-Core", -# "bro", "MS", "EDAM", "EDAM", "HL7", "PD_ST", "JERM", "CLO", "NCBITaxon", "pma", "NCIM", "IMGT", "TMA", -# "bodysystem", "TMO", "ICECI", "MTHCH", "INO", "ICPS", "RNAO", "CPT", "ICD10", "FMA", "NCIt", "SIO", -# "SNOMED-Ethnic-Group", "SNOMEDCT-MAS", "BAO", "EHDAA2", "BO", "DDI", "NIGO", "RoleO", "HL7_UCSF_DSCHRG", -# "HL7_Dschrg", "HOM-ICD9"] - - -# acronyms = ["HOM-PROCS2/", "bioportal/", "bioportal", "npo/", "bro/", "MS/", "EDAM/", "apollo", "HOM_Discharge", -# "UCSF_TSI_Discharge", "HOM_ICD9", "HOM_ICD9", "SNOMED-Clinical-Findings", "SNOMEDCT-CF", "CO_Wheat", "REPO", -# "GALEN", "VAO", "ExO", "HPI", "vHOG", "UCare_Demographics", "UCare-Demographics", "TM-MER", -# "TM-SIGNS-AND-SYMPTS", "TM-OTHER-FACTORS", "TM-CONST", "obi-device", "OGMD", "FDA-MedDevice", "HOM-EHS", -# "PVOnto", "CELDAC", "PHARE", "1580", "AERO", "COA", "BP", "PO", "COR", "PR", "CMO", "XCO", "MMO", -# "Chem2Bio2OWL", "PRePPO", "PhylOnt", "IxnO", "BDO", "OntoDT", "OMRSE", "HLTH_INDICS", "FBcv", "HUGO", -# "CogPO", "ICD10CM", "NMR", "CSP", "OBOE", "oboe-sbc", "NeuMORE", "MCBCC", "LiPrO", "NeoMarkOntology", -# "ACGT", "BT", "CDAO", "CPTAC", "OGR", "ODGI", "CBO", "CPR", "pseudo", "SAO", "ZEA", "UBERON", "fged-obi", -# "obi-fged", "ICD9_PROCEDURES", "OPL", "HOMERUN_UHC", "BiositemapIM", "BRO-Activity", "BRO-AreaOfResearch", -# "BRO-Chinese", "cogat", "EMO", "HC", "OntoDM", "NCBI_NMOsp", "invertebrata", "VT", "IDOBRU"] - -# acronyms = ["HOM-DATASOURCE_OSHPD", "EPIC-SRC", "HOM-EPIC", "HOMERUN", "HOM-TX", "HPIO", "SHR", "MFOEM", "PedTerm", -# "AEO", "OAE", "FYPO", "HOM-CPT", "SYN", "DATASRC-ORTHO", "HOM-ORTHO", "vivo", "OPB.properties", -# "proftest.4205", "PO_PAE", "PO_PSDS", "GRO_CPD", "GRO_CPGA", "VSAO", "RH-MESH", "BAO-GPCR", "FMA-SUBSET", -# "COSTART", "ORDO", "FB-CV", "cms.36739", "chi.76398", "chi.41353", "I9I10CMMOST", "project", "DTVPrecision", -# "bibliographic", "dcterms", "foaf", "SciRes", "BRIDG", "ICD9CM-PROC", "CPT-mod", "SWEET", -# "cms.29622", "HOM-CHI", "nhds.7763", "NIF-RTH", "shrine.20924", "IDODEN", "gs1", "MEDO", "IFAR", "PathLex", -# "journal-test", "testvdb", "UCSF-ICD910CM", "Pizza-Example", "OPE", "NCBO_TEST_NOTES", "CABRO", "MTHMSTFRE", -# "GCO", "pomoc", "bco", "OVAE", "GOWGP_B", "BOF", "RNPRIO", "ONL-DP", "SSE", "IMGT-ONTOLOGY", "InterNano", -# "OBI_BCGO", "suicideo", "CARD", "MISHA_TEST2", "TEST-ONT-0-0", "MSV", "MATO", "STATO", "ELIXHAUSER", -# "SNOMED_TEST", "BWT", "ZC", "AI", "NIFCELL", "NIFDYS", "MESH"] - -# acronyms = ["OBCS", "FLOPO", "HUPSON", "NMOSP_1_6", "NMOSP", "CCC", "DEMOGRAPH", "GLOB", "SNOMED_ANATOMY", "SNOMED_ORG", -# "SNOMED_CF", "ICO", "BWT2", "NGSONTO", "HIVO004", "OGG", "TESTING", "RCTOntology", "CEEROntology", -# "dikb-evidence", "ICDO3", "QIBO", "HOM-OSHPD-SC", "SRC-OSHPDSC", "test_pradip_purl", "NDF-RT", "envo_153", -# "HOM-ORTHOSURG", "eufut", "MCV", "OBI_IEDB_view", "HOM-I9I10MAPS", "MF", "VANDF", "GWAS_EFO_SKOS", "NatPrO", -# "HSDB_OCRe", "test-111", "OoEVV", "HOM-ICU", "CareLex", "DiagnosticOnt", "Phys_Med_Rehab", "PMR", -# "ICD9toICD10PCS", "HOM-DXandVCODES_OSHPD", "HOM_HARVARD", "HOM-PROCS_OSHPD", "HOM-ICD9CM-ECODES", -# "HOM-ICD9CM_PROCEDURES", "HOM-MDCDRG_OSHPD", "HOM_MDCs_DRGs", "HOM_EHS", "HOM-PCSTEST", "MESH-OWL", "PTSD", -# "HOM-UPENN-MEDS", "HOM-UPENN_MEDS", "SDO", "SOY", "SPTO", "CanCO", "CTX", "QUDT", "HOM-UPENNMEDS", "WHOFRE", -# "ICPCFRE", "MDRFRE", "thesaurus", "ontologia", "thealternativa", "SNMD_BDY", "SWO", "ICF-NoCodeLabel", -# "testabbr", "bccl", "bccl1", "HOM-MEDABBS", "HOM-TESTKM", "HOM-I9CM", "HOM-MDCDRG", "NonRCTOntology", -# "HOM-DEMOGR"] - - -# acronyms = ["uni-ece", "PFO", "PhenXTK", "HCPCS", "test.20230", "test.20930", "HOM-DATSRCTEST", "i2b2-patvisdims", -# "homv2", "HOM-EPICI2B2", "ICD9CM", "TAHH", "TAHE", "HOM-I9PCS", "CAO", "HOM-UCARE", "EpilepOnto", -# "HOM-DXPCS_MDCDRG", "HOM-PCS_OSHPD", "HOM-I9-ECODES", "HOM-VCODES_OSHPD", "HOM-SRCE_OSHPD", "UnitsOntology", -# "HOM-OSHPD_UseCas", "pharmgkb-owl", "phenomeblast-owl", "CAMRQ", "tJADNI2", "BIOA3", "TestHDB", "tJADNI", -# "BIOA", "UCBH", "BIOA2", "NeoMark", "MEO", "test-purl", "yipd", "BioModels", "CPO", "PhyMeRe", "onto", -# "TEO", "test-1", "ADW", "HOM-OSHPD", "test.13773", "test.16355", "profectus.test.installation.21838", -# "profectus.test.installation.21775", "profectus.test.installation.21934", -# "profectus.test.installation.22027", "profectus.test.installation.21975", -# "profectus.test.installation.22081", "profectus.test.installation.22611", -# "profectus.test.installation.22704", "profectus.test.installation.22770", -# "OntoDM-KDD"] - -# acronyms = ["PhenX", "HOMICD910PCS-150", "dsfs", "HOM-I910TESTPLUS", "DwC", "HOM-DATSRCTESTo", "HOM-DATASRCTESTn", -# "HOM-I910PCS-150", "HOM-ORTHOTEST", "test.8300", "HOM-OCHILDTEST", "OntoMA", "HOM-CLINIC", "DwC_test", -# "UCSF_15", "UCSF_41", "profectus.deid.installation.27982", "profectus.deid.installation.28036", -# "profectus.deid.installation.28409", "profectus.deid.installation.32434", -# "profectus.deid.installation.21442", "HOM-OCC", "profectus.deid.installation.7616", -# "profectus.deid.installation.6250", "DC_test", "CCONT", "profectus.deid.installation.14032", -# "profectus.deid.installation.14522", "profectus.deid.installation.19214", "BIOA3v", -# "profectus.deid.installation.3433", "profectus.deid.installation.16341", "RPO", -# "profectus.deid.installation.11714", "profectus.deid.installation.19037", -# "profectus.deid.installation.27298", "profectus.deid.installation.31807", "OBIws", "ooevv-tractTrace", -# "ooevv-vaccine", "pco", "VSO", "profectus.oshpd.installation.5931", "NIF-Subcell", "DwC_translations", -# "PabloTest", "ConsentOntology", "pablotest2", "prov-o", "oshpd.39537", "nhds.13469", "nhds.13896", -# "HOM-NHDS", "ONSTR", "miRNAO", "chi.58970", "chi.61020", "chi.76831", "HOM-CMS", "Clinical_Eval", -# "documentStatus", "geopolitical", "citation", "dcelements", "event", "FaBiO", "provenance", "skos", -# "chi.20896", "NTDO", "oshpd.33038", "OntoKBCF", "HCPCS-mod", "chi.63559", "VariO", "NIAID-GSC-BRC", -# "cms.11830", "HOM-CHICMS", "nhds.7622", "HOM-GLOB", "jnnn", "GlycO"] - -# acronyms = ["shrine.20839", "shrine.20676", "chi.50064", "MetaCT", "ThomCan", "Genomic-CDS", "OntoPneumo", "bd-test2", -# "avnguyen-test2", "Top-Menelas", "MCCV", "SemPhysKB-Human", "ZIP", "biocode", "PORO", "ATOL", "IDQA", -# "TRAK", "UCSD", "mixs", "bao_gpcr", "TrOn", "MTHMST", "DermaO", "rsa", "DermO", "ECGT", "UCSFI9I10ALL", -# "SBOLv", "DILIo", "Eligibility", "CHDWiki", "BOFrdf", "VTO", "PDO", "CSSO", "NHSQI", "SEDI", "SuicidO", -# "ONL-MSA", "EDDA", "ONL-MR-DA", "ADO", "OntoVIP", "CHMO", "BdOK", "BOFf", "SBOL", "RadLex_v3.91", -# "STNFRDRXDEMO", "OntoBioUSP", "Radlex3.9.1", "BNO", "DCO", "PROV", "BSAO", "HRDO", "SNMI", "FIX"] - -LinkedData::Models::Ontology.all.each do |ont| - ont.bring(:acronym) - acronym = ont.acronym - - if (purl_client.purl_exists(acronym)) - puts "#{acronym} exists" - purl_client.fix_purl(acronym) - else - puts "#{acronym} DOES NOT exist" - purl_client.create_purl(acronym) - end -end - - - - -# acronyms.each do |acronym| -# purl_client.fix_purl(acronym) -# end diff --git a/helpers/access_control_helper.rb b/helpers/access_control_helper.rb index 1de3bee5..071d3afe 100644 --- a/helpers/access_control_helper.rb +++ b/helpers/access_control_helper.rb @@ -6,15 +6,12 @@ module AccessControlHelper ## # For a given object, check the access control settings. If they are restricted, handle appropriately. - # For a list, this will filter out results. For single objects, if will throw an error if access is denied. + # For a list, this will filter out results. + # For single objects, if will throw an error if access is denied. def check_access(obj) return obj unless LinkedData.settings.enable_security if obj.is_a?(Enumerable) - if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on? - check_access(obj.first) - else filter_access(obj) - end else if obj.respond_to?(:read_restricted?) && obj.read_restricted? readable = obj.readable?(env["REMOTE_USER"]) @@ -28,7 +25,7 @@ def check_access(obj) # For a given object, check if the current user has permission to perform writes. def check_write_access(obj) return obj unless LinkedData.settings.enable_security - if obj.is_a?(LinkedData::Models::Base) && obj.write_restricted? + if obj.is_a?(LinkedData::Models::Base) writable = obj.writable?(env["REMOTE_USER"]) error 403, "Access denied for this resource" unless writable end diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 18163099..b7d26990 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -261,9 +261,10 @@ def month_param(params=nil) if params["month"] month = params["month"].strip if %r{(?^(0[1-9]|[1-9]|1[0-2])$)}x === month - month.to_i + return month.to_i.to_s end end + nil end # validates year for starting with 1 or 2 and containing 4 digits @@ -272,9 +273,10 @@ def year_param(params=nil) if params["year"] year = params["year"].strip if %r{(?^([1-2]\d{3})$)}x === year - year.to_i + return year.to_i.to_s end end + nil end ## diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 43ff9fca..7734efa8 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -4,7 +4,6 @@ #docker-compose build #docker-compose up --exit-code-from unit-test -# wait-for-it is useful since solr container might not get ready quick enough for the unit tests -docker-compose run --rm api wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v' -#docker-compose run --rm api wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v' TEST='./test/controllers/test_annotator_controller.rb' -docker-compose down +docker-compose run --rm api bundle exec rake test TESTOPTS='-v' +#docker-compose run --rm apibundle exec rake test TESTOPTS='-v' TEST='./test/controllers/test_annotator_controller.rb' +docker-compose stop diff --git a/search_index.rb b/search_index.rb deleted file mode 100755 index 6f696aa0..00000000 --- a/search_index.rb +++ /dev/null @@ -1,38 +0,0 @@ -require 'logger' -require 'ontologies_linked_data' -require 'ncbo_annotator' -require_relative 'config/config' -#require_relative 'config/environments/production' -require_relative 'config/environments/stage' - -logger = Logger.new("indexing.log") - -# clear the index -logger.info("Clearing existing index.") -LinkedData::Models::Class.indexClear() -LinkedData::Models::Class.indexCommit() - -only_index = ["TAO", "VHOG", "QIBO", "SDO", "XAO", "GLOB", "DEMOGRAPH", "BHO", "FMA-SUBSET", "RCD"] -# only_index = ["FMA-SUBSET"] -# only_index = [] - -logger.info("Began indexing ontologies...") -submissions = LinkedData::Models::OntologySubmission.where(submissionStatus: [code: "RDF"]).include(:submissionId, ontology: LinkedData::Models::Ontology.attributes).all - -submissions.each do |s| - if only_index.empty? || only_index.include?(s.ontology.acronym) - begin - s.process_submission(logger, - process_rdf: false, - index_search: true, index_commit: true, - run_metrics: false, reasoning: false) - rescue Exception => e - logger.error e - end - end -end - -logger.info("Completed indexing ontologies.") -logger.info("Optimizing index...") -LinkedData::Models::Class.indexOptimize() -logger.info("Completed optimizing index.") diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4713b699..da8c6c11 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -1,3 +1,4 @@ +require 'webrick' require_relative '../test_case' class TestOntologiesController < TestCase @@ -29,6 +30,8 @@ def self._set_vars hasOntologyLanguage: "OWL", administeredBy: ["tom"] } + @@server_thread = nil + @@server_url = nil end def self._create_user @@ -253,9 +256,68 @@ def test_download_acl_only end end + def test_on_demand_ontology_pull + ont = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true)[2].first + ont.bring_remaining + acronym = ont.acronym + sub = ont.submissions.first + sub.bring(:pullLocation) if sub.bring?(:pullLocation) + assert_equal(nil, sub.pullLocation, msg="Pull location should be nil at this point in the test") + + allowed_user = ont.administeredBy.first + allowed_user.bring(:apikey) if allowed_user.bring?(:apikey) + + post "/ontologies/#{acronym}/pull?apikey=#{allowed_user.apikey}" + assert_equal(404, last_response.status, msg="This ontology is NOT configured to be remotely pulled at this point in the test. It should return status 404") + + begin + start_server + sub.pullLocation = RDF::IRI.new(@@server_url) + sub.save + LinkedData.settings.enable_security = true + post "/ontologies/#{acronym}/pull?apikey=#{allowed_user.apikey}" + assert_equal(204, last_response.status, msg="The ontology admin was unable to execute the on-demand pull") + + blocked_user = User.new({ + username: "blocked", + email: "test@example.org", + password: "12345" + }) + blocked_user.save + post "/ontologies/#{acronym}/pull?apikey=#{blocked_user.apikey}" + assert_equal(403, last_response.status, msg="An unauthorized user was able to execute the on-demand pull") + ensure + stop_server + LinkedData.settings.enable_security = false + end + end private + def start_server + ont_path = File.expand_path("../../data/ontology_files/BRO_v3.2.owl", __FILE__) + file = File.new(ont_path) + port = Random.rand(55000..65535) # http://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports + @@server_url = "http://localhost:#{port}/" + @@server_thread = Thread.new do + server = WEBrick::HTTPServer.new(Port: port) + server.mount_proc '/' do |req, res| + contents = file.read + file.rewind + res.body = contents + end + begin + server.start + ensure + server.shutdown + end + end + end + + def stop_server + Thread.kill(@@server_thread) if @@server_thread + end + def check400(response) assert response.status >= 400 assert MultiJson.load(response.body)["errors"] diff --git a/test/controllers/test_ontology_analytics_controller.rb b/test/controllers/test_ontology_analytics_controller.rb index 67ab5529..b8e36dce 100644 --- a/test/controllers/test_ontology_analytics_controller.rb +++ b/test/controllers/test_ontology_analytics_controller.rb @@ -3,195 +3,195 @@ class TestOntologyAnalyticsController < TestCase ANALYTICS_DATA = { "NCIT" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 9 => 0, 10 => 2850, 11 => 1631, 12 => 1323 + "2013" => { + "1" => 0, "2" => 0, "3" => 0, "4" => 0, "5" => 0, "6" => 0, "7" => 0, "8" => 0, "9" => 0, "10" => 2850, "11" => 1631, "12" => 1323 }, - 2014 => { - 1 => 1004, 2 => 1302, 3 => 2183, 4 => 2191, 5 => 1005, 6 => 1046, 7 => 1261, 8 => 1329, 9 => 1100, 10 => 956, 11 => 1105, 12 => 893 + "2014" => { + "1" => 1004, "2" => 1302, "3" => 2183, "4" => 2191, "5" => 1005, "6" => 1046, "7" => 1261, "8" => 1329, "9" => 1100, "10" => 956, "11" => 1105, "12" => 893 }, - 2015 => { - 1 => 840, 2 => 30, 3 => 50, 4 => 20, 5 => 30, 6 => 10, 7 => 100, 8 => 80, 9 => 20, 10 => 90, 11 => 200, 12 => 50 + "2015" => { + "1" => 840, "2" => 30, "3" => 50, "4" => 20, "5" => 30, "6" => 10, "7" => 100, "8" => 80, "9" => 20, "10" => 90, "11" => 200, "12" => 50 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 730 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 730 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 730 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 730 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 730 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 730 } }, "ONTOMA" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 9 => 0, 10 => 6, 11 => 15, 12 => 0 + "2013" => { + "1" => 0, "2" => 0, "3" => 0, "4" => 0, "5" => 0, "6" => 0, "7" => 0, "8" => 0, "9" => 0, "10" => 6, "11" => 15, "12" => 0 }, - 2014 => { - 1 => 2, 2 => 0, 3 => 0, 4 => 2, 5 => 2, 6 => 0, 7 => 6, 8 => 8, 9 => 0, 10 => 0, 11 => 0, 12 => 2 + "2014" => { + "1" => 2, "2" => 0, "3" => 0, "4" => 2, "5" => 2, "6" => 0, "7" => 6, "8" => 8, "9" => 0, "10" => 0, "11" => 0, "12" => 2 }, - 2015 => { - 1 => 30, 2 => 90, 3 => 90, 4 => 50, 5 => 30, 6 => 20, 7 => 80, 8 => 90, 9 => 250, 10 => 230, 11 => 120, 12 => 70 + "2015" => { + "1" => 30, "2" => 90, "3" => 90, "4" => 50, "5" => 30, "6" => 20, "7" => 80, "8" => 90, "9" => 250, "10" => 230, "11" => 120, "12" => 70 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 730 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 730 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 730 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 730 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 730 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 730 } }, "CMPO" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 9 => 0, 10 => 64, 11 => 75, 12 => 22 + "2013" => { + "1" => 0, "2" => 0, "3" => 0, "4" => 0, "5" => 0, "6" => 0, "7" => 0, "8" => 0, "9" => 0, "10" => 64, "11" => 75, "12" => 22 }, - 2014 => { - 1 => 15, 2 => 15, 3 => 19, 4 => 12, 5 => 13, 6 => 14, 7 => 22, 8 => 12, 9 => 36, 10 => 6, 11 => 8, 12 => 10 + "2014" => { + "1" => 15, "2" => 15, "3" => 19, "4" => 12, "5" => 13, "6" => 14, "7" => 22, "8" => 12, "9" => 36, "10" => 6, "11" => 8, "12" => 10 }, - 2015 => { - 1 => 7, 2 => 40, 3 => 140, 4 => 320, 5 => 560, 6 => 320, 7 => 210, 8 => 230, 9 => 220, 10 => 10, 11 => 220, 12 => 880 + "2015" => { + "1" => 7, "2" => 40, "3" => 140, "4" => 320, "5" => 560, "6" => 320, "7" => 210, "8" => 230, "9" => 220, "10" => 10, "11" => 220, "12" => 880 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 30 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 30 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 430 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 430 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 830 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 830 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 380 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 380 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 794 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 794 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 738 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 738 } }, "AEO" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 9 => 0, 10 => 129, 11 => 142, 12 => 70 + "2013" => { + "1" => 0, "2" => 0, "3" => 0, "4" => 0, "5" => 0, "6" => 0, "7" => 0, "8" => 0, "9" => 0, "10" => 129, "11" => 142, "12" => 70 }, - 2014 => { - 1 => 116, 2 => 93, 3 => 85, 4 => 132, 5 => 96, 6 => 137, 7 => 69, 8 => 158, 9 => 123, 10 => 221, 11 => 163, 12 => 43 + "2014" => { + "1" => 116, "2" => 93, "3" => 85, "4" => 132, "5" => 96, "6" => 137, "7" => 69, "8" => 158, "9" => 123, "10" => 221, "11" => 163, "12" => 43 }, - 2015 => { - 1 => 25, 2 => 230, 3 => 330, 4 => 220, 5 => 650, 6 => 320, 7 => 840, 8 => 440, 9 => 220, 10 => 110, 11 => 210, 12 => 270 + "2015" => { + "1" => 25, "2" => 230, "3" => 330, "4" => 220, "5" => 650, "6" => 320, "7" => 840, "8" => 440, "9" => 220, "10" => 110, "11" => 210, "12" => 270 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 335 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 335 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 732 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 732 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 734 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 734 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 790 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 790 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 350 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 350 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 730 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 730 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 730 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 730 } }, "SNOMEDCT" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 9 => 0, 10 => 20721, 11 => 22717, 12 => 18565 + "2013" => { + "1" => 0, "2" => 0, "3" => 0, "4" => 0, "5" => 0, "6" => 0, "7" => 0, "8" => 0, "9" => 0, "10" => 20721, "11" => 22717, "12" => 18565 }, - 2014 => { - 1 => 17966, 2 => 17212, 3 => 20942, 4 => 20376, 5 => 21063, 6 => 18734, 7 => 18116, 8 => 18676, 9 => 15728, 10 => 16348, 11 => 13933, 12 => 9533 + "2014" => { + "1" => 17966, "2" => 17212, "3" => 20942, "4" => 20376, "5" => 21063, "6" => 18734, "7" => 18116, "8" => 18676, "9" => 15728, "10" => 16348, "11" => 13933, "12" => 9533 }, - 2015 => { - 1 => 9036, 2 => 430, 3 => 550, 4 => 110, 5 => 990, 6 => 320, 7 => 630, 8 => 250, 9 => 270, 10 => 880, 11 => 330, 12 => 280 + "2015" => { + "1" => 9036, "2" => 430, "3" => 550, "4" => 110, "5" => 990, "6" => 320, "7" => 630, "8" => 250, "9" => 270, "10" => 880, "11" => 330, "12" => 280 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 230 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 230 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 130 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 130 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 330 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 330 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 920 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 920 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 130 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 130 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 930 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 930 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 230 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 230 } }, "TST" => { - 2013 => { - 1 => 0, 2 => 0, 3 => 23, 4 => 0, 5 => 0, 6 => 0, 7 => 20, 8 => 0, 9 => 0, 10 => 234, 11 => 7654, 12 => 2311 + "2013" => { + "1" => 0, "2" => 0, "3" => 23, "4" => 0, "5" => 0, "6" => 0, "7" => 20, "8" => 0, "9" => 0, "10" => 234, "11" => 7654, "12" => 2311 }, - 2014 => { - 1 => 39383, 2 => 239, 3 => 40273, 4 => 3232, 5 => 2, 6 => 58734, 7 => 11236, 8 => 23, 9 => 867, 10 => 232, 11 => 1111, 12 => 8 + "2014" => { + "1" => 39383, "2" => 239, "3" => 40273, "4" => 3232, "5" => 2, "6" => 58734, "7" => 11236, "8" => 23, "9" => 867, "10" => 232, "11" => 1111, "12" => 8 }, - 2015 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 710 + "2015" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 710 }, - 2016 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 520, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 720 + "2016" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 520, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 720 }, - 2017 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 220, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 170, 11 => 750, 12 => 73 + "2017" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 220, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 170, "11" => 750, "12" => 73 }, - 2018 => { - 1 => 2000, 2 => 220, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 120, 11 => 750, 12 => 60 + "2018" => { + "1" => 2000, "2" => 220, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 120, "11" => 750, "12" => 60 }, - 2019 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 70 + "2019" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 70 }, - 2020 => { - 1 => 2000, 2 => 210, 3 => 640, 4 => 540, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 790 + "2020" => { + "1" => 2000, "2" => 210, "3" => 640, "4" => 540, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 790 }, - 2021 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 550, 5 => 270, 6 => 980, 7 => 340, 8 => 320, 9 => 630, 10 => 110, 11 => 750, 12 => 70 + "2021" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 550, "5" => 270, "6" => 980, "7" => 340, "8" => 320, "9" => 630, "10" => 110, "11" => 750, "12" => 70 }, - 2022 => { - 1 => 2000, 2 => 230, 3 => 640, 4 => 540, 5 => 270, 6 => 990, 7 => 340, 8 => 320, 9 => 610, 10 => 180, 11 => 750, 12 => 30 + "2022" => { + "1" => 2000, "2" => 230, "3" => 640, "4" => 540, "5" => 270, "6" => 990, "7" => 340, "8" => 320, "9" => 610, "10" => 180, "11" => 750, "12" => 30 } } } diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 0098969e..40532cd0 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -125,6 +125,10 @@ def test_download_submission # Clear restrictions on downloads LinkedData::OntologiesAPI.settings.restrict_download = [] # see also test_ontologies_controller::test_download_ontology + + # Test downloads of nonexistent ontology + get "/ontologies/BOGUS66/submissions/55/download" + assert_equal(422, last_response.status, "failed to handle downloads of nonexistent ontology" + get_errors(last_response)) end def test_download_ontology_submission_rdf @@ -188,4 +192,48 @@ def test_download_acl_only end end + def test_ontology_submissions_access_controller + count, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 2, submission_count: 1, process_submission: false) + # case first submission is private + acronym = created_ont_acronyms.first + ont = onts.first.bring_remaining + + begin + allowed_user = User.new({ + username: "allowed", + email: "test@example.org", + password: "12345" + }) + allowed_user.save + blocked_user = User.new({ + username: "blocked", + email: "test@example.org", + password: "12345" + }) + blocked_user.save + + ont.acl = [allowed_user] + ont.viewingRestriction = "private" + ont.save + + LinkedData.settings.enable_security = true + + get "/submissions?apikey=#{allowed_user.apikey}" + assert_equal 200, last_response.status + submissions = MultiJson.load(last_response.body) + assert_equal 2, submissions.size + + get "/submissions?apikey=#{blocked_user.apikey}" + assert_equal 200, last_response.status + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + ensure + LinkedData.settings.enable_security = false + del = User.find("allowed").first + del.delete if del + del = User.find("blocked").first + del.delete if del + end + end + end diff --git a/test/controllers/test_slices_controller.rb b/test/controllers/test_slices_controller.rb index 92ce6b1d..308b9fb7 100644 --- a/test/controllers/test_slices_controller.rb +++ b/test/controllers/test_slices_controller.rb @@ -3,28 +3,77 @@ class TestSlicesController < TestCase def self.before_suite - onts = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 0)[2] + ont_count, ont_acronyms, @@onts = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 0) @@slice_acronyms = ["tst-a", "tst-b"].sort - _create_slice(@@slice_acronyms[0], "Test Slice A", onts) - _create_slice(@@slice_acronyms[1], "Test Slice B", onts) + _create_slice(@@slice_acronyms[0], "Test Slice A", @@onts) + _create_slice(@@slice_acronyms[1], "Test Slice B", @@onts) + + @@user = User.new({ + username: "test-slice", + email: "test-slice@example.org", + password: "12345" + }).save + @@new_slice_data = { acronym: 'tst-c', name: "Test Slice C", ontologies: ont_acronyms} + enable_security + end + + def self.after_suite + LinkedData::Models::Slice.all.each(&:delete) + @@user.delete + reset_security + end + + def setup + self.class.reset_security + self.class.reset_to_not_admin(@@user) + LinkedData::Models::Slice.find(@@new_slice_data[:acronym]).first&.delete end def test_all_slices get "/slices" assert last_response.ok? slices = MultiJson.load(last_response.body) - assert_equal @@slice_acronyms, slices.map {|s| s["acronym"]}.sort + assert_equal @@slice_acronyms, slices.map { |s| s["acronym"] }.sort + end + + def test_create_slices + self.class.enable_security + + post "/slices?apikey=#{@@user.apikey}", MultiJson.dump(@@new_slice_data), "CONTENT_TYPE" => "application/json" + assert_equal 403, last_response.status + + self.class.make_admin(@@user) + + post "/slices?apikey=#{@@user.apikey}", MultiJson.dump(@@new_slice_data), "CONTENT_TYPE" => "application/json" + + assert 201, last_response.status + end + + def test_delete_slices + self.class.enable_security + LinkedData.settings.enable_security = @@old_security_setting + self.class._create_slice(@@new_slice_data[:acronym], @@new_slice_data[:name], @@onts) + + + delete "/slices/#{@@new_slice_data[:acronym]}?apikey=#{@@user.apikey}" + assert_equal 403, last_response.status + + self.class.make_admin(@@user) + + delete "/slices/#{@@new_slice_data[:acronym]}?apikey=#{@@user.apikey}" + assert 201, last_response.status end private def self._create_slice(acronym, name, ontologies) slice = LinkedData::Models::Slice.new({ - acronym: acronym, - name: "Test #{name}", - ontologies: ontologies - }) + acronym: acronym, + name: "Test #{name}", + ontologies: ontologies + }) slice.save end + end diff --git a/test/controllers/test_users_controller.rb b/test/controllers/test_users_controller.rb index 337da52e..6064132e 100644 --- a/test/controllers/test_users_controller.rb +++ b/test/controllers/test_users_controller.rb @@ -6,7 +6,7 @@ def self.before_suite @@usernames = %w(fred goerge henry ben mark matt charlie) # Create them again - @@usernames.each do |username| + @@users = @@usernames.map do |username| User.new(username: username, email: "#{username}@example.org", password: "pass_word").save end @@ -21,6 +21,17 @@ def self._delete_users end end + def test_admin_creation + existent_user = @@users.first #no admin + + refute _create_admin_user(apikey: existent_user.apikey), "A no admin user can't create an admin user or update it to an admin" + + existent_user = self.class.make_admin(existent_user) + assert _create_admin_user(apikey: existent_user.apikey), "Admin can create an admin user or update it to be an admin" + self.class.reset_to_not_admin(existent_user) + _delete_user(@@username) + end + def test_all_users get '/users' assert last_response.ok? @@ -48,7 +59,8 @@ def test_create_new_user assert last_response.ok? assert MultiJson.load(last_response.body)["username"].eql?(@@username) - delete created_user["@id"] + _delete_user(created_user["username"]) + post "/users", MultiJson.dump(user.merge(username: @@username)), "CONTENT_TYPE" => "application/json" assert last_response.status == 201 assert MultiJson.load(last_response.body)["username"].eql?(@@username) @@ -79,13 +91,21 @@ def test_update_patch_user end def test_delete_user - delete "/users/ben" - assert last_response.status == 204 + self.class.enable_security + + delete "/users/ben?apikey=#{@@users.first.apikey}" + assert_equal 403, last_response.status + + self.class.make_admin(@@users.first) + delete "/users/ben?apikey=#{@@users.first.apikey}" + assert_equal 204, last_response.status @@usernames.delete("ben") + self.class.reset_security + self.class.reset_to_not_admin(@@users.first) get "/users/ben" - assert last_response.status == 404 + assert_equal 404, last_response.status end def test_user_not_found @@ -100,4 +120,36 @@ def test_authentication assert user["username"].eql?(@@usernames.first) end + + private + + def _delete_user(username) + LinkedData::Models::User.find(@@username).first&.delete + end + def _create_admin_user(apikey: nil) + user = {email: "#{@@username}@example.org", password: "pass_the_word", role: ['ADMINISTRATOR']} + _delete_user(@@username) + + put "/users/#{@@username}", MultiJson.dump(user), "CONTENT_TYPE" => "application/json", "Authorization" => "apikey token=#{apikey}" + assert last_response.status == 201 + created_user = MultiJson.load(last_response.body) + assert created_user["username"].eql?(@@username) + + get "/users/#{@@username}?apikey=#{apikey}" + assert last_response.ok? + user = MultiJson.load(last_response.body) + assert user["username"].eql?(@@username) + + return true if user["role"].eql?(['ADMINISTRATOR']) + + patch "/users/#{@@username}", MultiJson.dump(role: ['ADMINISTRATOR']), "CONTENT_TYPE" => "application/json", "Authorization" => "apikey token=#{apikey}" + assert last_response.status == 204 + + get "/users/#{@@username}?apikey=#{apikey}" + assert last_response.ok? + user = MultiJson.load(last_response.body) + assert user["username"].eql?(@@username) + + true if user["role"].eql?(['ADMINISTRATOR']) + end end diff --git a/test/data/dictionary.txt b/test/data/dictionary.txt new file mode 100644 index 00000000..8fda643e --- /dev/null +++ b/test/data/dictionary.txt @@ -0,0 +1,487 @@ +4756335899466655033 EXPERIMENTAL PROTOCOL +-774463963537217159 CODE PROFILER +4447791899527133840 GENE THERAPY FACILITY +8459621249258197215 MONOCLONAL ANTIBODY FACILITY +-6785780030486581682 TECHNICAL SUPPORT +3232659116104897312 DISSEMINATION VEHICLE +7828929368629805534 IMAGE ALGORITHM +-6560018995977105782 MOLECULAR VISUALIZATION +5793220988038836234 CALENDAR SCHEDULE AND RESOURCE MANAGEMENT +-6517106492820232063 STATISTICAL ALGORITHM +3384577157043768072 PROTEOMICS +5256624978246529952 IMAGE PROCESSING +7739598960357869921 FLATFILE DISTRIBUTION +-8849988027318638467 BIOINFORMATICS +2127697897814542247 DATA TRANSFER AND COMMUNICATION +7224675013892117270 PHYSIOLOGY FACILITY +8158733784858426335 WEBSITE +-816017611189475787 CT SCANNER +-4064194439757350154 PRECLINICAL +2171827417877078723 AUTHENTICATION +2504247712508322234 STRUCTURE-BASED PROTEIN CLASSIFICATION +-5132706058848658492 WEBSITE HOSTING +-2276929190164417676 MESHING +-1726033975408720031 MODEL WITH CLOSED FORM SOLUTIONS +8088391824786644848 BIOSTATISTICS +-332939505887504185 PASS THROUGH LICENSE +-3346428803780558892 IRB +8673679222785859230 GRAPH ANALYSIS +-4091631528644994336 MAILING LIST +-8801480136462875937 NUMERICAL CALCULATION OF ELECTROSTATIC POTENTIAL +-8832279635634013935 VISUALIZATION +-2139953831711188397 DEPRECATED AREA OF RESEARCH +3696202828817838769 NETWORK CHARACTERIZATION +-1572042243049590518 PROTEIN-PROTEIN INTERACTION +8602984357580688165 CROSS-LANGUAGE WRAPPING +-3714358777107137746 INFORMATION RESOURCE +-4388973080159728420 DISCRETE DATA +-578849145746211761 BIOCHEMICAL SUPPLY RESOURCE +7536156830284634138 GRID COMPUTING +691558772552310682 MOLECULAR TRAJECTORIES +8431289079399714038 FILTERING +5137101329018983403 WEB ACCESS +6225410828137123812 ELECTRODE HOLDER +6457063246351992826 MODELING AND SIMULATION +8515460212604055874 LICENSE +-8930288288795974497 BIOMOLECULAR INTERACTION FACILITY +8610731640963512937 NETWORK MODEL +6724302182838282045 BINARY EXECUTABLE +2796345205148685087 OBJECT DATABASE +3451049717110536458 RESOURCE INTEGRATION COMPONENT +-2694441121243529820 MASS SPECTRA IDENTIFICATION +7571648235956266820 FREE TEXT +9152672707382700618 OUTCOMES RESEARCH +-5677998118067441263 INTERACTIVE NETWORK ANALYSIS +-6644764709513999944 DATA EDITOR +4410077876670437312 PCR FACILITY +-3865784332941342773 NARRATIVE RESOURCE +7540354788167522952 SOFTWARE +2358522258519952240 GENE EXPRESSION +-7797101775828639317 PCR INSTRUMENT +7274438605812833086 CONTINUING MEDICAL EDUCATION +-6377823176968675219 KNOWLEDGE MINING AND CAPTURING +7387020880380688199 RACK +-1067703126915461138 CENTRALITY ANALSYSIS +5351147866943696444 SERVER +-8549854008016971004 THESIS +8964729098645794820 CARDIOVASCULAR MODEL +-56272531993556950 ONTOLOGY VISUALIZATION +3657277684606349615 AVERAGING AND AGLOMERATION +5284394010525333746 HIGH DIMENSIONAL DATA +-8448891449940160066 IMAGING +-5795306557850638382 PHLEBOTOMY FACILITY +3346565502341141481 SIGNAL PROCESSING +6336999463013548705 STEM CELL THERAPY FACILITY +-7461691765935655042 SOCIAL NETWORKING +-5522177423302570516 THERAPEUTICS +-9128936520102714693 REGISTRATION +3507244053598959575 IDENTITY MANAGEMENT +-4003660710910182727 CONCEPT +-4885750806139943170 REPORT +-8854591128439542032 NUMERICAL METHODS +3282719800092256661 REGULATORY COMPLIANCE +435258444534793371 DATA COMPRESSION +8771347753116640807 NETWORK AND COMMUNICATION +-585516113569421411 NETWORK EDITOR +-9172417821965798170 CHAMBER +414578339480025648 RNA MODEL +6398233197180253086 SCALAR DATA +-227971307603812190 CODE TESTING FRAMEWORK +-59732373348765455 FACILITY CORE +1012416768906633876 REGULATORY POLICY RESOURCE +-4988167584538761621 DATA VISUALIZATION +-7339442702646477651 PHARMACOKINETICS PHARMACODYNAMICS EXPERTISE +-93171194175618237 ELECTRON MICROSCOPE +5047042978571140832 GRAPH ALGORITHM +-7298741150593786020 CONTACT MODELING +2360806855141682703 KNOWLEDGEBASE +4139523412986867709 ROBOTIC ARRAYER +-6351615082591917623 ANALYSIS OF GEL-ELECTROPHORESIS +1244091636959553141 VIBROTOME +303059711193738248 HIERARCHICAL TREE +8167439359829555800 TUTORIAL +-5263553231192396116 ANATOMIC ATLAS +4975210567286536915 INSTRUMENT +-5661427820033844380 CLINICAL TRIAL DATA +7331782580826243103 FABRICATION FACILITY +-8586179846560657134 PHARMACOKINETICS PHARMACODYNAMICS +-6862769945305630418 PHENOTYPIC MEASUREMENT +8524013364654522951 COMPUTATIONAL BIOLOGY +-2316095867731096563 GRAPH ALIGNMENT +-6435722579816125806 XML DATABASE +-3024715758846827778 THREE DIMENSIONAL DATA +-5662148782558810976 PATENT +566804011962769688 CERTIFICATE PROGRAM +-8673876299175543481 FEATURE ANALYSIS +-8466720388329608933 NMR INSTRUMENT +7669799769031294458 CONCEPT SCHEME +5560883352534573768 INDIVIDUAL HUMAN DATA +4724998136410977745 TUTORIALS +7177779093459638055 DATA OBJECT +-5139981900148257629 VERSION INFORMATION +7212676971349954647 DATA MINING AND INFERENCE +2785958543908061420 COMPUTATIONAL MODEL +-9105060075331231316 BIOMATERIAL MANUFACTURE +1618671815481011977 PARTIAL PARSING +-2224963628896019753 SEQUENCE ALIGNMENT +-319630454746520665 PROVENANCE +-5747083848182057176 METABOLOMICS FACILITY +-1167852989877426510 OSCILLOSCOPE +-436946742380088882 SOFTWARE ENGINEERING TOOL +1990918318322929935 APPLICATION PROGRAMMING INTERFACE +-3202021354415519826 HEAT MAP +-962123967256821251 MOLECULAR FORCE FIELD CALCULATOR +2404985851460375050 DATA ANALYSIS SOFTWARE +-6036965506481035898 AUTOMATED NATURAL LANGUAGE OUTPUT +3520683361471150664 MEDICAL DEVICE +-8050585872279757665 EXPLORATORY DATA ANALYSIS +5554447578485257787 AMPLIFIER +-2612263830181082857 COURSE MATERIAL +7268309969911178287 TRAINING RESOURCE +-5511598143389987311 WHITE PAPER +1582002275337575692 CENTRALITY ANALYSIS +5737025750116019341 TEMPERATURE CONTROL UNIT +-2115447232982869994 BIOMATERIAL SUPPLY RESOURCE +8912113243556041514 UNSTRUCTURED DATA +6487207896766081495 REAGENT MANUFACTURE +988295774902733112 DATABASE SOFTWARE +5002746404249470857 MANIFOLD VIEWER +-6430986056182430261 MANIFOLD VIEWERS +-2019181376301583011 DOCUMENT GENERATION +8853793116451465061 CELL PROCESSING CLEAN ROOM FACILITY +1372351193001488971 NONHUMAN DATA +-5471585109431199213 DOCUMENT RETRIEVAL +-6313399101276208485 BIOSITEMAPS INFORMATION MODEL +-3260167765662321127 CALORIMETER +7746926100946510399 TISSUE ORGAN FACILITY +-4388406906182257223 DATA NORMALIZATION +-8367798763114760527 BACKUP ARCHIVES AND DISASTER RECOVERY +779622185528365817 HYPOTHESIS TESTING ALGORITHM +-2287483260289774880 MICROSCOPE +-5900915720989703470 FAST FOURIER TRANSFORM +-7772718363427923071 GRAPH VIEWERS +-6281605990774434024 DATA COMPUTATION SERVICE +3445925524936541339 PATTERN RECOGNITION +-3627741320900865850 ROOT FINDER +-4976862181296691531 EDUCATION +7119330700872196825 ANIMAL COMPLIANCE RESOURCE +-6263657911230979695 PAPER +-5958067784959383263 SEMI-STRUCTURED KNOWLEDGE RESOURCE +5155679555729262645 ONTOLOGY DEVELOPMENT AND MANAGEMENT +-2261012904798281440 MOLECULAR TRAJECTORY DATA +5820183474834717763 LANGUAGE SUMMARIZATION +5628807321853199901 HYPERBOLIC GRAPH +-4315385799113878364 GOVERNMENT PUBLICATION +7122371398502322475 GENE THERAPY +4614690355367730050 DEPRECATED DATA RESOURCE +5991557591048575233 STRUCTURAL MODEL +1227219103094211683 CONFERENCE PROCEEDING +624096804428015915 NEUROLOGICAL FACILITY +4046825039778868554 KNOWLEDGE EXTRACTION +9079056035351709176 STRUCTURED KNOWLEDGE RESOURCE +-7156538974399828026 MATERIAL SERVICE +-1578041323081488812 GENOMIC PHENOTYPIC ANALYSIS +7700464748495667971 DATA ANNOTATION +3092499500482404104 NUTRITION FACILITY +-4915812981911727118 STATISTICAL ANALYSIS +-7992673575628130111 REGULATORY SIGNALING NETWORK RECONSTRUCTION +3521727835059681798 PROTEOMICS FACILITY +3181909073792110933 SUMMARIZATION +4141641657483559969 HOSTING +1852067263132208958 INTEGRATED HOSTING +-2813025106421946228 IMMUNOHISTOCHEMISTRY FACILITY +-2068592356535058149 DEEP PARSING +-8209392013301403402 INDUSTRY PARTNERSHIP EXPERTISE +-6202843443771268115 PROTEIN INTERACTION MODELING +-7173249183586248987 BIOMATERIAL ANALYSIS SERVICE +-2923163574102768536 NUMERICAL METHOD +7505471248873867872 DATA PROCESSING SOFTWARE +-5756165969015691070 NEWS +8828460842067128424 DEPRECATED ACTIVITY +-7828537020885535645 DOCUMENT +-5502215740284694942 DYNAMIC MODEL +-1541583217050667694 FLOW CYTOMETRY CELL SORTING FACILITY +8821192262924101526 COMMUNITY STRUCTURE ANALYSIS +-1648790377503946479 DELIMITED TABLE +9158638916015643258 SERVICE RESOURCE +1688506111631839432 CLINICAL RESEARCH DATA +-1145102124651953919 MOLECULAR MODEL +3903446762531360002 SMALL MOLECULE +5285764926588027481 IMMUNE MONITORING FACILITY +-7382503786929208893 ONLINE COURSE +-7524708475708985506 INTEGRATION +4676484357009308837 HETEROGENEOUS FUNDING RESOURCE +339608058151474415 INTELLECTUAL PROPERTY RESOURCE +8800291947508569505 DATA STORAGE +-2213725773137222331 POWER CONDITIONING UNIT +-2089834745621000237 NETWORK INTERACTION MODEL +-6881065111050506395 ACTIVITY +6148865148179567287 STATISTICAL TEXT ANALYSIS +9064452841080637490 ONTOLOGY +-3583481090954004476 INTERACTIVE TOOL +6811834454343764356 AVERAGING AND AGGLOMERATION +-1278353842138158049 SKULL STRIPPING +3027195517736941620 MATERIAL RESOURCE +865036111135859326 PUBLICATION +8308330418435225803 RESOURCE +2051884501545730973 PARSING +-642903430252811467 STATISTICAL PACKAGE +7044339183136991309 PREDICTION OF SIDE-CHAIN CONFORMATIONS +3889110379840129844 CONTROLLERS +3134856781465806249 AUTHORIZATION +-336086834847239444 WORKBENCH +-8437290100923474137 WAVELET TRANSFORM +-7000468927197561364 DEPRECATED RESOURCE +5293874429892776020 PRIVATE FUNDING RESOURCE +8076545481595029722 PHYSICO-CHEMICAL MODEL +-4195353038606669928 STIMULATOR +-2369624360764868450 DATA DISTRIBUTION +7019477027896712883 PRESSURE REGULATOR +-6345681279110820818 SIGNALING NETWORK RECONSTRUCTION +-3557335442980611467 PROGRAMMATIC ACCESS +-8433949686095365387 PROCESSING PIPELINE +-6658233369423728041 INTERACTIVE TOOLS +-984174973964636073 REAGENT RESOURCE +-1713450155734297916 COMPILER +-9494380740410042 MOLECULAR BIOLOGY FACILITY +2113033764809361864 CELL CULTURE FACILITY +-8442828037182129263 EXPERIMENTAL MEASUREMENT +2326617354023636581 MAPPER +-4236259982716274364 ASSESSMENT MATERIAL RESOURCE +-496700233841224390 SOFTWARE DEVELOPMENT +-8973652183617293647 WAVELET MODEL +1070231432032909816 FEDERAL FUNDING RESOURCE +3094915033235142952 MASS SPECTROMETER +-3303027801628953625 TEXT MINING +1126794748004198495 PORTLET COMPONENT +2194088469768301343 FARADAY CAGE +8800037609510495681 SOFTWARE DEVELOPMENT TOOL +7905554153396263879 WEB POSTING +-8925293915124688645 PHYSICO CHEMICAL MODELS +5891799176729121565 SEMANTIC NETWORK +-7224021834631483889 PIPELINE MANAGER +8673532949448207566 ONTOLOGY DIFF AND ALIGNMENT +-8288663833115544548 SEGMENTATION +2595095113503292004 PET SCANNER +8976701599788629819 SOURCE CODE +-2361690319314302722 PERSONAL COMMUNICATION +5806433517401929614 STANDALONE APPLICATION +-6886018312441917171 DOCUMENTATION GENERATION +1119310048859840646 MOLECULAR AND CELLULAR DATA +5919427642961956154 SOURCE CONTROL +-1878318608289223393 PSYCHOMETRICS +-8023432638566396884 PATHOLOGY LABORATORY SCREENING +8737657457527632122 KNOWLEDGE ENVIRONMENT +8929397480597855274 SEARCHING SORTING AND INDEXING +3912406653198969800 CARDIOVASCULAR FACILITY +-4263177198306834727 OUTREACH PROGRAM +2677753508471966650 SOFTWARE DEVELOPMENT ENVIRONMENT +1861256702773523030 XML DATA +9069523284613590300 SOFTWARE DISTRIBUTION +-6443882781963939441 COMMUNICATION AND COLLABORATIVE WORK +-7056702194239778972 LEXICAL ANALYSIS +-3717665799601385640 FUNDING RESOURCE +-5554214983644883879 PRE-PROCESSING +-8625338912081426402 SEMINAR SERIES +3392622287151631970 COMPUTATIONAL GEOMETRY +-3030914422741558975 PORTAL +-4332326572206091891 RADIOISOTOPES FACILITY +4405350425079816619 RESEARCH LAB MANAGEMENT +-7705721494338660224 ANIMAL MODELS FACILITY +-7447099181811248454 SOFTWARE DOCUMENTATION +-1941419865091456173 PATTERN INFERENCE ALGORITHM +5862431836336744939 DATABASE HOSTING +1930473943063838641 GRAPH VIEWER +8893727956126142727 BIOSAFETY LEVEL FACILITY +-2544404142957798420 DATA ACQUISITION SOFTWARE +4261021975454818476 INTERACTION NETWORK +5544108415396590100 CELL BIOLOGY FACILITY +-133282546453585982 OUTLIER DETECTION AND REMOVAL +353521543804399137 LIST +8894163312907803573 BUG REPORTING +-1511828135858399555 MODULAR COMPONENT +7835783034196472353 SEQUENCE ANNOTATION +797998526817601663 ALGORITHM +8980029764728152039 BOOK +3064918251979350923 INHOMOGENEITY CORRECTION +2674835120067629187 PHENOTYPE MANAGEMENT +5980262757398018946 PHYSIOLOGY +5955229869348425378 TWO D IMAGE +7369142015830491822 IMAGE +-2677492610733720418 LINEAR ALGEBRA TOOL +2735639526077431350 LINEAR ALGEBRA TOOLS +-8275429304360078731 GENOMICS FACILITY +1124200881700755841 GRANT PREPARATION EXPERTISE +8691279025847830283 NUMERICAL INTEGRATORS +-4995499746447473706 NATURAL LANGUAGE PROCESSING +3904538636057234398 HUMAN EMBRYONIC STEM CELL RESOURCE +-1589866974394606966 NON COMPUTATIONAL SERVICE +7814914657560589369 DATA EXPLORATION +-151331255892588485 MICRO DISSECTION FACILITY +5308907751493434626 HEALTH SERVICES +3016161795926674060 BEHAVIORAL SCIENCE +3360493709099181739 CELL MODEL +2525187811188351986 WEB SERVICE +9057442110303967124 CROSS-PLATFORM TOOL +9151074843022790355 CLINICAL DATA +1596985459212611202 FOUR DIMENSIONAL DATA +-5066929782099117821 NEUROMUSCULAR MODEL +-8709618144670939205 EPIDEMIOLOGY +7669166602719270901 INTERACTIVE WEB-BASED TOOL +-2825977831925757492 COLLECTION +7605257255913385512 STRUCTURED DATA +-1517367433454959948 NUMERICAL INTEGRATOR +-5277257370156868864 MONTE CARLO SIMULATION +-4036657303956928022 CLINICAL STUDIES +3703660474518287163 SYMBOLIC AND ANALYTIC MODEL +8196204751238997544 MATRIX +1288094256668071342 DATA REPOSITORY +-9131292505048416164 FINITE ELEMENT MODEL +-8319683882892217334 ENVIRONMENTAL HEALTH AND SAFETY RESOURCE +-2061804163362800169 BACKUP ARCHIVE SERVICE +-5951931723882913708 OPTIMIZER +4170372872267296263 HOMOLOGY MODELING +-6438089494042416462 NEXT GENERATION SEQUENCER +775247564204354128 AGGREGATE HUMAN DATA +-7070837237745868184 MOLECULAR INTERACTION +73830057926527996 SCRIPTING LANGUAGE AND ENVIRONMENT +6976066472889274868 MICROARRAY SCANNER +-8425575275404574139 EXPERTISE +-4686774464775391858 TIME SERIES ANALYSIS +780652882472391671 PDE SOLVER +7170693928105057961 SEQUENCE SIMILARITY SEARCHING +4234640679392787396 TOXICOLOGY EXPERTISE +-5450037272157747817 DISASTER RECOVERY SERVICE +6968411095898103204 MULTIBODY DYNAMICS +9004852016214168051 GRAPHICAL PROCESSING WORKFLOW ENVIRONMENTS +-7127659686929139924 PEDIATRICS +-392032358395931512 JOURNAL ARTICLE +5277325554620175799 PATHOLOGY +-1094022366975192164 SPECTRAL TRANSFORM +-2587603585491274540 MESH MODEL +-9205733258737709680 GENOMICS +-285123169382313912 TOXICOLOGY +-5861704188603297455 DATA STORAGE REPOSITORY +3690926088643602863 BEHAVIORAL ANALYSIS SERVICE +6738351777808689590 RESOURCE DESCRIPTION +-7395153106952105227 SOFTWARE TECHNOLOGY PROTOCOL +842330879816094339 BIOSPECIMEN MANAGEMENT +-402988747762396798 CONVOLUTION +2747218425573150663 METABOLOMICS +-440099031618820900 TESTING TOOLS +-5198022459726719599 COMPUTATIONAL SERVICE +487880524215884433 MICROSCOPY FACILITY +-2927464584956785003 ONLINE SUPPORT +-38174989616215727 DRUG DELIVERY DEVICE +1395286951435315476 MICROMANIPULATOR +5683334469831348934 COMPUTER-INSTRUMENTATION INTERFACE +2515776721117963260 FEATURE REQUEST +-1007884355447876957 GRAPHICAL INTEGRATION +4747859625397031711 HUMAN STUDIES COMPLIANCE RESOURCE +-8669777927430686276 COMMUNITY ENGAGEMENT +-334827724731581531 CONTINUUM METHOD SIMULATION +7020874442973664790 LICENSE AND TERMS FOR USE +3606852921416223051 SEQUENCE VISUALIZATION +5457210976849622591 CORTICAL MODELING +1821634065604157233 ANTIBODY PRODUCTION +-5114216767953883810 TRAINING +-4951366026958226145 AREA OF RESEARCH +-3013957274191836311 RANDOM NUMBER SIMULATION +-4801479170368061376 MOLECULAR MODELING AND CLASSIFICATION +7124138846028480788 ANIMAL CARE FACILITY +-4592615262450718771 DATABASE +-8469151766865800098 CLINICAL CHARTS +-3063713273527892124 PROTEIN MODEL +-1485201938445400410 DATA TRANSFORMS +7730737109682636667 BIOMEDICAL SUPPLY RESOURCE +4433191364968140618 NURSING +2692870098093647332 PEOPLE RESOURCE +-5635134906594287826 VERSION SOURCE CONTROL SYSTEM +-9062877798066397916 BIBLIOGRAPHIC RESOURCE +8373080092210713001 LICENSE AND TERMS FOR REDISTRIBUTION +-2712925346168453020 INFERENCE FROM DATA +2619330566210514004 ELECTRODE PULLER +4977932995550596597 INTERACTION MODELING +-965488722851272282 CROSS-SECTIONAL VIEWER +758444367051359964 STRUCTURED FILE +-376584281417999868 METABOLISM FACILITY +-7060686384806985951 RESOURCE INVENTORY +3105803313380115501 EDA +3740767673626025177 COMPUTATIONAL HOSTING +7440398134666174764 NUMERICAL MODEL +6558666137202398220 IMAGE RECONSTRUCTION +-5590085196166378417 NAMED ENTITY NORMALIZATION +8853017830293309920 DATA RESOURCE +2001393698684362834 ORDERED COLLECTION +5995972920328441532 APPROXIMATE GRAPH ALIGNMENT +-4294896960897134594 RESEARCH IT +2952378223667240688 TRAINING SERVICE +1309661695629100323 EXTERNALRESOURCE +-5963013168768058249 PROVENANCE AND INTELLECTUAL PROPERTY +-4968271972884930120 IMAGING FACILITY +-716661050690708000 DOCUMENT STRUCTURE PARSING +-4058511230043709601 REGULARIZATION AND SMOOTHING +8172304837744543353 CLINICAL CHART +142018379149098313 ONTOLOGY DEVELOPMENT +-6973326864011584332 TOOLKIT +-126623286739899875 CALCULATION OF SOLVENT ACCESSIBLE AREA +893836019065385360 MOLECULAR DYNAMICS +-7400584317170814112 RESEARCH ANIMALS FACILITY +2990057929363808023 CONFLICT OF INTEREST RESOURCE +-3347124503414366030 PSYCHOMETRICS EXPERTISE +1838814007471325678 STATE FUNDING RESOURCE +7702356401741029404 PATTERN AND MOTIF INFERENCE +-537261703111816791 CONTINUOUS DATA +-7434252324548603076 SHAPE ANALYSIS +-2918046187878101742 DATA SERVICE +-987782117907517057 HEATH SERVICES +-4950153647862613431 FOURIER TRANSFORM +-6231684838800935824 MEDICAL DEVICE DEVELOPMENT +5890473261703278944 INSTRUMENT MANUFACTURE +5091108103250477121 DATA TRANSFORM +4089314104197163440 RDF DATA +-224986520987265691 RESEARCH SUPPLIES +8800071305970328785 PHYSIOLOIGCAL MODEL +2939298226421184493 VOLUME MODEL SCENE DATA +-3703159830815685720 THREE D IMAGE +6800346767688044500 ATLAS +-731295259628771642 MICROTOME +214949725022967710 GRAPHICAL COMPOSITION +1584081409579139366 STANDARD SPECIFICATION +2318347146220427102 MECHANICAL SIMULATION +3361508253440516997 PROTEIN EXPRESSION +4890649640579942970 OSCILLOGRAPH +180888094967194583 VECTOR DATA +3390452449985783851 MRI SCANNER +-2065459142552764692 STOCHASTIC DATA +-5505862321998976068 SUPPORT +-9065955719201914617 COMMUNICATION INTERFACE +-3839066522373151064 ATLAS GENERATION +3303000017296909672 RELATIONAL DATABASE +-7682370418855761817 RESEARCH FUNDING +6998801798507610892 VIBRATION ISOLATION TABLE +3714347569265131269 MODEL FITTING ALGORITHM +2442815397923643499 NMR FACILITY +-5387646376598103435 UNSTRUCTURED KNOWLEDGE RESOURCE +-1255201352826667856 DATA STORAGE SERVICE +-3528784233823133726 SOFTWARE DEVELOPMENT RESOURCE +-1964236386542507481 X-RAY CRYSTALLOGRAPHY FACILITY +-9183213580893532402 SURGICAL PROCEDURE +5888659505527179355 CONNECTIVITY MATRIX +-2164926424563027494 LABORATORY SUPPLY RESOURCE +5090046131763254254 REGULATORY COMPLIANCE RESOURCE +-4102752849698102221 MATERIAL ANALYSIS SERVICE +1077257838803746079 SENTENCE SPLITTING +-2599968756975048886 NAMED ENTITY RECOGNITION +6175391927001537311 NOVEL THERAPEUTICS +3240100337358073746 BIOBANK FACILITY +3137824409842508021 MATERIAL STORAGE SERVICE +1241930190909111308 CLINICAL CARE DATA +8804108645404693143 LIGHT MICROSCOPE +2936917633860068509 PROTOCOL DEVELOPMENT EXPERTISE +6616082133809878169 INTEGRATION AND INTEROPERABILITY TOOL +5721354105663069554 INTEGRATION AND INTEROPERABILITY TOOLS +-3443484219203796878 INFORMATION RETRIEVAL +2603520518096331877 ELECTRON MICROSCOPE IN VERSION 3.2 +6506998931399529858 ACTIVITIES +5139999688215083590 MOLECULAR INTERACTION. IN VERSION 3.2. diff --git a/test/middleware/test_rack_attack.rb b/test/middleware/test_rack_attack.rb index 43143080..42b1ddf2 100644 --- a/test/middleware/test_rack_attack.rb +++ b/test/middleware/test_rack_attack.rb @@ -5,14 +5,14 @@ RACK_CONFIG = File.join([settings.root, "config.ru"]) class TestRackAttack < TestCase - + def self.before_suite # Store app settings @@auth_setting = LinkedData.settings.enable_security @@throttling_setting = LinkedData.settings.enable_throttling @@req_per_sec_limit = LinkedData::OntologiesAPI.settings.req_per_second_per_ip @@safe_ips = LinkedData::OntologiesAPI.settings.safe_ips - + LinkedData.settings.enable_security = true LinkedData::OntologiesAPI.settings.enable_throttling = true LinkedData::OntologiesAPI.settings.req_per_second_per_ip = 1 @@ -29,6 +29,8 @@ def self.before_suite @@admin.save # Redirect output or we get a bunch of noise from Rack (gets reset in the after_suite method). + # Disable output redirect when debugging + $stdout = File.open("/dev/null", "w") $stderr = File.open("/dev/null", "w") @@ -65,15 +67,15 @@ def self.after_suite LinkedData::OntologiesAPI.settings.enable_throttling = @@throttling_setting LinkedData::OntologiesAPI.settings.req_per_second_per_ip = @@req_per_sec_limit LinkedData::OntologiesAPI.settings.safe_ips = @@safe_ips - - Process.kill("HUP", @@pid1) + + Process.kill("TERM", @@pid1) Process.wait(@@pid1) - Process.kill("HUP", @@pid2) + Process.kill("TERM", @@pid2) Process.wait(@@pid2) - + $stdout = STDOUT $stderr = STDERR - + @@admin.delete @@user.delete @@bp_user.delete @@ -81,7 +83,7 @@ def self.after_suite def test_throttling_limit request_in_threads do - assert_raises(OpenURI::HTTPError) { request() } + assert_raises(OpenURI::HTTPError) { request } end end @@ -90,10 +92,10 @@ def test_throttling_limit def test_throttling_limit_with_forwarding limit = LinkedData::OntologiesAPI.settings.req_per_second_per_ip headers = {"Authorization" => "apikey token=#{@@user.apikey}", "X-Forwarded-For" => "1.2.3.6"} - + exception = assert_raises(OpenURI::HTTPError) do (limit * 5).times do - open("http://127.0.0.1:#{@@port1}/ontologies", headers) + URI.open("http://127.0.0.1:#{@@port1}/ontologies", headers) end end assert_match /429 Too Many Requests/, exception.message @@ -101,7 +103,7 @@ def test_throttling_limit_with_forwarding def test_throttling_admin_override request_in_threads do - assert_raises(OpenURI::HTTPError) { request() } + assert_raises(OpenURI::HTTPError) { request } request(user: @@admin) do |r| assert r.status[0].to_i == 200 @@ -111,14 +113,14 @@ def test_throttling_admin_override def test_two_servers_one_ip request_in_threads do - assert_raises(OpenURI::HTTPError) { request() } + assert_raises(OpenURI::HTTPError) { request } assert_raises(OpenURI::HTTPError) { request(port: @@port2) } end end def test_throttling_ui_override request_in_threads do - assert_raises(OpenURI::HTTPError) { request() } + assert_raises(OpenURI::HTTPError) { request } request(user: @@bp_user) do |r| assert r.status[0].to_i == 200 @@ -133,8 +135,8 @@ def test_throttling_safe_ips_override safe_ips.each do |safe_ip| headers = {"Authorization" => "apikey token=#{@@user.apikey}", "X-Forwarded-For" => "#{safe_ip}"} (limit * 5).times do - response = open("http://127.0.0.1:#{@@port1}/ontologies", headers) - refute_match /429/, response.status.first, "Requests from a safelisted IP address were throttled" + response = URI.open("http://127.0.0.1:#{@@port1}/ontologies", headers) + refute_match /429/, response.status.first, "Requests from a safelisted IP address were throttled" end end end @@ -150,7 +152,7 @@ def request(user: nil, port: nil) # Sometimes a single request can get through without failing depending # on the order of the request as it coincides with the threaded requests. (LinkedData::OntologiesAPI.settings.req_per_second_per_ip * 2).times do - open("http://127.0.0.1:#{port}/ontologies", headers) + URI.open("http://127.0.0.1:#{port}/ontologies", headers) end end @@ -162,7 +164,7 @@ def request_in_threads(&block) threads << Thread.new do while true sleep(0.2) - request() rescue next + request rescue next end end end diff --git a/test/ontologies_report.json b/test/ontologies_report.json new file mode 100644 index 00000000..c853f036 --- /dev/null +++ b/test/ontologies_report.json @@ -0,0 +1,473 @@ +{ + "ontologies": { + "PDRO": { + "problem": true, + "format": "OWL", + "date_created": "09/25/2017, 06:38 PM", + "administeredBy": [ + "vendetti" + ], + "logFilePath": "", + "report_date_updated": "08/27/2020, 08:43 AM", + "errErrorStatus": [ + "ERROR_ANNOTATOR", + "ERROR_INDEXED", + "ERROR_INDEXED_PROPERTIES" + ], + "errMissingStatus": [ + "INDEXED_PROPERTIES", + "ANNOTATOR", + "INDEXED" + ], + "errRunningReport": "Error while running report on component good_classes(): Goo::Base::AttributeNotLoaded: Attribute `classes` is not loaded for http://data.bioontology.org/ontologies/PDRO/submissions/8/metrics. Loaded attributes: #." + }, + "STY": { + "problem": false, + "format": "UMLS", + "date_created": "06/28/2021, 07:01 PM", + "administeredBy": [ + "admin" + ], + "logFilePath": "STY/1/parsing.log", + "report_date_updated": "06/28/2021, 04:36 PM" + }, + "ENVTHES": { + "problem": true, + "format": "SKOS", + "date_created": "01/02/2022, 01:54 PM", + "administeredBy": [ + "mabablue" + ], + "logFilePath": "ENVTHES/21/parsing.log", + "report_date_updated": "12/02/2022, 10:18 AM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "BSPO": { + "problem": true, + "format": "OWL", + "date_created": "07/30/2008, 08:32 PM", + "administeredBy": [ + "balhoff", + "obo-anatomy@lists.sourceforge.net" + ], + "logFilePath": "BSPO/31/parsing.log", + "report_date_updated": "02/02/2023, 01:24 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "AFO": { + "problem": true, + "format": "OWL", + "date_created": "04/08/2019, 06:51 PM", + "administeredBy": [ + "DG_Allotrope_AFO" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:09 PM", + "errMissingStatus": [ + "INDEXED_PROPERTIES", + "METRICS", + "ANNOTATOR", + "INDEXED" + ] + }, + "CL": { + "problem": true, + "format": "OWL", + "date_created": "05/29/2008, 04:32 PM", + "administeredBy": [ + "adiehl@informatics.jax.org" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:10 PM", + "errMissingStatus": [ + "ANNOTATOR" + ] + }, + "DDPHENO": { + "problem": false, + "format": "OBO", + "date_created": "09/20/2017, 02:34 PM", + "administeredBy": [ + "Pfey" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:14 PM" + }, + "DOID": { + "problem": true, + "format": "OWL", + "date_created": "03/26/2008, 05:15 PM", + "administeredBy": [ + "allenbaron", + "lynn.schriml@gmail.com" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:25 PM", + "errMissingStatus": [ + "ANNOTATOR" + ] + }, + "ECOCORE": { + "problem": false, + "format": "OWL", + "date_created": "09/04/2019, 01:02 PM", + "administeredBy": [ + "vendetti" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:25 PM" + }, + "ECTO": { + "problem": true, + "format": "OWL", + "date_created": "05/13/2020, 09:38 AM", + "administeredBy": [ + "vendetti" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:25 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ], + "errMissingStatus": [ + "ANNOTATOR" + ], + "errNoAnnotator": "Annotator - FEW results for: mesotrione | diazaalkane | concentration of organic molecular entity in soil | olive fat or oil refined food product | fluazifop | iloperidone | food stabiliser | lipoteichoic acid | L-dopa | antifolate" + }, + "EFO": { + "problem": false, + "format": "OWL", + "date_created": "11/20/2008, 07:39 PM", + "administeredBy": [ + "zmp", + "paolaroncaglia", + "simonjupp" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:27 PM" + }, + "FLU": { + "problem": true, + "format": "OWL", + "date_created": "01/12/2010, 10:30 PM", + "administeredBy": [ + "burkesquires@gmail.com" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:28 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "FPLX": { + "problem": false, + "format": "OBO", + "date_created": "03/26/2018, 05:57 PM", + "administeredBy": [ + "bengyori" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:30 PM" + }, + "GLYCORDF": { + "problem": false, + "format": "OWL", + "date_created": "08/04/2015, 10:10 PM", + "administeredBy": [ + "Rene Ranzinger", + "issaku" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:32 PM" + }, + "GO": { + "problem": false, + "format": "OBO", + "date_created": "06/26/2008, 02:06 AM", + "administeredBy": [ + "rama", + "go@geneontology.org" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:33 PM" + }, + "GO-PLUS": { + "problem": false, + "format": "OWL", + "date_created": "08/28/2015, 05:42 AM", + "administeredBy": [ + "davidos" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:34 PM" + }, + "GSSO": { + "problem": true, + "format": "OWL", + "date_created": "09/11/2019, 09:31 PM", + "administeredBy": [ + "superraptor2" + ], + "logFilePath": "GSSO/75/parsing.log", + "report_date_updated": "05/03/2023, 04:24 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "HO": { + "problem": true, + "format": "OWL", + "date_created": "07/05/2017, 04:42 PM", + "administeredBy": [ + "claxima" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:38 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "ICEO": { + "problem": false, + "format": "OWL", + "date_created": "04/14/2019, 04:25 PM", + "administeredBy": [ + "MiaLIU" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:38 PM" + }, + "ISO19115MI": { + "problem": true, + "format": "OWL", + "date_created": "07/16/2017, 06:38 PM", + "administeredBy": [ + "kdurante" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:39 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "MA": { + "problem": true, + "format": "OBO", + "date_created": "07/30/2008, 08:31 PM", + "administeredBy": [ + "anatomy@informatics.jax.org" + ], + "logFilePath": "MA/128/parsing.log", + "report_date_updated": "12/12/2023, 04:06 PM", + "errMissingStatus": [ + "ANNOTATOR", + "METRICS", + "INDEXED_PROPERTIES", + "INDEXED" + ] + }, + "MADS-RDF": { + "problem": false, + "format": "OWL", + "date_created": "10/15/2017, 03:47 PM", + "administeredBy": [ + "nlorimer" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM" + }, + "MARC-LANGUAGES": { + "problem": true, + "format": "SKOS", + "date_created": "05/07/2018, 12:59 PM", + "administeredBy": [ + "nlorimer" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ] + }, + "MF": { + "problem": false, + "format": "OWL", + "date_created": "01/29/2012, 08:19 AM", + "administeredBy": [ + "Janna Hastings" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM" + }, + "MS": { + "problem": true, + "format": "OBO", + "date_created": "09/04/2009, 02:42 AM", + "administeredBy": [ + "mayerg97" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM", + "errMissingStatus": [ + "ANNOTATOR" + ] + }, + "NBO": { + "problem": false, + "format": "OWL", + "date_created": "05/05/2011, 12:23 PM", + "administeredBy": [ + "gkoutos", + "aridag" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM" + }, + "QUDT": { + "problem": true, + "format": "OWL", + "date_created": "04/05/2012, 01:43 AM", + "administeredBy": [ + "egonw" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:40 PM", + "errMissingStatus": [ + "ANNOTATOR" + ] + }, + "ROCKNROLLTEST": { + "problem": true, + "format": "SKOS", + "date_created": "03/18/2021, 11:27 AM", + "administeredBy": [ + "niva" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:41 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ], + "errMissingStatus": [ + "OBSOLETE", + "INDEXED_PROPERTIES", + "METRICS", + "INDEXED" + ] + }, + "SKOSPLAY-EX4": { + "problem": true, + "format": "SKOS", + "date_created": "10/18/2020, 05:26 PM", + "administeredBy": [ + "graybeal" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:41 PM", + "errErrorStatus": [ + "ERROR_DIFF" + ], + "errMissingStatus": [ + "OBSOLETE", + "INDEXED_PROPERTIES", + "METRICS", + "INDEXED" + ] + }, + "SKOSTEST": { + "problem": true, + "format": "SKOS", + "date_created": "11/02/2018, 03:20 PM", + "administeredBy": [ + "graybeal" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 12:41 PM", + "errMissingStatus": [ + "OBSOLETE", + "INDEXED_PROPERTIES", + "METRICS", + "INDEXED" + ] + }, + "FOODON": { + "problem": true, + "format": "OWL", + "date_created": "03/29/2017, 12:46 PM", + "administeredBy": [ + "damion" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 01:01 PM", + "errMissingStatus": [ + "ANNOTATOR" + ], + "errNoSearch": "Search - NO results for: noodle (japanese-style) | Carcharhinus | 09340 - preserved tomatoes not concentrated (efsa foodex2) | obsolete: habanero pepper plant | concentration of quercetin 3-O-β-D-glucosyl-(1→2)-β-D-glucoside in material entity | 20970 - rabbit, other slaughtering products (efsa foodex2) | concentration of cis-caffeic acid in material entity | concentration of Isobetanin in material entity | soda water (artifically sweetened) | obsolete: giant gourami" + }, + "VFB_DRIVERS": { + "problem": true, + "format": "OWL", + "date_created": "10/13/2021, 05:31 AM", + "administeredBy": [ + "clare.pilgrim" + ], + "logFilePath": "", + "report_date_updated": "04/04/2023, 01:07 PM", + "errMissingStatus": [ + "ANNOTATOR" + ], + "errNoSearch": "Search - NO results for: P{GawB}62y | P{VT046771-GAL4} | P{R66B12-GAL4.DBD} ∩ P{R93A02-p65.AD} expression pattern | P{GMR94A10-GAL4} | P{lacZ.w[+]}R122 | P{GMR76A06-GAL4} | P{GMR23G03-GAL4} | P{GMR47C11-GAL4} | P{GawB}RpS11[NP0211] | P{sal-lacZ.BO}" + }, + "MONDO": { + "problem": true, + "format": "OBO", + "date_created": "09/25/2017, 05:43 PM", + "administeredBy": [ + "vendetti" + ], + "logFilePath": "MONDO/53/parsing.log", + "report_date_updated": "05/17/2023, 08:45 AM", + "errErrorStatus": [ + "ERROR_RDF_LABELS" + ], + "errMissingStatus": [ + "INDEXED_PROPERTIES", + "METRICS", + "ANNOTATOR", + "INDEXED" + ], + "errRunningReport": "Error while running report on component good_classes: ArgumentError: Not supported case for embed" + }, + "TEST-ONT-1": { + "problem": true, + "format": "OWL", + "date_created": "12/16/2023, 03:14 PM", + "administeredBy": [ + "tim" + ], + "logFilePath": "TEST-ONT-1/2/parsing.log", + "report_date_updated": "12/16/2023, 03:14 PM", + "errMissingStatus": [ + "METRICS", + "INDEXED_PROPERTIES", + "INDEXED" + ], + "errNoSearch": "Search - NO results for: Experimental Protocol | Code Profiler | Gene Therapy Facility | Monoclonal Antibody Facility | Technical Support | Dissemination Vehicle | Image Algorithm | Molecular Visualization | Calendar Schedule and Resource Management | Statistical Algorithm" + }, + "TEST-ONT-0": { + "problem": false, + "format": "OWL", + "date_created": "12/16/2023, 05:34 PM", + "administeredBy": [ + "tim" + ], + "logFilePath": "TEST-ONT-0/2/parsing.log", + "report_date_updated": "12/16/2023, 05:35 PM" + } + }, + "report_date_generated": "08/27/2020, 08:36 AM" +} \ No newline at end of file diff --git a/test/test_case.rb b/test/test_case.rb index 7d3d0716..98b02442 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -192,4 +192,26 @@ def get_errors(response) return errors.strip end + def self.enable_security + @@old_security_setting = LinkedData.settings.enable_security + LinkedData.settings.enable_security = true + end + + def self.reset_security(old_security = @@old_security_setting) + LinkedData.settings.enable_security = old_security + end + + + def self.make_admin(user) + user.bring_remaining + user.role = [LinkedData::Models::Users::Role.find(LinkedData::Models::Users::Role::ADMIN).first] + user.save + end + + def self.reset_to_not_admin(user) + user.bring_remaining + user.role = [LinkedData::Models::Users::Role.find(LinkedData::Models::Users::Role::DEFAULT).first] + user.save + end + end