Skip to content

Commit 06087dd

Browse files
BorisYourichBorisYourich
committed
FTP upload not working yet
1 parent 14c1a4e commit 06087dd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+999
-55
lines changed

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,5 @@ On Debian 11 the default version of `ansible` is 2.10.08 which fails to extract
9090
Use `sudo apt install python3-docker` before running the `usegalaxy.rabbitmqserver` role.
9191

9292
For influxdb apt key error: https://www.influxdata.com/blog/linux-package-signing-key-rotation/
93+
94+
For proftpd add "LoadModule mod_tls.c" to roles/galaxyproject.proftpd/templates/tls.conf.j2 just before the TLSEngine

‎galaxy.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
- geerlingguy.redis
4444
- usegalaxy_eu.flower
4545
- galaxyproject.nginx
46+
- galaxyproject.proftpd
4647
- geerlingguy.docker
4748
- usegalaxy_eu.rabbitmqserver
4849
- galaxyproject.tiaas2

‎group_vars/galaxyservers.yml

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ galaxy_job_config:
2222
pulsar_runner:
2323
load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
2424
galaxy_url: "https://{{ inventory_hostname }}"
25-
amqp_url: "pyamqp://galaxy:{{ rabbitmq_users_password.galaxy }}@{{ inventory_hostname }}:5671/galaxy?ssl=1"
25+
amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.galaxy }}@{{ inventory_hostname }}:5671/pulsar?ssl=1"
2626
amqp_acknowledge: true
2727
amqp_ack_republish_time: 1200
2828
amqp_consumer_timeout: 2
@@ -52,44 +52,26 @@ galaxy_job_config:
5252
value: /tmp
5353
pulsar:
5454
runner: pulsar_runner
55-
default_file_action: remote_rsync_transfer
55+
default_file_action: remote_transfer
5656
dependency_resolution: remote
5757
jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-test/files/staging"
5858
persistence_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-test/files/persistent"
5959
remote_metadata: false
6060
rewrite_parameters: true
61+
transport: curl
6162
outputs_to_working_directory: false
6263
submit_native_specification: '-l select=1:ncpus=2:mem=8gb:scratch_local=50gb -l walltime=12:00:00 -q [email protected]'
6364
singularity_enabled: true
6465
singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR"
6566
## Following configuration works!
66-
#singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro"
67+
# singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro"
6768
container_resolvers:
6869
- type: explicit_singularity
6970
- type: mulled_singularity
70-
ssh_user: "{{ galaxy_user.name }}"
71-
ssh_host: "{{ inventory_hostname }}"
72-
ssh_port: 22
73-
ssh_key: |
74-
-----BEGIN OPENSSH PRIVATE KEY-----
75-
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
76-
QyNTUxOQAAACAonATHbBRX3FMl1tbOgsoh2DKBTzrMWkAJnhtAMfX9TgAAAJh2JLntdiS5
77-
7QAAAAtzc2gtZWQyNTUxOQAAACAonATHbBRX3FMl1tbOgsoh2DKBTzrMWkAJnhtAMfX9Tg
78-
AAAEAjJEh7bxnKxWkbTld8BUKj8wEEMrAFpKcd5BW7csc4wSicBMdsFFfcUyXW1s6CyiHY
79-
MoFPOsxaQAmeG0Ax9f1OAAAAEGdhbGF4eUB1c2VnYWxheHkBAgMEBQ==
80-
-----END OPENSSH PRIVATE KEY-----
8171
env:
8272
# Ensuring a consistent collation environment is good for reproducibility.
8373
- name: LC_ALL
8474
value: C
85-
# The cache directory holds the docker containers that get converted
86-
#- name: SINGULARITY_CACHEDIR
87-
# value: "/storage/praha5-elixir/home/galaxyeu/usegalaxy_data/cache/singularity"
88-
# Singularity uses a temporary directory to build the squashfs filesystem
89-
#- name: SINGULARITY_TMPDIR
90-
# value: "/storage/praha5-elixir/home/galaxyeu/usegalaxy_data/cache/singularity"
91-
#- name: SINGULARITY_BINDPATH
92-
# value: "/storage/praha5-elixir/home/galaxyeu:/home/galaxyeu:rw, $SCRATCHDIR:/scratch"
9375
- name: TMPDIR
9476
value: "$SCRATCHDIR"
9577
- name: TMP
@@ -119,6 +101,9 @@ galaxy_job_config:
119101
galaxy_config:
120102
galaxy:
121103
# Main Configuration
104+
enable_oidc: true
105+
oidc_config_file: "{{ galaxy_config_dir }}/oidc_config.xml"
106+
oidc_backends_config_file: "{{ galaxy_config_dir }}/oidc_backends_config.xml"
122107
logo_src: "https://www.e-infra.cz/img/logo.svg"
123108
themes_config_file: "{{ galaxy_config_dir }}/themes.yml"
124109
admin_users:
@@ -172,6 +157,9 @@ galaxy_config:
172157
# Monitoring
173158
statsd_host: localhost
174159
statsd_influxdb: true
160+
# FTP
161+
ftp_upload_dir: /data/uploads
162+
ftp_upload_site: "{{ inventory_hostname }}"
175163
gravity:
176164
process_manager: systemd
177165
galaxy_root: "{{ galaxy_root }}/server"
@@ -234,6 +222,10 @@ galaxy_config_templates:
234222
dest: "{{ galaxy_config.galaxy.job_resource_params_file }}"
235223
- src: templates/galaxy/config/reports.yml
236224
dest: "{{ galaxy_config.gravity.reports.config_file }}"
225+
- src: templates/galaxy/config/oidc_config.xml
226+
dest: "{{ galaxy_config_dir }}/oidc_config.xml"
227+
- src: templates/galaxy/config/oidc_backends_config.xml
228+
dest: "{{ galaxy_config_dir }}/oidc_backends_config.xml"
237229

238230
galaxy_extra_dirs:
239231
- /data
@@ -257,11 +249,13 @@ certbot_environment: production
257249
certbot_well_known_root: /srv/nginx/_well-known_root
258250
certbot_share_key_users:
259251
- www-data
252+
- proftpd
260253
certbot_share_key_ids:
261254
- "999:999"
262255
certbot_post_renewal: |
263256
systemctl restart nginx || true
264257
docker restart rabbit_hole || true
258+
systemctl restart proftpd || true
265259
certbot_domains:
266260
- "{{ inventory_hostname }}"
267261
certbot_agree_tos: --agree-tos
@@ -272,6 +266,7 @@ nginx_servers:
272266
- redirect-ssl
273267
nginx_ssl_servers:
274268
- galaxy
269+
# - sentry
275270
nginx_enable_default_server: false
276271
nginx_conf_http:
277272
client_max_body_size: 1g
@@ -331,17 +326,17 @@ rabbitmq_config:
331326
consumer_timeout: 21600000 # 6 hours in milliseconds
332327

333328
rabbitmq_vhosts:
334-
- galaxy
329+
- pulsar
335330
- galaxy_internal
336331

337332
rabbitmq_users:
338333
- user: debian
339334
password: "{{ rabbitmq_users_password.mqadmin }}"
340335
tags: administrator
341336
vhost: /
342-
- user: galaxy
337+
- user: pulsar
343338
password: "{{ rabbitmq_users_password.galaxy }}"
344-
vhost: galaxy
339+
vhost: pulsar
345340
- user: galaxy
346341
password: "{{ vault_rabbitmq_password_galaxy }}"
347342
vhost: galaxy_internal
@@ -398,3 +393,24 @@ tiaas_dir: /srv/tiaas
398393
tiaas_admin_user: admin
399394
tiaas_admin_pass: changeme
400395

396+
# Proftpd:
397+
proftpd_galaxy_auth: yes
398+
galaxy_ftp_upload_dir: "{{ galaxy_config.galaxy.ftp_upload_dir }}"
399+
proftpd_display_connect: |
400+
{{ inventory_hostname }} FTP server
401+
402+
Unauthorized access is prohibited
403+
proftpd_create_ftp_upload_dir: yes
404+
proftpd_options:
405+
- User: galaxy
406+
- Group: galaxy
407+
- Port: 21
408+
proftpd_sql_db: galaxy@/var/run/postgresql
409+
proftpd_sql_user: galaxy
410+
proftpd_conf_ssl_certificate: /etc/letsencrypt/live/{{ inventory_hostname }}/cert.pem
411+
proftpd_conf_ssl_certificate_key: /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem
412+
proftpd_global_options:
413+
- PassivePorts: 56000 60000
414+
proftpd_use_mod_tls_shmcache: false
415+
proftpd_tls_options: NoSessionReuseRequired
416+

‎group_vars/secret.yml

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,28 @@
11
$ANSIBLE_VAULT;1.1;AES256
2-
64346264363635373433303434653062656233303862383266626336343866623838633439616562
3-
3930363839653363366230353932396530363761663435300a386337653161303265393038646335
4-
33613363613266663834306332356433363532643235613630373139623761366338303732393365
5-
3538613964313436310a393466386361386437373934366635666633303235303130396162626233
6-
32336662613366333164666230373766643565343938366337393934303430393932313530393961
7-
37313331626130393963616238323732336439373763333766646537306565336131326433653264
8-
65363731383865316163633036383933333932346532363035396635376535333563616536633633
9-
61303438633638633563616463303764616465323934613434646132623362316139396665616335
10-
64336335376164333565633366636565346436623032633932613863303534653232393966646637
11-
38303361343736393366623835376539626463616563376564636239333131323262373565316433
12-
64623461376231313430386336616561613630303663306565373966633939353362383533383734
13-
32343038393337313666353265613032333031356634373534346263666637386462623133363739
14-
34343136636633643866613161313161366333623236393830646630333763623638326331396161
15-
34623564343933633066303035613435363662666466396666613464376263396666373033653431
16-
31656535366138653563306431626537643564313432353739366534323763383736373662393235
17-
37353336633234633737353939666264383963343730626562646230316230313335303530656136
18-
31623062393566613233316165663361333738353861623937373162653131303232623234633362
19-
39313931306261623034643339636437663765633033653435633936363136363334623339653531
20-
63623131313165633230646165373066346331396132316230616131613962393564303234636436
21-
32613462373864383666393237313835653739313261316531313864623466363966313565663033
22-
30353133613336363964353534393966623033373665663339633736616534643465623034396639
23-
31396232636462326161396265373533663266396130366164303631303937363265633830316338
24-
30366335356163323432303262356463316261633033386534373165326138396161396163663562
25-
32343837396430613063373439646535343661326562383161616534653930336235616631326561
26-
3938
2+
39323962323361343763363439383965383037613333363437353038643766343134616661616235
3+
3463653335613935613134616435366561363332383334630a366436623838636231396539633332
4+
37656332393737303938373232326435303138646361613966363236653936333534306133306635
5+
3965666563626666390a393539393932376532663333343461633834636538626436333838643936
6+
61303730393632366661363930626130323035383536633366356362363961383062313830396237
7+
31633339643334366333373037333139373664316238396330343862636363363064393564323936
8+
33396232316137373762623862653536303736623735396433623663393062323736366634613766
9+
64356238373939663738343338663162313035336639333036376265326562356233333436303436
10+
31616533613438306332613763383232313062613265323461346631363831373035383338643364
11+
35386163386235656638383535636432303038336133306230663965656235366663326166353531
12+
35313236346163636538356464613831346639656262643061396664386139653233306633633430
13+
32646163306131393137343532663064383030343061316262363430663432366630303366613830
14+
31383266336231356132643834326239386431616437383366346636323965316630323164376631
15+
63346430343862643939393735346165353633616366346339366437396335386137383234656466
16+
38336262303462633766396330643934656638663139306431343835373963363363363730376264
17+
31636165343133616632316637376634383133643632383837353531333764306234346633663662
18+
37366466386562396563356330333962353238363864643532326431656331313736333464303462
19+
64633465616565316138613161633065353563333036363630313736383739373765303734633331
20+
63633265356438323238613436343230333534663936613638303666373236653838303535313563
21+
64323862363066343139356130636532323562656633336330303834613536663536323263306133
22+
30633638326462313330613935656335333339373961303637313833633437636462616133363135
23+
38343030383363353237373033303637353762636536633165313063626466366532326337376334
24+
38393462303837656537666633393465353361343363613831346665646639313530353039633637
25+
64653663633833346335356361643864303637656361653339343361643933663731353663346636
26+
31326466353739363033626133643165616434353631303262373365643932313665393562373762
27+
30626166386130306439336462333063343033666463343032313536306561383033653133613134
28+
306132646333386630373232653564356530

‎group_vars/sentryservers.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
sentry_version: 23.3.1
2+
sentry_url: "https://{{ sentry_domain }}"
3+
sentry_docker_compose_project_folder: /srv/sentry
4+
sentry_superusers:
5+
- email: [email protected]
6+
password: "{{ vault_sentry_password }}"
7+

‎hosts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@ usegalaxy-test.cerit-sc.cz ansible_connection=local ansible_user=debian
44
galaxyservers
55
[monitoring]
66
usegalaxy-test.cerit-sc.cz ansible_connection=local ansible_user=debian
7+
[sentryservers]
8+
usegalaxy-test.cerit-sc.cz ansible_connection=local ansible_user=debian
9+

‎requirements.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,10 @@
4949
# Training Infrastructure as a Service
5050
- src: galaxyproject.tiaas2
5151
version: 2.1.5
52+
# Sentry
53+
- name: mvdbeek.sentry_selfhosted
54+
src: https://github.com/mvdbeek/ansible-role-sentry/archive/main.tar.gz
55+
# Our FTP Server
56+
- src: galaxyproject.proftpd
57+
version: 0.3.1
5258

‎roles/galaxyproject.proftpd/.travis.yml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
language: python
3+
python: "2.7"
4+
5+
# Use the new container infrastructure
6+
sudo: false
7+
8+
# Install ansible
9+
addons:
10+
apt:
11+
packages:
12+
- python-pip
13+
14+
install:
15+
# Install ansible
16+
- pip install ansible
17+
18+
# Check ansible version
19+
- ansible --version
20+
21+
# Create ansible.cfg with correct roles_path
22+
- printf '[defaults]\nroles_path=../' >ansible.cfg
23+
24+
script:
25+
# Basic role syntax check
26+
- ansible-playbook tests/test.yml -i tests/inventory --syntax-check
27+
28+
notifications:
29+
webhooks: https://galaxy.ansible.com/api/v1/notifications/

0 commit comments

Comments
 (0)