Merge pull request #272 from rleigh-codelibre/bsd-windows-deps

Further ansible roles and playbooks for C++ build slaves

README.md

@@ -21,8 +21,12 @@ provided in the [ansible README](ansible/README.md).
   first steps.
 - [Local setup](docs/openstack/local-setup.md) details the steps to
   access OpenStack from a client system
+- [CentOS image creation](docs/openstack/centos-image.md)
 - [FreeBSD image creation](docs/openstack/freebsd-image.md)
+- [Ubuntu image creation](docs/openstack/ubuntu-image.md)
 - [Windows image creation](docs/openstack/windows-image.md)
+- [CI setup](docs/openstack/ci-setup.md) details the steps to
+  set up a CI node
 
 ## GPFS
 

ansible/ci-playbooks/ci-initial-setup.yml

@@ -0,0 +1,30 @@
+---
+# Initial setup to enable administration with ansible.
+
+- hosts: ci-jenkins-linux
+  gather_facts: False
+
+  tasks:
+    # Python might not yet be installed (needed for ansible).
+    - name: ci-initial-setup | Install python
+      raw: "if test -x /usr/bin/apt; then test -e /usr/bin/python || (apt -y update && apt install -y python-minimal); fi"
+      become: yes
+    - name: ci-initial-setup (debian) | Install pip
+      raw: "if test -x /usr/bin/apt; then test -e /usr/bin/pip || (apt -y update && apt install -y python-pip); fi"
+      become: yes
+    - name: ci-initial-setup (redhat) | Install pip
+      raw: "if test -f /etc/redhat-release; then test -e /usr/bin/pip || (yum -y install epel-release && yum -y install python-pip); fi"
+      become: yes
+    # aptitude might not yet be installed (needed for package tasks).
+    - name: ci-initial-setup | Install aptitude
+      raw: "if test -x /usr/bin/apt; then test -e /usr/bin/aptitude || (apt -y update && apt install -y aptitude); fi"
+      become: yes
+
+- hosts: all
+  roles:
+    - role: system-base-update
+    - role: system-package-update
+    - role: ci-base-setup
+    - role: ci-admin-user
+    - role: ci-admin-auth
+    - role: ci-scratch-storage

ansible/ci-playbooks/ci-setup.yml

@@ -0,0 +1,69 @@
+---
+# Set up host as a CI worker
+
+- hosts: all
+  roles:
+    - role: ci-base-setup
+    - role: ci-admin-auth
+
+- hosts: role-ome-data
+  roles:
+    - role: ci-data-repo
+
+- hosts: role-jenkins-slave:role-java
+  roles:
+    - role: pkg-jdk8
+
+- hosts: role-jenkins-slave
+  roles:
+    - role: ci-jenkins-user
+    - role: ci-jenkins-workspace
+    - role: pkg-jenkins-tools
+    - role: ci-jenkins-service
+
+- hosts: all
+  roles:
+    - role: pkg-git
+    - role: pkg-zip
+
+- hosts: role-cpp
+  roles:
+    - role: pkg-cmake
+    - role: pkg-ninja
+    - role: pkg-cpp-tools
+
+- hosts: role-cpp-full
+  roles:
+    # Due to the big dependency collection required for doxygen, we
+    # don't want it for minimal systems since the libraries it pulls
+    # in may conflict with our builds.
+    - role: pkg-doxygen
+    - role: pkg-cpp-libs
+
+- hosts: role-java
+  roles:
+    - role: pkg-java-tools
+
+- hosts: role-python2
+  roles:
+    - role: pkg-python27
+
+- hosts: role-docs
+  roles:
+    - role: pkg-sphinx
+
+- hosts: role-devtoolset-4
+  roles:
+    - role: pkg-devtoolset-4
+
+- hosts: role-vs2015
+  roles:
+    - role: pkg-vs2015
+
+- hosts: role-vs2017
+  roles:
+    - role: pkg-vs2017
+
+- hosts: role-cygwin
+  roles:
+    - role: cygwin

ansible/os-ci-create-machine.yml → ansible/ci-playbooks/os-ci-create-machine.yml

@@ -4,11 +4,20 @@
 #
 # Usage:
 #   Provision machine:
-#   $ ansible-playbook -e "ci_host=<host>" -i inventory/ci-hosts ansible/os-ci-create-machine.yml --vault-password-file=<vault>
+#   $ ansible-playbook -e "ci_host=<host>" -i inventory/ci-hosts ansible/os-ci-create-machine.yml
+#
 #   Initial setup:
 #     (FreeBSD)
-#     $ ssh freebsd@<host> # To add host key fingerprint
-#     $ ansible-playbook -l <host> -e "ansible_user=freebsd" -u freebsd -i inventory/ci-hosts ansible/ci-initial-setup.yml --vault-password-file=<vault>
+#       $ ssh freebsd@<host> # To add host key fingerprint
+#       $ ansible-playbook -l <host> -e "ansible_user=freebsd" -u freebsd -i inventory/ci-openstack-hosts ansible/ci-initial-setup.yml
+#
+#     (Ubuntu)
+#       $ ansible-playbook -l <host> -e "ansible_user=ubuntu" -u ubuntu -i inventory/ci-openstack-hosts ./ci-initial-setup.yml
+#
+#     (Windows)
+#       nova get-password ci-<host> | base64 -d | openssl rsautl -decrypt -inkey ~/.ssh/<ci key>
+#       ==> Admin password
+#       $ ansible-playbook -l <host> -e "ansible_user=Admin ansible_password=<password>" -u Admin -i inventory/ci-openstack-hosts ansible/ci-initial-setup.yml
 
 - hosts: localhost
   connection: local

ansible/ci-playbooks/roles/ci-admin-auth/README.md

@@ -0,0 +1,15 @@
+CI admin auth
+=============
+
+Add SSH public keys for the ci-admin user to
+~ci-admin/authorized_keys.
+
+Requirements
+------------
+
+None.
+
+Author Information
+------------------
+
+[email protected]

ansible/ci-playbooks/roles/ci-admin-auth/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- include: unix.yml
+  static: no
+  when: ansible_os_family != 'Windows'

ansible/ci-playbooks/roles/ci-admin-auth/tasks/unix.yml

@@ -0,0 +1,21 @@
+---
+- name: ci-admin-auth (unix) | Create .ssh
+  file:
+    path: /home/ci-admin/.ssh
+    state: directory
+    owner: ci-admin
+    group: ci-admin
+    mode: 0755
+  become: yes
+
+- name: ci-admin-auth (unix) | Update .ssh/authorized_keys
+  lineinfile:
+    path: /home/ci-admin/.ssh/authorized_keys
+    create: yes
+    owner: ci-admin
+    group: ci-admin
+    mode: 0600
+    state: "{{ item.state }}"
+    line: "{{ item.sshpubkey }}"
+  with_items: "{{ ci_admin_authorized_keys }}"
+  become: yes

ansible/ci-playbooks/roles/ci-admin-user/files/90-ome-admin-users

@@ -0,0 +1,2 @@
+# User rules for ci-admin
+ci-admin ALL=(ALL) NOPASSWD:ALL

ansible/ci-playbooks/roles/ci-admin-user/tasks/debian.yml

@@ -0,0 +1,6 @@
+---
+- name: ci-admin-user (debian) | Add ci-admin to admin groups
+  user:
+    name: ci-admin
+    groups: [adm,sudo]
+  become: yes

ansible/ci-playbooks/roles/ci-admin-user/tasks/linux.yml

@@ -0,0 +1,9 @@
+---
+- name: ci-admin-user (linux) | Add ci-admin to sudoers
+  copy:
+    src: 90-ome-admin-users
+    dest: /etc/sudoers.d/90-ome-admin-users
+    owner: root
+    group: root
+    mode: 0440
+  become: yes

ansible/ci-playbooks/roles/ci-admin-user/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- include: unix.yml
+  static: no
+  when: ansible_os_family != 'Windows'
+
+- include: linux.yml
+  static: no
+  when: ansible_system == 'Linux'
+
+- include: debian.yml
+  static: no
+  when: ansible_os_family == 'Debian'
+
+- include: freebsd.yml
+  static: no
+  when: ansible_os_family == 'FreeBSD'
+
+- include: windows.yml
+  static: no
+  when: ansible_os_family == 'Windows'

ansible/ci-playbooks/roles/ci-admin-user/tasks/unix.yml

@@ -0,0 +1,15 @@
+---
+- name: ci-admin-user (unix) | Create ci-admin group
+  group:
+    name: ci-admin
+    state: present
+  become: yes
+
+- name: ci-admin-user (unix) | Create ci-admin user
+  user:
+    name: ci-admin
+    comment: "CI System Administrator for Ansible use"
+    group: ci-admin
+    password: "{{ ci_admin_password_hash }}"
+    state: present
+  become: yes

ansible/ci-playbooks/roles/ci-admin-user/tasks/windows.yml

@@ -0,0 +1,17 @@
+---
+- name: ci-admin-user (windows) | Create ci-admins group
+  win_group:
+    name: ci-admins
+    description: "CI Administrator group"
+    state: present
+
+- name: ci-admin-user (windows) | Create ci-admin user
+  win_user:
+    name: ci-admin
+    fullname: "CI Administrator"
+    description: "CI System Administrator for Ansible use"
+    groups:
+      - ci-admins
+      - Administrators
+    password: "{{ ci_admin_password }}"
+    state: present

ansible/ci-playbooks/roles/ci-base-setup/tasks/debian.yml

@@ -0,0 +1,26 @@
+---
+- name: ci-base-setup (debian) | Set hostname
+  hostname:
+    name: "{{ inventory_hostname }}"
+  become: yes
+
+- name: ci-base-setup (debian) | Install Python build dependencies
+  apt:
+    name:  "{{ item }}"
+    state: present
+  with_items:
+    - libssl-dev
+    - python-dev
+    - libffi-dev
+  become: yes
+  when: ansible_distribution_release == 'trusty'
+
+- name: ci-base-setup (debian) | Install Python packages for SNI
+  pip:
+    name: "{{ item }}"
+  with_items:
+    - pyOpenSSL
+    - ndg-httpsclient
+    - pyasn1
+  become: yes
+  when: ansible_distribution_release == 'trusty'

ansible/roles/ci-base-setup/tasks/freebsd.yml → ansible/ci-playbooks/roles/ci-base-setup/tasks/freebsd.yml

@@ -48,7 +48,9 @@
       ci:\
             :umask=002:\
             :path=~/bin /usr/local/bin /usr/local/sbin /bin /sbin /usr/bin /usr/sbin:\
-            :setenv=CMAKE_PREFIX_PATH=/usr/local:
+            :setenv=CMAKE_PREFIX_PATH=/usr/local:\
+            :charset=UTF-8:\
+            :lang=en_US.UTF-8:\
             :tc=default:
   become: yes
   register: ci_login

ansible/ci-playbooks/roles/ci-base-setup/tasks/linux.yml

@@ -0,0 +1,15 @@
+---
+- name: ci-base-setup (linux) | Create tools directory
+  file:
+    path: "{{ tools_location }}/bin"
+    state: directory
+  become: yes
+
+- name: ci-base-setup (linux) | Add tools/bin to global PATH
+  lineinfile:
+    dest: /etc/environment
+    state: present
+    backrefs: yes
+    regexp: 'PATH=(["]*)((?!.*?/opt/tools/bin).*?)(["]*)$'
+    line: 'PATH=\1/opt/tools/bin:\2\3'
+  become: yes

ansible/ci-playbooks/roles/ci-base-setup/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- include: debian.yml
+  static: no
+  when: ansible_os_family == 'Debian'
+
+- include: redhat.yml
+  static: no
+  when: ansible_os_family == 'RedHat'
+
+- include: linux.yml
+  static: no
+  when: ansible_system == 'Linux'
+
+- include: freebsd.yml
+  static: no
+  when: ansible_os_family == 'FreeBSD'
+
+- include: windows.yml
+  static: no
+  when: ansible_os_family == 'Windows'

ansible/ci-playbooks/roles/ci-base-setup/tasks/redhat.yml

@@ -0,0 +1,25 @@
+---
+- name: ci-base-setup (redhat) | Set hostname
+  hostname:
+    name: "{{ inventory_hostname }}"
+  become: yes
+
+- name: ci-base-setup (redhat) | Add PATH to /etc/environment
+  lineinfile:
+    dest: /etc/environment
+    state: present
+    create: yes
+    regexp: '^PATH='
+    line: 'PATH=/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin'
+  become: yes
+
+- name: ci-base-setup (redhat) | Install Python packages for SNI
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - pyOpenSSL
+    - python2-ndg_httpsclient
+    - python-pyasn1
+  become: yes
+  when: ansible_distribution_major_version == '6'