Merge pull request #649 from splunk/develop

Release/8.2.2

docs/ADVANCED.md

@@ -243,6 +243,21 @@ splunk:
   ...
 ```
 
+Splunk Apps and Add-on archive files can also be extracted and installed using the `app_paths_install:` option.  This configuration will install a list of apps directly to the configure `app_paths` directory, bypassing any local install then copy to bundle directory that `apps_location` uses for cluster managing roles such as CM or Deployer.  The suported `app_paths` are default (local apps), shc, idxc, and deployment.  An example of a CM config with one local app and two cluster apps is:
+```
+splunk:
+  ...
+  app_paths_install:
+    default:
+      - /tmp/local_app.tgz
+    idxc:
+      - /tmp/cluster_app1.tgz
+      - /tmp/cluster_app2.spl
+  ...
+```
+
+The `shc` and `idxc` apps specified in `app_paths_install` are not installed locally so Apps that require a local installation prior to a cluster-wide implementation (such as Enterprise Security Suite) would not be support.  Those apps would still need to use the `apps_location` field for proper installation.
+
 ---
 
 ## SmartStore

docs/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Navigation
 
+* [8.2.2](#822)
 * [8.2.1](#821)
 * [8.2.0](#820)
 * [8.1.5](#815)
@@ -53,6 +54,16 @@
 
 ---
 
+## 8.2.2
+
+#### What's New?
+* Support for installing apps directly to a given path using `app_paths_install`. See our [documentation](https://splunk.github.io/splunk-ansible/ADVANCED.html#apps) for details.
+
+#### Changes
+* Bugfixes
+
+---
+
 ## 8.2.1
 
 #### What's New?

docs/advanced/default.yml.spec.md

@@ -283,6 +283,23 @@ splunk:
     * Path in filesystem of search head cluster apps (this will depend on splunk.home)
     * Default: /opt/splunk/etc/shcluster/apps
 
+  app_paths_install:
+    default: <list>
+    * List of apps to install into app_paths.default - elements can be in the form of a URL or a location in the filessytem
+    * Default: null
+
+    deployment: <list>
+    * List of apps to install into app_paths.deployment - elements can be in the form of a URL or a location in the filessytem
+    * Default: null
+
+    idxc: <list>
+    * List of apps to install into app_paths.idxc on the CM to be pushed to the Indexer Cluster - elements can be in the form of a URL or a location in the filessytem
+    * Default: null
+
+    shc: <list>
+    * List of apps to install into app_paths.shc on the deployer to be pushed to the Search Head Cluster- elements can be in the form of a URL or a location in the filessytem
+    * Default: null
+
   hec:
     enable: <bool>
     * Determine whether or not to disable setting up the HTTP event collector (HEC)

inventory/environ.py

@@ -255,7 +255,7 @@ def getSplunkWebSSL(vars_scope):
     """
     # TODO: Split this into its own splunk.http.* section
     splunk_vars = vars_scope["splunk"]
-    splunk_vars["http_enableSSL"] = bool(os.environ.get('SPLUNK_HTTP_ENABLESSL', splunk_vars.get("http_enableSSL")))
+    splunk_vars["http_enableSSL"] = os.environ.get('SPLUNK_HTTP_ENABLESSL', splunk_vars.get("http_enableSSL"))
     splunk_vars["http_enableSSL_cert"] = os.environ.get('SPLUNK_HTTP_ENABLESSL_CERT', splunk_vars.get("http_enableSSL_cert"))
     splunk_vars["http_enableSSL_privKey"] = os.environ.get('SPLUNK_HTTP_ENABLESSL_PRIVKEY', splunk_vars.get("http_enableSSL_privKey"))
     splunk_vars["http_enableSSL_privKey_password"] = os.environ.get('SPLUNK_HTTP_ENABLESSL_PRIVKEY_PASSWORD', splunk_vars.get("http_enableSSL_privKey_password"))
@@ -400,6 +400,29 @@ def getSplunkAppsLocal(vars_scope):
                 appListLocal.append(app)
     vars_scope["splunk"]["apps_location_local"] = appListLocal
 
+def getSplunkAppPathInstall(vars_scope):
+    """
+    Determine the set of Splunk apps to install at which location as a
+    union of defaults.yml.  The actual path to install to is determined by
+    the splunk.app_paths.* variable
+    """
+    appPaths = ["shc", "idxc", "default", "deployment"]
+    if not "app_paths_install" in vars_scope["splunk"]:
+        vars_scope["splunk"]["app_paths_install"] = {}
+        for path in appPaths:
+            vars_scope["splunk"]["app_paths_install"][path] = []
+    else:
+        for path in appPaths:
+            appList = []
+            if path in vars_scope["splunk"]["app_paths_install"]:
+                # From default.yml
+                if type(vars_scope["splunk"]["app_paths_install"][path]) == str:
+                    appList = vars_scope["splunk"]["app_paths_install"][path].split(",")
+                elif type(vars_scope["splunk"]["app_paths_install"][path]) == list:
+                    appList = vars_scope["splunk"]["app_paths_install"][path]
+
+            vars_scope["splunk"]["app_paths_install"][path] = appList
+
 def getSecrets(vars_scope):
     """
     Parse sensitive passphrases

roles/splunk_cluster_master/tasks/main.yml

@@ -14,6 +14,13 @@
 
 - include_tasks: initialize_cluster_master.yml
 
+- include_tasks: ../../../roles/splunk_common/tasks/extract_app.yml
+  when: "'app_paths_install' in splunk and 'idxc' in splunk.app_paths_install and splunk.app_paths_install.idxc"
+  vars:
+    app_filepath: "{{ item }}"
+    extract_path: "{{ splunk.app_paths.idxc }}"
+  loop: "{{ splunk.app_paths_install.idxc }}"
+
 - include_tasks: apply_cluster_bundle.yml
 
 # This installed_apps list gets mutated by ESS bundle generation, hence we
@@ -31,6 +38,12 @@
   vars:
     app_list: "{{ splunk.apps_location_local }}"
 
+- include_tasks: ../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
+
 - include_tasks: setup_multisite.yml
   when:
     - splunk.site is defined

roles/splunk_common/tasks/enable_splunkweb_ssl.yml

@@ -4,7 +4,7 @@
     dest: "{{ splunk.home }}/etc/system/local/web.conf"
     section: settings
     option: "enableSplunkWebSSL"
-    value: "True"
+    value: "{{ splunk.http_enableSSL }}"
     owner: "{{ splunk.user }}"
     group: "{{ splunk.group }}"
     mode: 0660

roles/splunk_common/tasks/extract_app.yml

@@ -0,0 +1,59 @@
+---
+- name: Install Splunkbase app
+  uri:
+    url: "{{ cert_prefix }}://127.0.0.1:{{ splunk.svc_port }}/services/apps/local"
+    method: POST
+    user: "{{ splunk.admin_user }}"
+    password: "{{ splunk.password }}"
+    force_basic_auth: yes
+    validate_certs: false
+    body:
+      name: "{{ app_filepath }}"
+      update: "true"
+      filename: "true"
+      auth: "{{ splunkbase_token }}"
+    body_format: "form-urlencoded"
+    status_code: [ 200, 201 ]
+    timeout: 300
+    use_proxy: no
+  when:
+    - app_filepath is defined
+    - "'splunkbase.splunk.com' in app_filepath"
+    - splunkbase_token is defined
+    - splunkbase_token != None
+  no_log: "{{ hide_password }}"
+
+- name: Install via app_paths
+  when: "'splunkbase.splunk.com' not in app_filepath"
+  block:
+  - name: Check local app
+    stat:
+      path: "{{ app_filepath }}"
+    register: app_local
+  - name: Download remote app
+    get_url:
+      url: "{{ app_filepath }}"
+      dest: /tmp/
+      mode: 0777
+      timeout: 120
+      validate_certs: no
+      force: yes
+    register: app_remote
+    when:
+      - app_filepath is match("^(https?|file)://.*")
+
+  - name: Infer app filepath
+    set_fact:
+      app_install_filepath: "{% if app_filepath | regex_search('^(https?|file)://.*') %}{{ app_remote.dest }}{% else %}{{ app_local.stat.path }}{% endif %}"
+
+  - name: Extract app to location
+    unarchive:
+      src: "{{ app_install_filepath }}"
+      dest: "{{ extract_path }}"
+      remote_src: true
+    become: yes
+    become_user: "{{ splunk.user }}"
+    no_log: "{{ hide_password }}"
+    when:
+      - app_install_filepath is defined
+

roles/splunk_common/tasks/main.yml

@@ -93,7 +93,6 @@
 - include_tasks: enable_splunkweb_ssl.yml
   when:
     - "'http_enableSSL' in splunk and splunk.http_enableSSL is not none"
-    - splunk.http_enableSSL | bool
 
 - include_tasks: enable_splunkd_ssl.yml
   when: "'ssl' in splunk and splunk.ssl"

roles/splunk_common/tasks/premium_apps/configure_ess.yml

@@ -20,6 +20,6 @@
   become_user: "{{ splunk.user }}"
   register: essinstall
   changed_when: essinstall.rc == 0
-  failed_when: essinstall.rc != 0
+  failed_when: essinstall.rc != 0 and 'Unknown search command' not in essinstall.stderr
   notify:
     - Restart the splunkd service

roles/splunk_deployer/tasks/bundle_push_to_search_heads.yml

@@ -0,0 +1,17 @@
+---
+- name: Apply shcluster bundle
+  command: "{{ splunk.exec }} apply shcluster-bundle -target {{ cert_prefix }}://{{ splunk.search_head_captain_url }}:{{ splunk.svc_port }} -auth {{ splunk.admin_user }}:{{ splunk.password }} --answer-yes -push-default-apps true"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  no_log: "{{ hide_password }}"
+  register: shcluster_bundle
+  until: shcluster_bundle.rc == 0 or "Found zero deployable apps" in shcluster_bundle.stderr
+  changed_when: shcluster_bundle.rc == 0
+  failed_when: shcluster_bundle.rc != 0 and "Found zero deployable apps" not in shcluster_bundle.stderr
+  retries: "{{ retry_num }}"
+  delay: "{{ retry_delay }}"
+  ignore_errors: true
+
+- debug:
+    msg: "WARNING: Applying shcluster bundle failed - proceeding anyways..."
+  when: shcluster_bundle.rc != 0

roles/splunk_deployer/tasks/main.yml

@@ -61,12 +61,36 @@
   vars:
     app_list: "{{ splunk.apps_location }}"
 
+# Only find and copy installed apps to the SHC bundle dir if we installed app via 'apps_location'
+- include_tasks: ../../../roles/splunk_common/tasks/find_installed_apps.yml
+  when:
+    - "'apps_location' in splunk and splunk.apps_location"
+    - splunk.deployment_server is not defined or not splunk.deployment_server
+
+- include_tasks: ../../../roles/splunk_common/tasks/copy_installed_apps.yml
+  when:
+    - "'apps_location' in splunk and splunk.apps_location"
+    - splunk.deployment_server is not defined or not splunk.deployment_server
+  vars:
+    apps: "{{ installed_apps }}"
+    dest: "{{ splunk.app_paths.shc }}"
+
 - include_tasks: ../../../roles/splunk_common/tasks/set_as_deployment_client.yml
   when: splunk.deployment_server is defined and splunk.deployment_server
 
 - include_tasks: push_apps_to_search_heads.yml
   when: "'apps_location' in splunk and splunk.apps_location"
 
+- include_tasks: ../../../roles/splunk_common/tasks/extract_app.yml
+  when: "'app_paths_install' in splunk and 'shc' in splunk.app_paths_install and splunk.app_paths_install.shc"
+  vars:
+    app_filepath: "{{ item }}"
+    extract_path: "{{ splunk.app_paths.shc }}"
+  loop: "{{ splunk.app_paths_install.shc }}"
+
+- include_tasks: bundle_push_to_search_heads.yml
+  when: "'app_paths_install' in splunk and 'shc' in splunk.app_paths_install and splunk.app_paths_install.shc"
+
 - include_tasks: ../../../roles/splunk_common/tasks/disable_installed_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
 
@@ -76,4 +100,10 @@
   vars:
     app_list: "{{ splunk.apps_location_local }}"
 
+- include_tasks: ../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
+
 - include_tasks: ../../../roles/splunk_common/tasks/check_for_required_restarts.yml

roles/splunk_deployment_server/tasks/main.yml

@@ -1,9 +1,22 @@
 ---
 - include_tasks: ../../../roles/splunk_common/tasks/enable_forwarding.yml
 
+- include_tasks: ../../../roles/splunk_common/tasks/extract_app.yml
+  when: "'app_paths_install' in splunk and 'deployment' in splunk.app_paths_install and splunk.app_paths_install.deployment"
+  vars:
+    app_filepath: "{{ item }}"
+    extract_path: "{{ splunk.app_paths.deployment }}"
+  loop: "{{ splunk.app_paths_install.deployment }}"
+
 - include_tasks: create_deployment_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
 
+- include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
+
 - include_tasks: generate_server_classes.yml
 
 - name: Reload deployment server

roles/splunk_heavy_forwarder/molecule/default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     volumes:
       - "/opt/splunk/var"
   - name: splunk-hf-debian10
-    image: geerlingguy/docker-debian10-ansible:latest
+    image: jgeusebroek/debian10-molecule:latest
     pre_build_image: true
     command: "/lib/systemd/systemd"
     volumes:

roles/splunk_heavy_forwarder/molecule/java/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     volumes:
       - "/opt/splunk/var"
   - name: splunk-hf-debian10
-    image: geerlingguy/docker-debian10-ansible:latest
+    image: jgeusebroek/debian10-molecule:latest
     pre_build_image: true
     command: "/lib/systemd/systemd"
     volumes:

roles/splunk_heavy_forwarder/molecule/manual-forwards/molecule.yml

@@ -15,7 +15,7 @@ platforms:
       https_proxy: "${https_proxy}"
       no_proxy: "${no_proxy}"
   - name: splunk-hf-debian10
-    image: geerlingguy/docker-debian10-ansible:latest
+    image: jgeusebroek/debian10-molecule:latest
     pre_build_image: true
     command: "/lib/systemd/systemd"
     volumes:

roles/splunk_heavy_forwarder/tasks/main.yml

@@ -17,6 +17,12 @@
   vars:
     app_list: "{{ splunk.apps_location }}"
 
+- include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
+
 - name: Execute Splunk commands
   command: "{{ splunk.exec }} {{ item }} -auth {{ splunk.admin_user }}:{{ splunk.password }}"
   become: yes

roles/splunk_indexer/molecule/default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     volumes:
       - "/opt/splunk/var"
   - name: splunk-standalone-debian10
-    image: geerlingguy/docker-debian10-ansible:latest
+    image: jgeusebroek/debian10-molecule:latest
     pre_build_image: true
     command: "/lib/systemd/systemd"
     volumes:

roles/splunk_indexer/tasks/main.yml

@@ -15,6 +15,16 @@
     - "'apps_location' in splunk and splunk.apps_location"
     - not splunk_indexer_cluster | bool
     - splunk.deployment_server is not defined or not splunk.deployment_server
+  vars:
+    app_list: "{{ splunk.apps_location}}"
+
+- include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+    - not splunk_indexer_cluster | bool
+    - splunk.deployment_server is not defined or not splunk.deployment_server
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
 
 - include_tasks: setup_multisite.yml
   when:

roles/splunk_license_master/tasks/main.yml

@@ -6,6 +6,12 @@
   vars:
     app_list: "{{ splunk.apps_location }}"
 
+- include_tasks: ../../splunk_common/tasks/provision_apps.yml
+  when:
+    - "'app_paths_install' in splunk and 'default' in splunk.app_paths_install and splunk.app_paths_install.default"
+  vars:
+    app_list: "{{ splunk.app_paths_install.default }}"
+
 - name: Enable license master local indexing
   ini_file:
     path: "{{ splunk.home }}/etc/system/local/outputs.conf"

roles/splunk_monitor/molecule/default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     volumes:
       - "/opt/splunk/var"
   - name: splunk-monitor-debian10
-    image: geerlingguy/docker-debian10-ansible:latest
+    image: jgeusebroek/debian10-molecule:latest
     pre_build_image: true
     command: "/lib/systemd/systemd"
     volumes: