Add tests for barclamp pacemaker

features/barclamp_pacemaker.feature

@@ -0,0 +1,46 @@
+@pacemaker
+Feature: Tests Pacemaker barclamp deployment
+    As an administrator
+    I want to make sure the pacemaker cluster is deployed successfully
+    And I can perform few operations on the components of the pacemaker cluster
+
+  Background:
+    Given the chef role "pacemaker-cluster-member" exists on admin node
+    And the chef role "pacemaker-config-data" exists on admin node
+    And the chef role "hawk-server" exists on admin node
+    And the "pacemaker" cookbook exists on the admin node
+
+  @packages
+  Scenario: Verify packages required for pacemaker deployment
+    Given the barclamp proposal for "pacemaker" is deployed
+    When the node with "pacemaker-cluster-member" role has been detected successfully
+    And the node with "hawk-server" role has been detected successfully
+    Then I can identify the primary and secondary nodes for the cluster
+    When the package "pacemaker" is installed in the controller node
+    And the package "corosync" is installed in the controller node
+    And the package "hawk" is installed in the controller node
+    And the package "drbd" is installed in the controller node
+
+  @drbd
+  Scenario: Verify functionality of DRBD deployment of rabbitmq and postgresql
+    Given I can identify the primary and secondary nodes for the cluster
+    Then I can check if the status of all crm nodes are ok
+    And I can identify the name of the Designated Controller node
+    And I can verify the location of rabbitmq and postgresql resources
+    And I can get the mountpoints for devices "drbd0" and "drbd1"
+    And I can ensure both primary and secondary nodes of drbd are UpToDate
+
+  @crm
+  Scenario: Verify Cluster Resource Manager features of deployed cluster
+    Given I can identify the primary and secondary nodes for the cluster
+    And I can identify the name of the Designated Controller node
+    Then I can list the available stonith devices
+    And I can obtain metadata for an arbitrary fencing agent
+    And I can check if cluster is configured with stonith-enabled
+    And I can check the failcount of stonith resources
+    And I can ensure that start operations for haproxy, rabbitmq, postgresql are complete
+    And I can update and delete new attribute to a crm node
+    And I can verify the status of corosync being active with no faults and correct ring addresses
+    And I can move a cluster node to standby and back to online without affecting resources
+    And I can run a cluster health check on the cluster
+

features/step_definitions/barclamp_pacemaker/pacemaker_steps.rb

@@ -0,0 +1,193 @@
+Given(/^the barclamp proposal for "([^"]*)" is deployed$/) do |package_name|
+    response = admin_node.exec!("crowbar #{package_name} list").output
+    @node_list = response.strip!.split(/\n/)
+    expect(@node_list).not_to be_empty
+end
+
+When(/^the node with "([^"]*)" role has been detected successfully$/) do |service_name|
+    @node_list.each do |node_name|
+      json_response = JSON.parse(admin_node.exec!("crowbar pacemaker show #{node_name}").output)
+      expect(json_response["deployment"]["pacemaker"]["elements"]["#{service_name}"]).not_to be_empty
+    end
+end
+
+Then(/^I can identify the primary and secondary nodes for the cluster$/) do
+    @primary_node = ""
+    @secondary_node = ""
+    json_response = JSON.parse(admin_node.exec!("crowbar pacemaker show data").output)
+    member_fqdns = json_response["deployment"]["pacemaker"]["elements"]["pacemaker-cluster-member"]
+    member_fqdns.each do |node_fqdn|
+        node_obj = nodes.find(fqdn:node_fqdn)
+        node_attr = node_obj.attributes()
+        dev_priority = node_obj.exec!("drbdsetup role 0").output.strip!
+        if dev_priority == "Primary/Secondary"
+            @primary_node = node_obj
+        elsif dev_priority == "Secondary/Primary"
+            @secondary_node = node_obj
+        end
+    end
+    puts "Primary Node: #{@primary_node.attributes()[:name]}"
+    puts "Secondary Node: #{@secondary_node.attributes()[:name]}"
+    expect(@primary_node.attributes()[:name]).not_to be_empty
+    expect(@secondary_node.attributes()[:name]).not_to be_empty
+end
+
+When(/^the package "([^"]*)" is installed in the controller node$/) do |package_name|
+    @primary_node.rpm_q(package_name)
+    @secondary_node.rpm_q(package_name)
+end
+
+And(/^I can check if the status of all crm nodes are ok$/) do
+    response = @primary_node.exec!("crmadmin --nodes").output
+    node_list = response.split(/\n/).reject { |c| c.empty?}
+    exp_states = ["S_IDLE (ok)", "S_NOT_DC (ok)"]
+    node_list.each do |node_entry|
+        node_name = node_entry.split(/ /)[3].strip!
+        stat_response = @primary_node.exec!("crmadmin --status=#{node_name}").output
+        node_state = stat_response.split(/:/)[1].strip!
+        expect(exp_states).to include(node_state)
+    end
+end
+
+And(/^I can identify the name of the Designated Controller node$/) do
+    response = @primary_node.exec!("crmadmin --dc_lookup").output
+    @dc_node = response.split(/:/)[-1].strip!
+    puts "Designated Controller Node : #{@dc_node}"
+    expect(@dc_node).not_to be_empty
+end
+
+And(/^I can verify the location of rabbitmq and postgresql resources$/) do
+    response = @primary_node.exec!("crm_resource --resource drbd-rabbitmq --locate").output
+    location = response.split(/:/)[-1].strip!
+    exp_location = ["#{@primary_node.attributes()[:name]}", @dc_node]
+    expect(exp_location).to include(location)
+    response = @primary_node.exec!("crm_resource --resource drbd-postgresql --locate").output
+    location = response.split(/:/)[-1].strip!
+    expect(exp_location).to include(location)
+end
+
+And(/^I can get the mountpoints for devices "drbd0" and "drbd1"$/) do
+    response = @primary_node.exec!("mount | grep drbd").output
+    mount_lines = response.split(/\n/)
+    mount_lines.each do |line|
+        drbd_dev = line.split(/ /)[0]
+        mount_pt = line.split(/ /)[2]
+        puts "Obtained mount point for #{drbd_dev} is #{mount_pt}"
+        expect(mount_pt).not_to be_empty
+    end
+end
+
+And(/^I can ensure both primary and secondary nodes of drbd are UpToDate$/) do
+    response = @primary_node.exec!("drbdadm dstate rabbitmq").output
+    expect(response.strip!).to eq("UpToDate/UpToDate")
+    response = @primary_node.exec!("drbdadm dstate postgresql").output
+    expect(response.strip!).to eq("UpToDate/UpToDate")
+end
+
+Then(/^I can list the available stonith devices$/) do
+    response = @primary_node.exec!("stonith_admin --list-registered").output
+    @stonith_device = response.split(/\n/)[0].strip!
+    expect(@stonith_device).to eq("stonith-#{@secondary_node.attributes()[:name]}")
+end
+
+And(/^I can obtain metadata for an arbitrary fencing agent$/) do
+    response = @primary_node.exec!("stonith_admin -M #{@stonith_device} --agent fence_pcmk").output
+    expect(response).not_to be_empty
+end
+
+And(/^I can check if cluster is configured with stonith-enabled$/) do
+    response = @primary_node.exec!("crm configure show type:property").output
+    expect(response).not_to be_empty
+end
+
+And(/^I can check the failcount of stonith resources$/) do
+    response = @primary_node.exec!("crm_failcount --resource-id=rabbitmq").output
+    fail_count = response.tr("\n", "").split(/ /)[-1]
+    expect(fail_count).to eq("value=0")
+end
+
+And(/^I can ensure that start operations for haproxy, rabbitmq, postgresql are complete$/) do
+    def get_node_resource_operations_status(node_to_check, resource_name)
+        response = @primary_node.exec!("crm_resource --list-operations --resource #{resource_name} --node #{node_to_check}").output
+        status = response.tr("\n","").split(/ /)[-1]
+        return status
+    end
+    prim_node_name = @primary_node.attributes()[:name]
+    expect(get_node_resource_operations_status(@dc_node, "haproxy")).to eq("complete")
+    expect(get_node_resource_operations_status(prim_node_name, "rabbitmq")).to eq("complete")
+    expect(get_node_resource_operations_status(prim_node_name, "postgresql")).to eq("complete")
+end
+
+And(/^I can update and delete new attribute to a crm node$/) do
+    @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --update office")
+    @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --query")
+    @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --delete")
+end
+
+And(/^I can verify the status of corosync being active with no faults and correct ring addresses$/) do
+    # Method to obtain Corosync ring status of a node in the cluster
+    def get_corosync_ring_status(node)
+        # Get ring status from the node
+        ring_addr = ""
+        ring_status = ""
+        response = node.exec!("crm corosync status").output
+        lines = response.tr("\t", " ").split(/\n/)
+        lines.each do |line|
+            if line.include?("id =")
+                ring_addr = line.split(/=/)[-1].strip!
+            end
+            if line.include?("status =")
+                ring_status = line.split(/=/)[-1].strip!
+            end
+        end
+        return ring_addr, ring_status
+    end
+
+    prim_ring_addr = ""
+    prim_status = ""
+    sec_ring_addr = ""
+    sec_status = ""
+
+    # Get ring address from node configuration
+    response = @primary_node.exec!("crm corosync get nodelist.node.ring0_addr").output
+    ring_addr_conf = response.split(/\n/)
+
+    # Get ring status from primary node
+    prim_ring_addr, prim_ring_status = get_corosync_ring_status(@primary_node)
+    sec_ring_addr, sec_ring_status = get_corosync_ring_status(@secondary_node)
+    expect(ring_addr_conf).to include(prim_ring_addr, sec_ring_addr)
+    expect(prim_ring_status).to eq("ring 0 active with no faults")
+    expect(sec_ring_status).to eq("ring 0 active with no faults")
+end
+
+And(/^I can move a cluster node to standby and back to online without affecting resources$/) do
+    # Method to return list of nodes in cluster in Standby and Online Mode
+    def return_standby_online_nodes (node)
+        response = node.exec!("crm_mon -1").output
+        lines = response.split(/\n/)
+        online_nodes = []
+        standby_nodes = []
+        lines.each do |line|
+            if line.include?("standby")
+                standby_nodes = [line.split(/ /)[1]]
+            elsif line.include?("Online:")
+                online_nodes = line.split(/:/)[-1].delete("[]").split(/ /).reject{ |c| c.empty?}
+            end
+        end
+        return online_nodes, standby_nodes
+    end 
+
+    @primary_node.exec!("crm node standby")
+    online_nodes, standby_nodes = return_standby_online_nodes(@primary_node)
+    expect(online_nodes).to include(@secondary_node.attributes()[:name])
+    expect(standby_nodes).to include(@primary_node.attributes()[:name])
+    @primary_node.exec!("crm node online")
+    online_nodes, standby_nodes = return_standby_online_nodes(@primary_node)
+    expect(online_nodes).to include(@primary_node.attributes()[:name], @secondary_node.attributes()[:name])
+    expect(standby_nodes).to be_empty
+end
+
+And(/^I can run a cluster health check on the cluster$/) do
+    response = @primary_node.exec!("crm cluster health").output
+end
+

tasks/features.rake

@@ -10,6 +10,7 @@ namespace :features do
 
   desc "Run barclamp tests"
   task :barclamps do
-
+    invoke_task "feature:barclamp:database"
+    invoke_task "feature:barclamp:pacemaker"
   end
 end

tasks/features/barclamp_pacemaker.rake

@@ -0,0 +1,22 @@
+namespace :feature do
+  feature_name "Tests Pacemaker barclamp deployment"
+
+  namespace :barclamp do
+    desc "Barclamp Pacemaker feature"
+    namespace :pacemaker do
+      desc "Check Package Installations"
+      feature_task :packages, tags: :@packages
+
+      desc "Test DRBD Features"
+      feature_task :drbd, tags: :@drbd
+
+      desc "Test CRM Features"
+      feature_task :crm, tags: :@crm
+
+      feature_task :all
+    end
+
+    desc "Verification of Pacemaker Cluster Deployment"
+    task :pacemaker => "pacemaker:all"
+  end
+end