diff --git a/features/barclamp_pacemaker.feature b/features/barclamp_pacemaker.feature new file mode 100644 index 0000000..e113945 --- /dev/null +++ b/features/barclamp_pacemaker.feature @@ -0,0 +1,46 @@ +@pacemaker +Feature: Tests Pacemaker barclamp deployment + As an administrator + I want to make sure the pacemaker cluster is deployed successfully + And I can perform few operations on the components of the pacemaker cluster + + Background: + Given the chef role "pacemaker-cluster-member" exists on admin node + And the chef role "pacemaker-config-data" exists on admin node + And the chef role "hawk-server" exists on admin node + And the "pacemaker" cookbook exists on the admin node + + @packages + Scenario: Verify packages required for pacemaker deployment + Given the barclamp proposal for "pacemaker" is deployed + When the node with "pacemaker-cluster-member" role has been detected successfully + And the node with "hawk-server" role has been detected successfully + Then I can identify the primary and secondary nodes for the cluster + When the package "pacemaker" is installed in the controller node + And the package "corosync" is installed in the controller node + And the package "hawk" is installed in the controller node + And the package "drbd" is installed in the controller node + + @drbd + Scenario: Verify functionality of DRBD deployment of rabbitmq and postgresql + Given I can identify the primary and secondary nodes for the cluster + Then I can check if the status of all crm nodes are ok + And I can identify the name of the Designated Controller node + And I can verify the location of rabbitmq and postgresql resources + And I can get the mountpoints for devices "drbd0" and "drbd1" + And I can ensure both primary and secondary nodes of drbd are UpToDate + + @crm + Scenario: Verify Cluster Resource Manager features of deployed cluster + Given I can identify the primary and secondary nodes for the cluster + And I can identify the name of the Designated Controller node + Then I can list the available stonith devices + And I can obtain metadata for an arbitrary fencing agent + And I can check if cluster is configured with stonith-enabled + And I can check the failcount of stonith resources + And I can ensure that start operations for haproxy, rabbitmq, postgresql are complete + And I can update and delete new attribute to a crm node + And I can verify the status of corosync being active with no faults and correct ring addresses + And I can move a cluster node to standby and back to online without affecting resources + And I can run a cluster health check on the cluster + diff --git a/features/step_definitions/barclamp_pacemaker/pacemaker_steps.rb b/features/step_definitions/barclamp_pacemaker/pacemaker_steps.rb new file mode 100644 index 0000000..edac63a --- /dev/null +++ b/features/step_definitions/barclamp_pacemaker/pacemaker_steps.rb @@ -0,0 +1,193 @@ +Given(/^the barclamp proposal for "([^"]*)" is deployed$/) do |package_name| + response = admin_node.exec!("crowbar #{package_name} list").output + @node_list = response.strip!.split(/\n/) + expect(@node_list).not_to be_empty +end + +When(/^the node with "([^"]*)" role has been detected successfully$/) do |service_name| + @node_list.each do |node_name| + json_response = JSON.parse(admin_node.exec!("crowbar pacemaker show #{node_name}").output) + expect(json_response["deployment"]["pacemaker"]["elements"]["#{service_name}"]).not_to be_empty + end +end + +Then(/^I can identify the primary and secondary nodes for the cluster$/) do + @primary_node = "" + @secondary_node = "" + json_response = JSON.parse(admin_node.exec!("crowbar pacemaker show data").output) + member_fqdns = json_response["deployment"]["pacemaker"]["elements"]["pacemaker-cluster-member"] + member_fqdns.each do |node_fqdn| + node_obj = nodes.find(fqdn:node_fqdn) + node_attr = node_obj.attributes() + dev_priority = node_obj.exec!("drbdsetup role 0").output.strip! + if dev_priority == "Primary/Secondary" + @primary_node = node_obj + elsif dev_priority == "Secondary/Primary" + @secondary_node = node_obj + end + end + puts "Primary Node: #{@primary_node.attributes()[:name]}" + puts "Secondary Node: #{@secondary_node.attributes()[:name]}" + expect(@primary_node.attributes()[:name]).not_to be_empty + expect(@secondary_node.attributes()[:name]).not_to be_empty +end + +When(/^the package "([^"]*)" is installed in the controller node$/) do |package_name| + @primary_node.rpm_q(package_name) + @secondary_node.rpm_q(package_name) +end + +And(/^I can check if the status of all crm nodes are ok$/) do + response = @primary_node.exec!("crmadmin --nodes").output + node_list = response.split(/\n/).reject { |c| c.empty?} + exp_states = ["S_IDLE (ok)", "S_NOT_DC (ok)"] + node_list.each do |node_entry| + node_name = node_entry.split(/ /)[3].strip! + stat_response = @primary_node.exec!("crmadmin --status=#{node_name}").output + node_state = stat_response.split(/:/)[1].strip! + expect(exp_states).to include(node_state) + end +end + +And(/^I can identify the name of the Designated Controller node$/) do + response = @primary_node.exec!("crmadmin --dc_lookup").output + @dc_node = response.split(/:/)[-1].strip! + puts "Designated Controller Node : #{@dc_node}" + expect(@dc_node).not_to be_empty +end + +And(/^I can verify the location of rabbitmq and postgresql resources$/) do + response = @primary_node.exec!("crm_resource --resource drbd-rabbitmq --locate").output + location = response.split(/:/)[-1].strip! + exp_location = ["#{@primary_node.attributes()[:name]}", @dc_node] + expect(exp_location).to include(location) + response = @primary_node.exec!("crm_resource --resource drbd-postgresql --locate").output + location = response.split(/:/)[-1].strip! + expect(exp_location).to include(location) +end + +And(/^I can get the mountpoints for devices "drbd0" and "drbd1"$/) do + response = @primary_node.exec!("mount | grep drbd").output + mount_lines = response.split(/\n/) + mount_lines.each do |line| + drbd_dev = line.split(/ /)[0] + mount_pt = line.split(/ /)[2] + puts "Obtained mount point for #{drbd_dev} is #{mount_pt}" + expect(mount_pt).not_to be_empty + end +end + +And(/^I can ensure both primary and secondary nodes of drbd are UpToDate$/) do + response = @primary_node.exec!("drbdadm dstate rabbitmq").output + expect(response.strip!).to eq("UpToDate/UpToDate") + response = @primary_node.exec!("drbdadm dstate postgresql").output + expect(response.strip!).to eq("UpToDate/UpToDate") +end + +Then(/^I can list the available stonith devices$/) do + response = @primary_node.exec!("stonith_admin --list-registered").output + @stonith_device = response.split(/\n/)[0].strip! + expect(@stonith_device).to eq("stonith-#{@secondary_node.attributes()[:name]}") +end + +And(/^I can obtain metadata for an arbitrary fencing agent$/) do + response = @primary_node.exec!("stonith_admin -M #{@stonith_device} --agent fence_pcmk").output + expect(response).not_to be_empty +end + +And(/^I can check if cluster is configured with stonith-enabled$/) do + response = @primary_node.exec!("crm configure show type:property").output + expect(response).not_to be_empty +end + +And(/^I can check the failcount of stonith resources$/) do + response = @primary_node.exec!("crm_failcount --resource-id=rabbitmq").output + fail_count = response.tr("\n", "").split(/ /)[-1] + expect(fail_count).to eq("value=0") +end + +And(/^I can ensure that start operations for haproxy, rabbitmq, postgresql are complete$/) do + def get_node_resource_operations_status(node_to_check, resource_name) + response = @primary_node.exec!("crm_resource --list-operations --resource #{resource_name} --node #{node_to_check}").output + status = response.tr("\n","").split(/ /)[-1] + return status + end + prim_node_name = @primary_node.attributes()[:name] + expect(get_node_resource_operations_status(@dc_node, "haproxy")).to eq("complete") + expect(get_node_resource_operations_status(prim_node_name, "rabbitmq")).to eq("complete") + expect(get_node_resource_operations_status(prim_node_name, "postgresql")).to eq("complete") +end + +And(/^I can update and delete new attribute to a crm node$/) do + @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --update office") + @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --query") + @primary_node.exec!("crm_attribute --node d52-54-01-77-77-01 --name location --delete") +end + +And(/^I can verify the status of corosync being active with no faults and correct ring addresses$/) do + # Method to obtain Corosync ring status of a node in the cluster + def get_corosync_ring_status(node) + # Get ring status from the node + ring_addr = "" + ring_status = "" + response = node.exec!("crm corosync status").output + lines = response.tr("\t", " ").split(/\n/) + lines.each do |line| + if line.include?("id =") + ring_addr = line.split(/=/)[-1].strip! + end + if line.include?("status =") + ring_status = line.split(/=/)[-1].strip! + end + end + return ring_addr, ring_status + end + + prim_ring_addr = "" + prim_status = "" + sec_ring_addr = "" + sec_status = "" + + # Get ring address from node configuration + response = @primary_node.exec!("crm corosync get nodelist.node.ring0_addr").output + ring_addr_conf = response.split(/\n/) + + # Get ring status from primary node + prim_ring_addr, prim_ring_status = get_corosync_ring_status(@primary_node) + sec_ring_addr, sec_ring_status = get_corosync_ring_status(@secondary_node) + expect(ring_addr_conf).to include(prim_ring_addr, sec_ring_addr) + expect(prim_ring_status).to eq("ring 0 active with no faults") + expect(sec_ring_status).to eq("ring 0 active with no faults") +end + +And(/^I can move a cluster node to standby and back to online without affecting resources$/) do + # Method to return list of nodes in cluster in Standby and Online Mode + def return_standby_online_nodes (node) + response = node.exec!("crm_mon -1").output + lines = response.split(/\n/) + online_nodes = [] + standby_nodes = [] + lines.each do |line| + if line.include?("standby") + standby_nodes = [line.split(/ /)[1]] + elsif line.include?("Online:") + online_nodes = line.split(/:/)[-1].delete("[]").split(/ /).reject{ |c| c.empty?} + end + end + return online_nodes, standby_nodes + end + + @primary_node.exec!("crm node standby") + online_nodes, standby_nodes = return_standby_online_nodes(@primary_node) + expect(online_nodes).to include(@secondary_node.attributes()[:name]) + expect(standby_nodes).to include(@primary_node.attributes()[:name]) + @primary_node.exec!("crm node online") + online_nodes, standby_nodes = return_standby_online_nodes(@primary_node) + expect(online_nodes).to include(@primary_node.attributes()[:name], @secondary_node.attributes()[:name]) + expect(standby_nodes).to be_empty +end + +And(/^I can run a cluster health check on the cluster$/) do + response = @primary_node.exec!("crm cluster health").output +end + diff --git a/tasks/features.rake b/tasks/features.rake index c0cf68a..f47452e 100644 --- a/tasks/features.rake +++ b/tasks/features.rake @@ -10,6 +10,7 @@ namespace :features do desc "Run barclamp tests" task :barclamps do - + invoke_task "feature:barclamp:database" + invoke_task "feature:barclamp:pacemaker" end end diff --git a/tasks/features/barclamp_pacemaker.rake b/tasks/features/barclamp_pacemaker.rake new file mode 100644 index 0000000..d469bf3 --- /dev/null +++ b/tasks/features/barclamp_pacemaker.rake @@ -0,0 +1,22 @@ +namespace :feature do + feature_name "Tests Pacemaker barclamp deployment" + + namespace :barclamp do + desc "Barclamp Pacemaker feature" + namespace :pacemaker do + desc "Check Package Installations" + feature_task :packages, tags: :@packages + + desc "Test DRBD Features" + feature_task :drbd, tags: :@drbd + + desc "Test CRM Features" + feature_task :crm, tags: :@crm + + feature_task :all + end + + desc "Verification of Pacemaker Cluster Deployment" + task :pacemaker => "pacemaker:all" + end +end