From 51f94a36e5107c7f2ae5b2e9a1c876d825b646e1 Mon Sep 17 00:00:00 2001 From: Sumit Jamgade Date: Thu, 29 Nov 2018 10:38:50 +0100 Subject: [PATCH 1/2] Added ability to include recipes smartly Roles can get smart by making recipe inform the role about its dependencies across the proposal. So a recipe will be included by the role only if the attributes registered by the recipe are actually changed in current chef-client run. a recipe can register its dependencies as ``` mydependson = { "horizon::server" => [ ['glance','api','bind_port'] ] } BarclampLibrary::Barclamp::DependsOn.add(mydependson) ``` So here the "horizon::server" is informing about its dependencies. This is accomplished by comparing the value of the current attributes of node against the one already stored in the databag after the previous successful chef-client run. BarclampLibrary::Barclamp::DependsOn.add: takes in a map: recipe_name => [ [barclampname,drill,down,till,value], [otherbarclampname,onlyhere], ] This map is flushed and recreated only everyrun, however like resources this could also be cached. this behaviour can be altered by adding flag to config and using it 'include_recipe_smartly', however that is an extension to this behavior and can be addressed in subsequent commits Use the role(proposal) that was committed to compare against databag This object enables us to do real comparison on proposal level. Thus every barclamp can have proposal level dependency hound fixes and refactor --- .../barclamp/libraries/barclamp_library.rb | 99 +++++++++++++++---- .../barclamp/libraries/smartroles.rb | 22 +++++ 2 files changed, 100 insertions(+), 21 deletions(-) create mode 100644 chef/cookbooks/barclamp/libraries/smartroles.rb diff --git a/chef/cookbooks/barclamp/libraries/barclamp_library.rb b/chef/cookbooks/barclamp/libraries/barclamp_library.rb index 06927cee8d..75811eca53 100644 --- a/chef/cookbooks/barclamp/libraries/barclamp_library.rb +++ b/chef/cookbooks/barclamp/libraries/barclamp_library.rb @@ -14,6 +14,7 @@ # require_relative "conduit_resolver.rb" +require "chef/role" module BarclampLibrary class Barclamp @@ -417,21 +418,93 @@ def self.size_to_bytes(s) end end + class DependsOn + class << self + def add(dependency) + @recipe_depedencies ||= Mash.new + @recipe_depedencies.merge!(dependency) + end + + def get(recipe) + @recipe_depedencies.fetch(recipe, []) + end + end + end + class Config class << self attr_accessor :node - def load(group, barclamp, instance = nil) - # If no instance is specified, see if this node uses an instance of - # this barclamp and use it + def last_two_configs(group, barclamp, instance) + prev_cfg = load(group, barclamp, instance) + instance = infer_instance_from_cached_databag(group, barclamp, instance) + cur_cfg = Chef::Role.load "#{barclamp}-config-#{instance}" + return prev_cfg, cur_cfg.default_attributes[barclamp] + end + + def loadattr(collection, attrlist) + begin + attrlist.each do |item| + unless collection.key?(item) + Chef::Log.info("[smart] #{item} is missing from collection") + return nil + end + collection = collection[item] + end + rescue NoMethodError + Chef::Log.info("[smart] collection did not respond to []/key? while looking for #{item}") + return nil + end + collection + end + + def guess_instance(barclamp, instance) if instance.nil? && @node[barclamp] && @node[barclamp][:config] instance = @node[barclamp][:config][:environment] end - # Accept environments passed as instances if instance =~ /^#{barclamp}-config-(.*)/ instance = $1 end + instance + end + + def infer_instance_from_cached_databag(group, barclamp, instance) + if instance.nil? + # try the "default" instance, and fallback on any existing instance + instance = "default" + unless @cache["groups"][group].fetch(instance, {}).key?(barclamp) + # sort to guarantee a consistent order + @cache["groups"][group].keys.sort.each do |key| + # ignore the id attribute from the data bag item, which is not + # an instance + next if key == "id" + if @cache["groups"][group][key].key?(barclamp) + instance = key + break + end + end + end + end + instance + end + + def changes_to_apply?(depedency, group = "openstack", instance = nil) + barclamp = depedency.shift + instance = guess_instance(barclamp, instance) + prev_cfg, curr_cfg = last_two_configs(group, barclamp, instance) + old = loadattr(prev_cfg, depedency) + new = loadattr(curr_cfg, depedency) + # Chef::Log.info("[smart] loadattr prev #{depedency}, #{prev_cfg}") + # Chef::Log.info("[smart] loadattr curr #{depedency}, #{curr_cfg.inspect}") + # Chef::Log.info("[smart] comparision #{old}, #{new}") + old != new + end + + def load(group, barclamp, instance = nil) + # If no instance is specified, see if this node uses an instance of + # this barclamp and use it + instance = guess_instance(barclamp, instance) # Cache the config we load from data bag items. # This cache needs to be invalidated for each chef-client run from @@ -453,23 +526,7 @@ def load(group, barclamp, instance = nil) {} end - if instance.nil? - # try the "default" instance, and fallback on any existing instance - instance = "default" - unless @cache["groups"][group].fetch(instance, {}).key?(barclamp) - # sort to guarantee a consistent order - @cache["groups"][group].keys.sort.each do |key| - # ignore the id attribute from the data bag item, which is not - # an instance - next if key == "id" - if @cache["groups"][group][key].key?(barclamp) - instance = key - break - end - end - end - end - + instance = infer_instance_from_cached_databag(group, barclamp, instance) @cache["groups"][group].fetch(instance, {}).fetch(barclamp, {}) end end diff --git a/chef/cookbooks/barclamp/libraries/smartroles.rb b/chef/cookbooks/barclamp/libraries/smartroles.rb new file mode 100644 index 0000000000..d956af081c --- /dev/null +++ b/chef/cookbooks/barclamp/libraries/smartroles.rb @@ -0,0 +1,22 @@ +require "chef/mixin/language_include_recipe" + +class Chef + module Mixin + module LanguageIncludeRecipe + def include_recipe_smartly(*list_of_recipes) + list_of_recipes.each do |recipe| + included = false + BarclampLibrary::Barclamp::DependsOn.get(recipe).each do |dependency| + next unless BarclampLibrary::Barclamp::Config.changes_to_apply?(dependency) + Chef::Log.info("[smart] including recipe: #{recipe}") + Chef::Log.debug("[smart] due to change in: #{dependency}") + include_recipe recipe + included = true + break + end # each + Chef::Log.info("[smart] recipe excluded: #{recipe}") unless included + end # each + end # def include_recipe_smartly + end + end +end From 5d04197f5956895d9058f8a6c8c301e725615eda Mon Sep 17 00:00:00 2001 From: Sumit Jamgade Date: Mon, 25 Feb 2019 14:03:41 +0100 Subject: [PATCH 2/2] added commnets --- .../barclamp/libraries/barclamp_library.rb | 13 ++++++++++++- chef/cookbooks/barclamp/libraries/smartroles.rb | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/barclamp/libraries/barclamp_library.rb b/chef/cookbooks/barclamp/libraries/barclamp_library.rb index 75811eca53..0db10db7f6 100644 --- a/chef/cookbooks/barclamp/libraries/barclamp_library.rb +++ b/chef/cookbooks/barclamp/libraries/barclamp_library.rb @@ -418,8 +418,11 @@ def self.size_to_bytes(s) end end + # this class holds the recipe_depedencies object + # the object is added as the class level class DependsOn class << self + # api to add recipe dependency def add(dependency) @recipe_depedencies ||= Mash.new @recipe_depedencies.merge!(dependency) @@ -442,6 +445,12 @@ def last_two_configs(group, barclamp, instance) return prev_cfg, cur_cfg.default_attributes[barclamp] end + # a attrlist is ['api','bind_port'] (array) + # and collection is assumed to be a hash like object + # this method will return collection['api']['bind_port'] + # + # - abort and return nil , if any key not found + # - log accordingly def loadattr(collection, attrlist) begin attrlist.each do |item| @@ -452,7 +461,9 @@ def loadattr(collection, attrlist) collection = collection[item] end rescue NoMethodError - Chef::Log.info("[smart] collection did not respond to []/key? while looking for #{item}") + msg = "[smart] collection variable did not " \ + "respond to []/key? while looking for #{item}" + Chef::Log.info(msg) return nil end collection diff --git a/chef/cookbooks/barclamp/libraries/smartroles.rb b/chef/cookbooks/barclamp/libraries/smartroles.rb index d956af081c..4e63451838 100644 --- a/chef/cookbooks/barclamp/libraries/smartroles.rb +++ b/chef/cookbooks/barclamp/libraries/smartroles.rb @@ -4,15 +4,28 @@ class Chef module Mixin module LanguageIncludeRecipe def include_recipe_smartly(*list_of_recipes) + # Iterate over the dependcies and find if either have changed + # if it has changed include the recipe + # "horizon::server" => [ + # ['glance','api','bind_port'], + # ['horizon'] + # ] + # - break immediately on first changed dependency + # - if dependency list is empty, include recipe + # - log accordingly list_of_recipes.each do |recipe| included = false - BarclampLibrary::Barclamp::DependsOn.get(recipe).each do |dependency| + dependancylist = BarclampLibrary::Barclamp::DependsOn.get(recipe) + dependancylist.each do |dependency| next unless BarclampLibrary::Barclamp::Config.changes_to_apply?(dependency) - Chef::Log.info("[smart] including recipe: #{recipe}") + Chef::Log.info("[smart] including recipe : #{recipe}") Chef::Log.debug("[smart] due to change in: #{dependency}") include_recipe recipe included = true break + end.empty? and begin + include_recipe recipe + Chef::Log.info("[smart] including recipe without depencies: #{recipe}") end # each Chef::Log.info("[smart] recipe excluded: #{recipe}") unless included end # each