Skip to content
This repository has been archived by the owner on Mar 4, 2021. It is now read-only.

Commit

Permalink
Merge pull request #269 from ebukoski/master
Browse files Browse the repository at this point in the history
Add support for exclusion rules to JanitorRuleEngine
  • Loading branch information
ebukoski authored Aug 29, 2016
2 parents b05e618 + caf3d6b commit a5fe562
Show file tree
Hide file tree
Showing 7 changed files with 347 additions and 63 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public class EddaImageJanitorCrawler implements JanitorCrawler {
private final Set<String> usedByInstance = Sets.newHashSet();
private final Set<String> usedByLaunchConfig = Sets.newHashSet();
private final Set<String> usedNames = Sets.newHashSet();
private final Map<String, String> imageIdToName = Maps.newHashMap();
protected final Map<String, String> imageIdToName = Maps.newHashMap();
private final Map<String, Long> imageIdToCreationTime = Maps.newHashMap();
private final Set<String> ancestorImageIds = Sets.newHashSet();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
*
* Copyright 2012 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.netflix.simianarmy.aws.janitor.rule.generic;

import com.netflix.simianarmy.Resource;
import com.netflix.simianarmy.janitor.Rule;
import org.apache.commons.lang.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

/**
* A rule for excluding resources that contain the provided tags (name and value).
*
* If a resource contains the tag and the appropriate value, it will be excluded from any
* other janitor rules and will not be cleaned.
*
*/
public class TagValueExclusionRule implements Rule {

/** The Constant LOGGER. */
private static final Logger LOGGER = LoggerFactory.getLogger(TagValueExclusionRule.class);
private final Map<String,String> tags;

/**
* Constructor for TagValueExclusionRule.
*
* @param tags
* Set of tags and values to match for exclusion
*/
public TagValueExclusionRule(Map<String, String> tags) {
this.tags = tags;
}

/**
* Constructor for TagValueExclusionRule. Use this constructor to pass names and values as separate args.
* This is intended for convenience when specifying tag names/values in property files.
*
* Each tag[i] = (name[i], value[i])
*
* @param names
* Set of names to match for exclusion. Size of names must match size of values.
* @param values
* Set of values to match for exclusion. Size of names must match size of values.
*/
public TagValueExclusionRule(String[] names, String[] values) {
tags = new HashMap<String,String>();
int i = 0;
for(String name : names) {
tags.put(name, values[i]);
i++;
}
}

@Override
public boolean isValid(Resource resource) {
Validate.notNull(resource);
for (String tagName : tags.keySet()) {
String resourceValue = resource.getTag(tagName);
if (resourceValue != null && resourceValue.equals(tags.get(tagName))) {
LOGGER.debug(String.format("The resource %s has the exclusion tag %s with value %s", resource.getId(), tagName, resourceValue));
return true;
}
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,6 @@
// CHECKSTYLE IGNORE MagicNumberCheck
package com.netflix.simianarmy.basic.janitor;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;
Expand All @@ -36,31 +27,12 @@
import com.netflix.simianarmy.MonkeyCalendar;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder;
import com.netflix.simianarmy.aws.janitor.ASGJanitor;
import com.netflix.simianarmy.aws.janitor.EBSSnapshotJanitor;
import com.netflix.simianarmy.aws.janitor.EBSVolumeJanitor;
import com.netflix.simianarmy.aws.janitor.ImageJanitor;
import com.netflix.simianarmy.aws.janitor.InstanceJanitor;
import com.netflix.simianarmy.aws.janitor.LaunchConfigJanitor;
import com.netflix.simianarmy.aws.janitor.RDSJanitorResourceTracker;
import com.netflix.simianarmy.aws.janitor.SimpleDBJanitorResourceTracker;
import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.EBSSnapshotJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.EBSVolumeJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.InstanceJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.LaunchConfigJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaASGJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaEBSSnapshotJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaEBSVolumeJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaImageJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaInstanceJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.crawler.edda.EddaLaunchConfigJanitorCrawler;
import com.netflix.simianarmy.aws.janitor.*;
import com.netflix.simianarmy.aws.janitor.crawler.*;
import com.netflix.simianarmy.aws.janitor.crawler.edda.*;
import com.netflix.simianarmy.aws.janitor.rule.ami.UnusedImageRule;
import com.netflix.simianarmy.aws.janitor.rule.asg.ASGInstanceValidator;
import com.netflix.simianarmy.aws.janitor.rule.asg.DiscoveryASGInstanceValidator;
import com.netflix.simianarmy.aws.janitor.rule.asg.DummyASGInstanceValidator;
import com.netflix.simianarmy.aws.janitor.rule.asg.OldEmptyASGRule;
import com.netflix.simianarmy.aws.janitor.rule.asg.SuspendedASGRule;
import com.netflix.simianarmy.aws.janitor.rule.asg.*;
import com.netflix.simianarmy.aws.janitor.rule.generic.TagValueExclusionRule;
import com.netflix.simianarmy.aws.janitor.rule.generic.UntaggedRule;
import com.netflix.simianarmy.aws.janitor.rule.instance.OrphanedInstanceRule;
import com.netflix.simianarmy.aws.janitor.rule.launchconfig.OldUnusedLaunchConfigRule;
Expand All @@ -69,13 +41,12 @@
import com.netflix.simianarmy.aws.janitor.rule.volume.OldDetachedVolumeRule;
import com.netflix.simianarmy.basic.BasicSimianArmyContext;
import com.netflix.simianarmy.client.edda.EddaClient;
import com.netflix.simianarmy.janitor.AbstractJanitor;
import com.netflix.simianarmy.janitor.JanitorCrawler;
import com.netflix.simianarmy.janitor.JanitorEmailBuilder;
import com.netflix.simianarmy.janitor.JanitorEmailNotifier;
import com.netflix.simianarmy.janitor.JanitorMonkey;
import com.netflix.simianarmy.janitor.JanitorResourceTracker;
import com.netflix.simianarmy.janitor.JanitorRuleEngine;
import com.netflix.simianarmy.janitor.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

/**
* The basic implementation of the context class for Janitor monkey.
Expand Down Expand Up @@ -178,9 +149,21 @@ public BasicJanitorMonkeyContext() {
janitors.add(getImageJanitor());
}
}
protected JanitorRuleEngine createJanitorRuleEngine() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.TagValueExclusionRule.enabled", false)) {
String tagsList = configuration().getStr("simianarmy.janitor.rule.TagValueExclusionRule.tags");
String valsList = configuration().getStr("simianarmy.janitor.rule.TagValueExclusionRule.vals");
if (tagsList != null && valsList != null) {
TagValueExclusionRule rule = new TagValueExclusionRule(tagsList.split(","), valsList.split(","));
ruleEngine.addExclusionRule(rule);
}
}
return ruleEngine;
}

private ASGJanitor getASGJanitor() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
boolean discoveryEnabled = configuration().getBoolOrElse("simianarmy.janitor.Eureka.enabled", false);
ASGInstanceValidator instanceValidator;
if (discoveryEnabled) {
Expand Down Expand Up @@ -234,7 +217,7 @@ && getUntaggedRuleResourceSet().contains("ASG")) {
}

private InstanceJanitor getInstanceJanitor() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.orphanedInstanceRule.enabled", false)) {
ruleEngine.addRule(new OrphanedInstanceRule(monkeyCalendar,
(int) configuration().getNumOrElse(
Expand Down Expand Up @@ -272,7 +255,7 @@ && getUntaggedRuleResourceSet().contains("INSTANCE")) {
}

private EBSVolumeJanitor getEBSVolumeJanitor() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.oldDetachedVolumeRule.enabled", false)) {
ruleEngine.addRule(new OldDetachedVolumeRule(monkeyCalendar,
(int) configuration().getNumOrElse(
Expand Down Expand Up @@ -310,7 +293,7 @@ && getUntaggedRuleResourceSet().contains("EBS_VOLUME")) {
}

private EBSSnapshotJanitor getEBSSnapshotJanitor() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.noGeneratedAMIRule.enabled", false)) {
ruleEngine.addRule(new NoGeneratedAMIRule(monkeyCalendar,
(int) configuration().getNumOrElse("simianarmy.janitor.rule.noGeneratedAMIRule.ageThreshold", 30),
Expand Down Expand Up @@ -344,7 +327,7 @@ && getUntaggedRuleResourceSet().contains("EBS_SNAPSHOT")) {
}

private LaunchConfigJanitor getLaunchConfigJanitor() {
JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.oldUnusedLaunchConfigRule.enabled", false)) {
ruleEngine.addRule(new OldUnusedLaunchConfigRule(monkeyCalendar,
(int) configuration().getNumOrElse(
Expand Down Expand Up @@ -386,7 +369,7 @@ private ImageJanitor getImageJanitor() {
throw new RuntimeException("Image Janitor only works when Edda is enabled.");
}

JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine();
JanitorRuleEngine ruleEngine = createJanitorRuleEngine();
if (configuration().getBoolOrElse("simianarmy.janitor.rule.unusedImageRule.enabled", false)) {
ruleEngine.addRule(new UnusedImageRule(monkeyCalendar,
(int) configuration().getNumOrElse(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,15 @@

package com.netflix.simianarmy.basic.janitor;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.simianarmy.Resource;
import com.netflix.simianarmy.janitor.JanitorRuleEngine;
import com.netflix.simianarmy.janitor.Rule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
* Basic implementation of janitor rule engine that runs all containing rules to decide if a resource should be
Expand All @@ -41,11 +40,15 @@ public class BasicJanitorRuleEngine implements JanitorRuleEngine {
/** The rules to decide if a resource should be a candidate for cleanup. **/
private final List<Rule> rules;

/** The rules to decide if a resource should be excluded for cleanup. **/
private final List<Rule> exclusionRules;

/**
* The constructor of JanitorRuleEngine.
*/
public BasicJanitorRuleEngine() {
rules = new ArrayList<Rule>();
exclusionRules = new ArrayList<Rule>();
}

/**
Expand All @@ -61,8 +64,16 @@ public BasicJanitorRuleEngine() {
*/
@Override
public boolean isValid(Resource resource) {
LOGGER.debug(String.format("Checking if resource %s of type %s is a cleanup candidate against %d rules.",
resource.getId(), resource.getResourceType(), rules.size()));
LOGGER.debug(String.format("Checking if resource %s of type %s is a cleanup candidate against %d rules and %d exclusion rules.",
resource.getId(), resource.getResourceType(), rules.size(), exclusionRules.size()));

for (Rule exclusionRule : exclusionRules) {
if (exclusionRule.isValid(resource)) {
LOGGER.info(String.format("Resource %s is not marked as a cleanup candidate because of an exclusion rule.", resource.getId()));
return true;
}
}

// We create a clone of the resource each time when we try the rule. In the first iteration of the rules
// we identify the rule with the nearest termination date if there is any rule considers the resource
// as a cleanup candidate. Then the rule is applied to the original resource.
Expand Down Expand Up @@ -98,10 +109,25 @@ public BasicJanitorRuleEngine addRule(Rule rule) {
rules.add(rule);
return this;
}


/** {@inheritDoc} */
@Override
public BasicJanitorRuleEngine addExclusionRule(Rule rule){
exclusionRules.add(rule);
return this;
}

/** {@inheritDoc} */
@Override
public List<Rule> getRules() {
return this.rules;
}


/** {@inheritDoc} */
@Override
public List<Rule> getExclusionRules() {
return this.exclusionRules;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,29 @@ public interface JanitorRuleEngine {
* @return The JanitorRuleEngine object.
*/
JanitorRuleEngine addRule(Rule rule);


/**
* Add a rule to decide if a resource should be excluded for cleanup.
* Exclusion rules are evaluated before regular rules. If a resource
* matches an exclusion rule, it is excluded from all other cleanup rules.
*
* @param rule
* The rule to decide if a resource should be excluded for cleanup.
* @return The JanitorRuleEngine object.
*/
JanitorRuleEngine addExclusionRule(Rule rule);

/**
* Get rules to find out what's planned for enforcement.
*
* @return An ArrayList of Rules.
*/
List<Rule> getRules();

/**
* Get rules to find out what's excluded for enforcement.
*
* @return An ArrayList of Rules.
*/
List<Rule> getExclusionRules();
}
Loading

0 comments on commit a5fe562

Please sign in to comment.