diff --git a/crawler/plugins/applications/apache/apache_container_crawler.py b/crawler/plugins/applications/apache/apache_container_crawler.py index d7c0eb75..427bf542 100644 --- a/crawler/plugins/applications/apache/apache_container_crawler.py +++ b/crawler/plugins/applications/apache/apache_container_crawler.py @@ -1,10 +1,11 @@ +from utils.namespace import run_as_another_namespace import logging - - +import json +import utils.misc import dockercontainer from icrawl_plugin import IContainerCrawler from plugins.applications.apache import apache_crawler -from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError logger = logging.getLogger('crawlutils') @@ -12,6 +13,7 @@ class ApacheContainerCrawler(IContainerCrawler): feature_type = 'application' feature_key = 'apache' + default_port = 80 def get_feature(self): return self.feature_key @@ -20,32 +22,44 @@ def crawl(self, container_id=None, **kwargs): c = dockercontainer.DockerContainer(container_id) - # check image name - if c.image_name.find("httpd") == -1: - - logger.error("%s is not %s container", - c.image_name, - self.feature_key) - raise CrawlError("%s does not have expected name for %s (name=%s)", - container_id, - self.feature_key, - c.image_name) - - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() - - # crawl all candidate ports - for port in ports: - try: - metrics = apache_crawler.retrieve_metrics(ip, port) - except CrawlError: - logger.error("can't find metrics endpoint at http://%s:%s", - ip, - port) - continue - return [(self.feature_key, metrics, self.feature_type)] + port = None + + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) - raise CrawlError("%s has no accessible endpoint for %s", - container_id, - self.feature_key) + else: + ports = c.get_container_ports() + + for each_port in ports: + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port + + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break + try: + metrics = apache_crawler.retrieve_metrics(ip, port) + return [(self.feature_key, metrics, self.feature_type)] + except: + logger.info("apache does not listen on port:%d", port) + raise ConnectionError("apache does not listen on port:%d", port) diff --git a/crawler/plugins/applications/db2/db2_container_crawler.py b/crawler/plugins/applications/db2/db2_container_crawler.py index 7843b291..f010ff75 100644 --- a/crawler/plugins/applications/db2/db2_container_crawler.py +++ b/crawler/plugins/applications/db2/db2_container_crawler.py @@ -1,9 +1,11 @@ import logging - import dockercontainer from icrawl_plugin import IContainerCrawler from plugins.applications.db2 import db2_crawler -from utils.crawler_exceptions import CrawlError +from utils.namespace import run_as_another_namespace +import json +import utils.misc +from requests.exceptions import ConnectionError logger = logging.getLogger('crawlutils') @@ -11,11 +13,12 @@ class DB2ContainerCrawler(IContainerCrawler): feature_type = 'application' feature_key = 'db2' + default_port = 50000 def get_feature(self): return self.feature_key - def crawl(self, container_id=None, **kwargs): + def get_opt(self, kwargs): password = "db2inst1" user = "db2inst1-pwd" db = "sample" @@ -29,37 +32,57 @@ def crawl(self, container_id=None, **kwargs): if "db" in kwargs: db = kwargs["db"] + return password, user, db + + def crawl(self, container_id=None, **kwargs): + + password, user, db = self.get_opt(kwargs) c = dockercontainer.DockerContainer(container_id) - # check image name - if c.image_name.find(self.feature_key) == -1: - logger.error("%s is not %s container", - c.image_name, - self.feature_key) - raise CrawlError("%s does not have expected name for %s (name=%s)", - container_id, - self.feature_key, - c.image_name) - - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() - - # crawl all candidate ports + port = None + + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) + + else: + ports = c.get_container_ports() + for each_port in ports: - try: - metrics = db2_crawler.retrieve_metrics( - host=ip, - user=user, - password=password, - db=db, - ) - except CrawlError: - logger.error("can't find metrics endpoint at %s db %s", - ip, db) - continue - return [(self.feature_key, metrics, self.feature_type)] + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port - raise CrawlError("%s has no accessible endpoint for %s", - container_id, - self.feature_key) + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break + + try: + metrics = db2_crawler.retrieve_metrics( + host=ip, + user=user, + password=password, + db=db, + ) + return [(self.feature_key, metrics, self.feature_type)] + except: + logger.info("db2 does not listen on port:%d", port) + raise ConnectionError("db2 does not listen on port:%d", port) diff --git a/crawler/plugins/applications/liberty/liberty_container_crawler.py b/crawler/plugins/applications/liberty/liberty_container_crawler.py index 098ec01a..4f723489 100644 --- a/crawler/plugins/applications/liberty/liberty_container_crawler.py +++ b/crawler/plugins/applications/liberty/liberty_container_crawler.py @@ -1,9 +1,11 @@ import logging - import dockercontainer from icrawl_plugin import IContainerCrawler from plugins.applications.liberty import liberty_crawler -from utils.crawler_exceptions import CrawlError +from utils.namespace import run_as_another_namespace +import json +import utils.misc +from requests.exceptions import ConnectionError logger = logging.getLogger('crawlutils') @@ -16,7 +18,7 @@ class LibertyContainerCrawler(IContainerCrawler): def get_feature(self): return self.feature_key - def crawl(self, container_id=None, **kwargs): + def get_opt(self, kwargs): password = "password" user = "user" @@ -26,32 +28,56 @@ def crawl(self, container_id=None, **kwargs): if "user" in kwargs: user = kwargs["user"] + return password, user + + def crawl(self, container_id=None, **kwargs): + + password, user = self.get_opt(kwargs) c = dockercontainer.DockerContainer(container_id) - # check image name - if c.image_name.find(self.feature_key) == -1: - logger.error("%s is not %s container", - c.image_name, - self.feature_key) - raise CrawlError("%s does not have expected name for %s (name=%s)", - container_id, - self.feature_key, - c.image_name) - - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() - - # crawl all candidate ports + port = None + + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) + + else: + ports = c.get_container_ports() + for each_port in ports: - return liberty_crawler.retrieve_metrics( - host=ip, - port=each_port, - user=user, - password=password, - feature_type=self.feature_type - ) - - raise CrawlError("%s has no accessible endpoint for %s", - container_id, - self.feature_key) + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port + + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break + + try: + return liberty_crawler.retrieve_metrics( + host=ip, + port=port, + user=user, + password=password, + feature_type=self.feature_type) + except: + logger.info("liberty does not listen on port:%d", port) + raise ConnectionError("liberty does not listen on port:%d", port) diff --git a/crawler/plugins/applications/nginx/nginx_container_crawler.py b/crawler/plugins/applications/nginx/nginx_container_crawler.py index 006ffab4..f666b2ee 100644 --- a/crawler/plugins/applications/nginx/nginx_container_crawler.py +++ b/crawler/plugins/applications/nginx/nginx_container_crawler.py @@ -1,9 +1,11 @@ import logging - import dockercontainer from icrawl_plugin import IContainerCrawler from plugins.applications.nginx import nginx_crawler -from utils.crawler_exceptions import CrawlError +from utils.namespace import run_as_another_namespace +from requests.exceptions import ConnectionError +import json +import utils.misc logger = logging.getLogger('crawlutils') @@ -11,6 +13,7 @@ class NginxContainerCrawler(IContainerCrawler): feature_type = 'application' feature_key = 'nginx' + default_port = 80 def get_feature(self): return self.feature_key @@ -18,31 +21,49 @@ def get_feature(self): def crawl(self, container_id=None, **kwargs): c = dockercontainer.DockerContainer(container_id) - # check image name - if c.image_name.find(self.feature_key) == -1: - logger.error("%s is not %s container", - c.image_name, - self.feature_key) - raise CrawlError("%s does not have expected name for %s (name=%s)", - container_id, - self.feature_key, - c.image_name) + port = None + + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) + + else: + ports = c.get_container_ports() - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() + for each_port in ports: + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port + + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break # crawl all candidate ports - for port in ports: - try: - metrics = nginx_crawler.retrieve_metrics(ip, port) - except CrawlError: - logger.error("can't find metrics endpoint at http://%s:%s", - ip, - port) - continue + try: + metrics = nginx_crawler.retrieve_metrics(ip, port) return [(self.feature_key, metrics, self.feature_type)] - - raise CrawlError("%s has no accessible endpoint for %s", - container_id, - self.feature_key) + except: + logger.error("can't find metrics endpoint at http://%s:%s", + ip, + port) + raise ConnectionError("can't find metrics endpoint" + "at http://%s:%s", ip, port) diff --git a/crawler/plugins/applications/redis/redis_container_crawler.py b/crawler/plugins/applications/redis/redis_container_crawler.py index f5d5314d..6e5d0608 100644 --- a/crawler/plugins/applications/redis/redis_container_crawler.py +++ b/crawler/plugins/applications/redis/redis_container_crawler.py @@ -3,12 +3,16 @@ import dockercontainer from requests.exceptions import ConnectionError import logging +from utils.namespace import run_as_another_namespace +import json +import utils.misc logger = logging.getLogger('crawlutils') class RedisContainerCrawler(IContainerCrawler): + ''' Crawling app provided metrics for redis container on docker. Usually redis listens on port 6379. @@ -18,40 +22,67 @@ class RedisContainerCrawler(IContainerCrawler): feature_key = "redis" default_port = 6379 + def get_port(self, c): + port = None + + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) + + else: + ports = c.get_container_ports() + + for each_port in ports: + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port + + return port + def get_feature(self): return self.feature_key def crawl(self, container_id=None, **kwargs): - import pip - pip.main(['install', 'redis']) - import redis + try: + import redis + except ImportError: + import pip + pip.main(['install', 'redis']) + import redis # only crawl redis container. Otherwise, quit. c = dockercontainer.DockerContainer(container_id) - if c.image_name.find(self.feature_key) == -1: - logger.debug("%s is not %s container" % - (c.image_name, self.feature_key)) - raise NameError("this is not target crawl container") - - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() - - # set default port number - if len(ports) == 0: - ports.append(self.default_port) - - # query to all available ports - for port in ports: - client = redis.Redis(host=ip, port=port) - try: - metrics = client.info() - except ConnectionError: - logger.info("redis does not listen on port:%d", port) - continue + port = self.get_port(c) + + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break + + client = redis.Redis(host=ip, port=port) + + try: + metrics = client.info() feature_attributes = feature.create_feature(metrics) return [(self.feature_key, feature_attributes, self.feature_type)] - - # any ports are not available - raise ConnectionError("no listen ports") + except: + logger.info("redis does not listen on port:%d", port) + raise ConnectionError("no listen at %d", port) diff --git a/crawler/plugins/applications/tomcat/tomcat_container_crawler.py b/crawler/plugins/applications/tomcat/tomcat_container_crawler.py index 69fea3d0..8bc484b7 100644 --- a/crawler/plugins/applications/tomcat/tomcat_container_crawler.py +++ b/crawler/plugins/applications/tomcat/tomcat_container_crawler.py @@ -1,9 +1,11 @@ import logging - import dockercontainer from icrawl_plugin import IContainerCrawler from plugins.applications.tomcat import tomcat_crawler -from utils.crawler_exceptions import CrawlError +from utils.namespace import run_as_another_namespace +import json +import utils.misc +from requests.exceptions import ConnectionError logger = logging.getLogger('crawlutils') @@ -16,7 +18,7 @@ class TomcatContainerCrawler(IContainerCrawler): def get_feature(self): return self.feature_key - def crawl(self, container_id=None, **kwargs): + def get_opt(self, kwargs): password = "password" user = "tomcat" @@ -26,32 +28,58 @@ def crawl(self, container_id=None, **kwargs): if "user" in kwargs: user = kwargs["user"] + return password, user + + def crawl(self, container_id=None, **kwargs): + + password, user = self.get_opt(kwargs) c = dockercontainer.DockerContainer(container_id) - # check image name - if c.image_name.find(self.feature_key) == -1: - logger.error("%s is not %s container", - c.image_name, - self.feature_key) - raise CrawlError("%s does not have expected name for %s (name=%s)", - container_id, - self.feature_key, - c.image_name) + port = None - # extract IP and Port information - ip = c.get_container_ip() - ports = c.get_container_ports() + if "annotation.io.kubernetes.container.ports" in\ + c.inspect['Config']['Labels']: + + ports = c.inspect['Config']['Labels'][ + 'annotation.io.kubernetes.container.ports'] + + ports = json.loads(ports) + + else: + ports = c.get_container_ports() - # crawl all candidate ports for each_port in ports: - return tomcat_crawler.retrieve_metrics( - host=ip, - port=each_port, - user=user, - password=password, - feature_type=self.feature_type - ) - - raise CrawlError("%s has no accessible endpoint for %s", - container_id, - self.feature_key) + tmp_port = None + if "containerPort" in each_port: + tmp_port = int(each_port['containerPort']) + else: + tmp_port = int(each_port) + + if tmp_port == self.default_port: + port = tmp_port + + if not port: + return + + state = c.inspect['State'] + pid = str(state['Pid']) + ips = run_as_another_namespace( + pid, ['net'], utils.misc.get_host_ip4_addresses) + + for each_ip in ips: + if each_ip != "127.0.0.1": + ip = each_ip + break + + # crawl all candidate ports + try: + return tomcat_crawler.retrieve_metrics( + host=ip, + port=port, + user=user, + password=password, + feature_type=self.feature_type) + except: + raise ConnectionError("%s has no accessible endpoint for %s", + container_id, + self.feature_key) diff --git a/tests/unit/test_app_apache.py b/tests/unit/test_app_apache.py index 83becfd8..d1a16c25 100644 --- a/tests/unit/test_app_apache.py +++ b/tests/unit/test_app_apache.py @@ -1,7 +1,5 @@ from unittest import TestCase - import mock - from plugins.applications.apache import apache_crawler from plugins.applications.apache.feature import ApacheFeature from plugins.applications.apache.apache_container_crawler \ @@ -9,10 +7,10 @@ from plugins.applications.apache.apache_host_crawler \ import ApacheHostCrawler from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError # expected format from apache status page - def mocked_wrong_status_page(host, port): return ('No Acceptable status page format') @@ -80,19 +78,36 @@ def read(self): ) -class MockedApacheContainer(object): +class MockedApacheContainer1(object): def __init__( self, container_id, ): - self.image_name = 'httpd-container' + ports = "[ {\"containerPort\" : \"80\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} - def get_container_ip(self): - return '1.2.3.4' + +class MockedApacheContainer2(object): + + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} + + def get_container_ports(self): + ports = ["80"] + return ports + + +class MockedApacheContainer3(object): + + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [80, 443] + ports = ["1234"] return ports @@ -225,35 +240,49 @@ def test_get_feature(self): @mock.patch('plugins.applications.apache.' 'apache_crawler.retrieve_status_page', mocked_retrieve_status_page) + @mock.patch(("plugins.applications.apache.apache_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) @mock.patch('dockercontainer.DockerContainer', - MockedApacheContainer) - def test_get_metrics(self): + MockedApacheContainer1) + def test_apache_container_crawler_forkube(self, *args): c = ApacheContainerCrawler() emitted = c.crawl()[0] self.assertEqual(emitted[0], 'apache') self.assertIsInstance(emitted[1], ApacheFeature) self.assertEqual(emitted[2], 'application') + @mock.patch('plugins.applications.apache.' + 'apache_crawler.retrieve_status_page', + mocked_retrieve_status_page) + @mock.patch(("plugins.applications.apache.apache_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) @mock.patch('dockercontainer.DockerContainer', - MockedNoPortContainer) - def test_no_available_port(self): + MockedApacheContainer2) + def test_apache_container_crawler_docker(self, *args): c = ApacheContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + emitted = c.crawl()[0] + self.assertEqual(emitted[0], 'apache') + self.assertIsInstance(emitted[1], ApacheFeature) + self.assertEqual(emitted[2], 'application') @mock.patch('dockercontainer.DockerContainer', - MockedNoNameContainer) - def test_none_apache_container(self): + MockedApacheContainer3) + def test_no_available_ports(self): c = ApacheContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + c.crawl() + pass @mock.patch('plugins.applications.apache.' 'apache_crawler.retrieve_status_page', mocked_no_status_page) + @mock.patch(("plugins.applications.apache.apache_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) @mock.patch('dockercontainer.DockerContainer', - MockedApacheContainer) - def test_no_accessible_endpoint(self): + MockedApacheContainer1) + def test_no_accessible_endpoint(self, *kwargs): c = ApacheContainerCrawler() - with self.assertRaises(CrawlError): + with self.assertRaises(ConnectionError): c.crawl("mockcontainer") diff --git a/tests/unit/test_app_db2.py b/tests/unit/test_app_db2.py index 483f6c6c..71afe85b 100644 --- a/tests/unit/test_app_db2.py +++ b/tests/unit/test_app_db2.py @@ -8,46 +8,39 @@ from plugins.applications.db2.db2_host_crawler \ import DB2HostCrawler from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError pip.main(['install', 'ibm_db']) -class MockedNoNameContainer(object): +class MockedDB2Container1(object): def __init__(self, container_id): - self.image_name = 'dummy' + ports = "[ {\"containerPort\" : \"50000\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} -class MockedNoPortContainer(object): +class MockedDB2Container2(object): - def __init__( - self, - container_id, - ): - self.image_name = 'db2' - - def get_container_ip(self): - return '1.2.3.4' + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [] + ports = ["50000"] return ports -class MockedDB2Container(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'db2' +class MockedDB2Container3(object): - def get_container_ip(self): - return '1.2.3.4' + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [500001, 500000] + ports = ["1234"] return ports @@ -200,35 +193,52 @@ def test_get_feature(self): 'db2_crawler.retrieve_metrics', mocked_retrieve_metrics) @mock.patch('dockercontainer.DockerContainer', - MockedDB2Container) - def test_get_metrics(self): + MockedDB2Container1) + @mock.patch(("plugins.applications.db2.db2_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_db2_container_crawler_forkube(self, *kwargs): c = DB2ContainerCrawler() options = {"password": "password", "user": "db2inst1", "db": "sample"} - emitted = c.crawl(**options)[0] + emitted = c.crawl(1234, **options)[0] self.assertEqual(emitted[0], 'db2') self.assertIsInstance(emitted[1], DB2Feature) self.assertEqual(emitted[2], 'application') + @mock.patch('plugins.applications.db2.' + 'db2_crawler.retrieve_metrics', + mocked_retrieve_metrics) @mock.patch('dockercontainer.DockerContainer', - MockedNoPortContainer) - def test_no_available_port(self): + MockedDB2Container2) + @mock.patch(("plugins.applications.db2.db2_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_db2_container_crawler_fordocker(self, *kwargs): c = DB2ContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + options = {"password": "password", "user": "db2inst1", "db": "sample"} + emitted = c.crawl(1234, **options)[0] + self.assertEqual(emitted[0], 'db2') + self.assertIsInstance(emitted[1], DB2Feature) + self.assertEqual(emitted[2], 'application') @mock.patch('dockercontainer.DockerContainer', - MockedNoNameContainer) - def test_none_apache_container(self): + MockedDB2Container3) + def test_no_available_port(self): c = DB2ContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + c.crawl("mockcontainer") + pass @mock.patch('plugins.applications.db2.' 'db2_crawler.retrieve_metrics', mocked_retrieve_metrics_error) @mock.patch('dockercontainer.DockerContainer', - MockedDB2Container) - def test_no_accessible_endpoint(self): + MockedDB2Container2) + @mock.patch(("plugins.applications.db2.db2_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_no_accessible_endpoint(self, *args): c = DB2ContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + with self.assertRaises(ConnectionError): + options = {"password": "password", + "user": "db2inst1", "db": "sample"} + c.crawl(1234, **options)[0] diff --git a/tests/unit/test_app_liberty.py b/tests/unit/test_app_liberty.py index 2a9a5ad9..ba7291d4 100644 --- a/tests/unit/test_app_liberty.py +++ b/tests/unit/test_app_liberty.py @@ -1,5 +1,4 @@ from unittest import TestCase - import mock from plugins.applications.liberty import liberty_crawler from plugins.applications.liberty import feature @@ -8,12 +7,47 @@ from plugins.applications.liberty.liberty_host_crawler \ import LibertyHostCrawler from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError def mocked_urllib2_open(request): return MockedURLResponse() +def mock_status_value(user, password, url): + raise CrawlError + + +class MockedLibertyContainer1(object): + + def __init__(self, container_id): + ports = "[ {\"containerPort\" : \"9443\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} + + +class MockedLibertyContainer2(object): + + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} + + def get_container_ports(self): + ports = ["9443"] + return ports + + +class MockedLibertyContainer3(object): + + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} + + def get_container_ports(self): + ports = ["1234"] + return ports + + class MockedURLResponse(object): def read(self): return open('tests/unit/liberty_response_time_details_mocked', @@ -74,44 +108,6 @@ def server_status_value(user, password, url): return return_value.get(last_word) -class MockedLibertyContainer(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'liberty' - - def get_container_ip(self): - return '1.2.3.4' - - def get_container_ports(self): - ports = [8080, 443] - return ports - - -class MockedNoPortContainer(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'liberty' - - def get_container_ip(self): - return '1.2.3.4' - - def get_container_ports(self): - ports = [] - return ports - - -class MockedNoNameContainer(object): - - def __init__(self, container_id): - self.image_name = 'dummy' - - class LibertyCrawlTests(TestCase): def setUp(self): @@ -220,24 +216,49 @@ def test_get_feature(self): 'liberty_crawler.retrieve_status_page', server_status_value) @mock.patch('dockercontainer.DockerContainer', - MockedLibertyContainer) - def test_get_metrics(self): + MockedLibertyContainer1) + @mock.patch(("plugins.applications.liberty.liberty_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_liberty_container_forkube(self, *args): c = LibertyContainerCrawler() options = {"password": "password", "user": "liberty"} emitted = list(c.crawl(**options)) self.assertEqual(emitted[0][0], 'liberty_servlet_status') self.assertEqual(emitted[0][2], 'application') + @mock.patch('plugins.applications.liberty.' + 'liberty_crawler.retrieve_status_page', + server_status_value) @mock.patch('dockercontainer.DockerContainer', - MockedNoPortContainer) - def test_no_available_port(self): + MockedLibertyContainer2) + @mock.patch(("plugins.applications.liberty.liberty_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_liberty_container_fordocker(self, *args): c = LibertyContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + options = {"password": "password", "user": "liberty"} + emitted = list(c.crawl(**options)) + self.assertEqual(emitted[0][0], 'liberty_servlet_status') + self.assertEqual(emitted[0][2], 'application') @mock.patch('dockercontainer.DockerContainer', - MockedNoNameContainer) - def test_none_liberty_container(self): + MockedLibertyContainer3) + def test_liberty_container_noport(self, *args): c = LibertyContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + c.crawl(1234) + pass + + @mock.patch('dockercontainer.DockerContainer', + MockedLibertyContainer1) + @mock.patch(("plugins.applications.liberty.liberty_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + @mock.patch('plugins.applications.liberty.' + 'liberty_crawler.retrieve_metrics', + mock_status_value) + def test_none_liberty_container(self, *args): + options = {"password": "password", "user": "liberty"} + c = LibertyContainerCrawler() + with self.assertRaises(ConnectionError): + c.crawl(1234, **options) diff --git a/tests/unit/test_app_nginx.py b/tests/unit/test_app_nginx.py index 214df2e5..197f3fd1 100644 --- a/tests/unit/test_app_nginx.py +++ b/tests/unit/test_app_nginx.py @@ -1,7 +1,5 @@ from unittest import TestCase - import mock - from plugins.applications.nginx import nginx_crawler from plugins.applications.nginx.feature import NginxFeature from plugins.applications.nginx.nginx_container_crawler \ @@ -9,6 +7,7 @@ from plugins.applications.nginx.nginx_host_crawler \ import NginxHostCrawler from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError # expected format from nginx status page @@ -21,7 +20,7 @@ def mocked_retrieve_status_page(host, port): def mocked_no_status_page(host, port): - #raise urllib2.HTTPError(1,2,3,4,5) + # raise urllib2.HTTPError(1,2,3,4,5) raise Exception @@ -43,42 +42,34 @@ def read(self): ) -class MockedNginxContainer(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'nginx-container' - - def get_container_ip(self): - return '1.2.3.4' - - def get_container_ports(self): - ports = [80, 443] - return ports +class MockedNginxContainer1(object): + def __init__(self, container_id): + ports = "[ {\"containerPort\" : \"80\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} -class MockedNoPortContainer(object): - def __init__( - self, - container_id, - ): - self.image_name = 'nginx-container' +class MockedNginxContainer2(object): - def get_container_ip(self): - return '1.2.3.4' + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [] + ports = ["80"] return ports -class MockedNoNameContainer(object): +class MockedNginxContainer3(object): def __init__(self, container_id): - self.image_name = 'dummy' + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} + + def get_container_ports(self): + ports = ["1234"] + return ports class NginxCrawlTests(TestCase): @@ -156,34 +147,48 @@ def test_get_feature(self): 'nginx_crawler.retrieve_status_page', mocked_retrieve_status_page) @mock.patch('dockercontainer.DockerContainer', - MockedNginxContainer) - def test_get_metrics(self): + MockedNginxContainer1) + @mock.patch(("plugins.applications.nginx.nginx_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_nginx_container_forkube(self, *args): c = NginxContainerCrawler() emitted = c.crawl()[0] self.assertEqual(emitted[0], 'nginx') self.assertIsInstance(emitted[1], NginxFeature) self.assertEqual(emitted[2], 'application') + @mock.patch('plugins.applications.nginx.' + 'nginx_crawler.retrieve_status_page', + mocked_retrieve_status_page) @mock.patch('dockercontainer.DockerContainer', - MockedNoPortContainer) - def test_no_available_port(self): + MockedNginxContainer2) + @mock.patch(("plugins.applications.nginx.nginx_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_nginx_container_fordocker(self, *args): c = NginxContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + emitted = c.crawl()[0] + self.assertEqual(emitted[0], 'nginx') + self.assertIsInstance(emitted[1], NginxFeature) + self.assertEqual(emitted[2], 'application') @mock.patch('dockercontainer.DockerContainer', - MockedNoNameContainer) - def test_none_nginx_container(self): + MockedNginxContainer3) + def test_nginx_container_noport(self, *args): c = NginxContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + c.crawl(1234) + pass @mock.patch('plugins.applications.nginx.' 'nginx_crawler.retrieve_status_page', mocked_no_status_page) @mock.patch('dockercontainer.DockerContainer', - MockedNginxContainer) - def test_no_accessible_endpoint(self): + MockedNginxContainer2) + @mock.patch(("plugins.applications.nginx.nginx_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_no_accessible_endpoint(self, *arg): c = NginxContainerCrawler() - with self.assertRaises(CrawlError): + with self.assertRaises(ConnectionError): c.crawl("mockcontainer") diff --git a/tests/unit/test_app_redis.py b/tests/unit/test_app_redis.py index d3d12833..12a6187e 100644 --- a/tests/unit/test_app_redis.py +++ b/tests/unit/test_app_redis.py @@ -9,12 +9,10 @@ import RedisContainerCrawler from requests.exceptions import ConnectionError - pip.main(['install', 'redis']) class MockedRedisClient(object): - def __init__(self, host='localhost', port=6379): self.host = host self.port = port @@ -132,30 +130,17 @@ def info(self): class MockedRedisContainer1(object): - def __init__( - self, - container_id, - ): - self.image_name = 'redis-container' - - def get_container_ip(self): - return "1.2.3.4" - - def get_container_ports(self): - ports = ["6379"] - return ports + def __init__(self, container_id): + ports = "[ {\"containerPort\" : \"6379\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} class MockedRedisContainer2(object): - def __init__( - self, - container_id, - ): - self.image_name = 'dummy' - - def get_container_ip(self): - return "1.2.3.4" + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): ports = ["6379"] @@ -164,33 +149,12 @@ def get_container_ports(self): class MockedRedisContainer3(object): - def __init__( - self, - container_id, - ): - self.image_name = 'redis-no-ports' - - def get_container_ip(self): - return "1.2.3.4" + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [] - return ports - - -class MockedRedisContainer4(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'redis-no-ports' - - def get_container_ip(self): - return "1.2.3.4" - - def get_container_ports(self): - ports = ["80", "8080"] + ports = ["1234"] return ports @@ -222,8 +186,11 @@ def test_get_feature(self): @mock.patch('dockercontainer.DockerContainer', MockedRedisContainer1) + @mock.patch(("plugins.applications.redis.redis_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) @mock.patch('redis.Redis', MockedRedisClient) - def test_redis_container_crawler(self): + def test_redis_container_crawler_forkube(self, *args): c = RedisContainerCrawler() emitted_tuple = c.crawl("mockcontainerid")[0] self.assertEqual(emitted_tuple[0], "redis", @@ -234,27 +201,37 @@ def test_redis_container_crawler(self): @mock.patch('dockercontainer.DockerContainer', MockedRedisContainer2) - def test_none_redis_container_crawler(self): + @mock.patch('redis.Redis', MockedRedisClient) + @mock.patch(("plugins.applications.redis.redis_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_redis_container_crawler_fordocker(self, *args): c = RedisContainerCrawler() - with self.assertRaises(NameError): - c.crawl("mockcontainerid") + emitted_tuple = c.crawl("mockcontainerid")[0] + self.assertEqual(emitted_tuple[0], "redis", + "feature key must be equal to redis") + self.assertIsInstance(emitted_tuple[1], RedisFeature) + self.assertEqual(emitted_tuple[2], "application", + "feature type must be equal to application") @mock.patch('dockercontainer.DockerContainer', - MockedRedisContainer4) - @mock.patch('redis.Redis', MockedRedisClient2) + MockedRedisContainer3) + @mock.patch('redis.Redis', MockedRedisClient) def test_no_available_ports(self): c = RedisContainerCrawler() - with self.assertRaises(ConnectionError): - c.crawl("mockcontainerid") + c.crawl(1234) + pass @mock.patch('dockercontainer.DockerContainer', - MockedRedisContainer3) - @mock.patch('redis.Redis', MockedRedisClient) - def test_set_default_port(self): + MockedRedisContainer2) + @mock.patch('redis.Redis', MockedRedisClient2) + @mock.patch(("plugins.applications.redis.redis_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_redis_container_no_connect(self, *args): c = RedisContainerCrawler() - emitted_tuple = c.crawl("mockcontainerid")[0] - self.assertEqual(emitted_tuple[0], "redis", - "feature key must be equal to redis") + with self.assertRaises(ConnectionError): + c.crawl(1234) class RedisHostCrawlTests(TestCase): diff --git a/tests/unit/test_app_tomcat.py b/tests/unit/test_app_tomcat.py index 702882fb..2338e70b 100644 --- a/tests/unit/test_app_tomcat.py +++ b/tests/unit/test_app_tomcat.py @@ -1,7 +1,5 @@ from unittest import TestCase - import mock - from plugins.applications.tomcat import tomcat_crawler from plugins.applications.tomcat import feature from plugins.applications.tomcat.tomcat_container_crawler \ @@ -9,6 +7,7 @@ from plugins.applications.tomcat.tomcat_host_crawler \ import TomcatHostCrawler from utils.crawler_exceptions import CrawlError +from requests.exceptions import ConnectionError def mocked_urllib2_open(request): @@ -19,6 +18,10 @@ def mocked_retrieve_status_page(host, port, user, password): return server_status_value() +def mock_status_value(host, user, password, url): + raise CrawlError + + def server_status_value(): return ('' '' @@ -69,42 +72,34 @@ def read(self): return server_status_value() -class MockedTomcatContainer(object): - - def __init__( - self, - container_id, - ): - self.image_name = 'tomcat' - - def get_container_ip(self): - return '1.2.3.4' - - def get_container_ports(self): - ports = [8080, 443] - return ports +class MockedTomcatContainer1(object): + def __init__(self, container_id): + ports = "[ {\"containerPort\" : \"8080\"} ]" + self.inspect = {"State": {"Pid": 1234}, "Config": {"Labels": + {"annotation.io.kubernetes.container.ports": ports}}} -class MockedNoPortContainer(object): - def __init__( - self, - container_id, - ): - self.image_name = 'tomcat' +class MockedTomcatContainer2(object): - def get_container_ip(self): - return '1.2.3.4' + def __init__(self, container_id): + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} def get_container_ports(self): - ports = [] + ports = ["8080"] return ports -class MockedNoNameContainer(object): +class MockedTomcatContainer3(object): def __init__(self, container_id): - self.image_name = 'dummy' + self.inspect = {"State": {"Pid": 1234}, + "Config": {"Labels": {"dummy": "dummy"}}} + + def get_container_ports(self): + ports = ["1234"] + return ports class TomcatCrawlTests(TestCase): @@ -252,24 +247,49 @@ def test_get_feature(self): 'tomcat_crawler.retrieve_status_page', mocked_retrieve_status_page) @mock.patch('dockercontainer.DockerContainer', - MockedTomcatContainer) - def test_get_metrics(self): + MockedTomcatContainer1) + @mock.patch(("plugins.applications.tomcat.tomcat_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_tomcat_container_forkube(self, *args): c = TomcatContainerCrawler() options = {"password": "password", "user": "tomcat"} emitted = list(c.crawl(**options)) self.assertEqual(emitted[0][0], 'tomcat_jvm') self.assertEqual(emitted[0][2], 'application') + @mock.patch('plugins.applications.tomcat.' + 'tomcat_crawler.retrieve_status_page', + mocked_retrieve_status_page) @mock.patch('dockercontainer.DockerContainer', - MockedNoPortContainer) - def test_no_available_port(self): + MockedTomcatContainer2) + @mock.patch(("plugins.applications.tomcat.tomcat_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + def test_tomcat_container_fordocker(self, *args): c = TomcatContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + options = {"password": "password", "user": "tomcat"} + emitted = list(c.crawl(**options)) + self.assertEqual(emitted[0][0], 'tomcat_jvm') + self.assertEqual(emitted[0][2], 'application') @mock.patch('dockercontainer.DockerContainer', - MockedNoNameContainer) - def test_none_tomcat_container(self): + MockedTomcatContainer3) + def test_tomcat_container_noport(self, *args): c = TomcatContainerCrawler() - with self.assertRaises(CrawlError): - c.crawl("mockcontainer") + c.crawl(1234) + pass + + @mock.patch('dockercontainer.DockerContainer', + MockedTomcatContainer1) + @mock.patch(("plugins.applications.tomcat.tomcat_container_crawler." + "run_as_another_namespace"), + return_value=['127.0.0.1', '1.2.3.4']) + @mock.patch('plugins.applications.tomcat.' + 'tomcat_crawler.retrieve_metrics', + mock_status_value) + def test_none_tomcat_container(self, *args): + options = {"password": "password", "user": "tomcat"} + c = TomcatContainerCrawler() + with self.assertRaises(ConnectionError): + c.crawl(1234, **options)