Skip to content

Add a function to get ip address for metrics crawler in k8s environment #335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 23, 2017
Merged

Add a function to get ip address for metrics crawler in k8s environment #335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 45 additions & 31 deletions crawler/plugins/applications/apache/apache_container_crawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
from utils.namespace import run_as_another_namespace
import logging


import json
import utils.misc
import dockercontainer
from icrawl_plugin import IContainerCrawler
from plugins.applications.apache import apache_crawler
from utils.crawler_exceptions import CrawlError
from requests.exceptions import ConnectionError

logger = logging.getLogger('crawlutils')


class ApacheContainerCrawler(IContainerCrawler):
feature_type = 'application'
feature_key = 'apache'
default_port = 80

def get_feature(self):
return self.feature_key
Expand All @@ -20,32 +22,44 @@ def crawl(self, container_id=None, **kwargs):

c = dockercontainer.DockerContainer(container_id)

# check image name
if c.image_name.find("httpd") == -1:

logger.error("%s is not %s container",
c.image_name,
self.feature_key)
raise CrawlError("%s does not have expected name for %s (name=%s)",
container_id,
self.feature_key,
c.image_name)

# extract IP and Port information
ip = c.get_container_ip()
ports = c.get_container_ports()

# crawl all candidate ports
for port in ports:
try:
metrics = apache_crawler.retrieve_metrics(ip, port)
except CrawlError:
logger.error("can't find metrics endpoint at http://%s:%s",
ip,
port)
continue
return [(self.feature_key, metrics, self.feature_type)]
port = None

if "annotation.io.kubernetes.container.ports" in\
c.inspect['Config']['Labels']:

ports = c.inspect['Config']['Labels'][
'annotation.io.kubernetes.container.ports']

ports = json.loads(ports)

raise CrawlError("%s has no accessible endpoint for %s",
container_id,
self.feature_key)
else:
ports = c.get_container_ports()

for each_port in ports:
tmp_port = None
if "containerPort" in each_port:
tmp_port = int(each_port['containerPort'])
else:
tmp_port = int(each_port)

if tmp_port == self.default_port:
port = tmp_port

if not port:
return

state = c.inspect['State']
pid = str(state['Pid'])
ips = run_as_another_namespace(
pid, ['net'], utils.misc.get_host_ip4_addresses)

for each_ip in ips:
if each_ip != "127.0.0.1":
ip = each_ip
break
try:
metrics = apache_crawler.retrieve_metrics(ip, port)
return [(self.feature_key, metrics, self.feature_type)]
except:
logger.info("apache does not listen on port:%d", port)
raise ConnectionError("apache does not listen on port:%d", port)
89 changes: 56 additions & 33 deletions crawler/plugins/applications/db2/db2_container_crawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
import logging

import dockercontainer
from icrawl_plugin import IContainerCrawler
from plugins.applications.db2 import db2_crawler
from utils.crawler_exceptions import CrawlError
from utils.namespace import run_as_another_namespace
import json
import utils.misc
from requests.exceptions import ConnectionError

logger = logging.getLogger('crawlutils')


class DB2ContainerCrawler(IContainerCrawler):
feature_type = 'application'
feature_key = 'db2'
default_port = 50000

def get_feature(self):
return self.feature_key

def crawl(self, container_id=None, **kwargs):
def get_opt(self, kwargs):
password = "db2inst1"
user = "db2inst1-pwd"
db = "sample"
Expand All @@ -29,37 +32,57 @@ def crawl(self, container_id=None, **kwargs):
if "db" in kwargs:
db = kwargs["db"]

return password, user, db

def crawl(self, container_id=None, **kwargs):

password, user, db = self.get_opt(kwargs)
c = dockercontainer.DockerContainer(container_id)

# check image name
if c.image_name.find(self.feature_key) == -1:
logger.error("%s is not %s container",
c.image_name,
self.feature_key)
raise CrawlError("%s does not have expected name for %s (name=%s)",
container_id,
self.feature_key,
c.image_name)

# extract IP and Port information
ip = c.get_container_ip()
ports = c.get_container_ports()

# crawl all candidate ports
port = None

if "annotation.io.kubernetes.container.ports" in\
c.inspect['Config']['Labels']:

ports = c.inspect['Config']['Labels'][
'annotation.io.kubernetes.container.ports']

ports = json.loads(ports)

else:
ports = c.get_container_ports()

for each_port in ports:
try:
metrics = db2_crawler.retrieve_metrics(
host=ip,
user=user,
password=password,
db=db,
)
except CrawlError:
logger.error("can't find metrics endpoint at %s db %s",
ip, db)
continue
return [(self.feature_key, metrics, self.feature_type)]
tmp_port = None
if "containerPort" in each_port:
tmp_port = int(each_port['containerPort'])
else:
tmp_port = int(each_port)

if tmp_port == self.default_port:
port = tmp_port

raise CrawlError("%s has no accessible endpoint for %s",
container_id,
self.feature_key)
if not port:
return

state = c.inspect['State']
pid = str(state['Pid'])
ips = run_as_another_namespace(
pid, ['net'], utils.misc.get_host_ip4_addresses)

for each_ip in ips:
if each_ip != "127.0.0.1":
ip = each_ip
break

try:
metrics = db2_crawler.retrieve_metrics(
host=ip,
user=user,
password=password,
db=db,
)
return [(self.feature_key, metrics, self.feature_type)]
except:
logger.info("db2 does not listen on port:%d", port)
raise ConnectionError("db2 does not listen on port:%d", port)
84 changes: 55 additions & 29 deletions crawler/plugins/applications/liberty/liberty_container_crawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import logging

import dockercontainer
from icrawl_plugin import IContainerCrawler
from plugins.applications.liberty import liberty_crawler
from utils.crawler_exceptions import CrawlError
from utils.namespace import run_as_another_namespace
import json
import utils.misc
from requests.exceptions import ConnectionError

logger = logging.getLogger('crawlutils')

Expand All @@ -16,7 +18,7 @@ class LibertyContainerCrawler(IContainerCrawler):
def get_feature(self):
return self.feature_key

def crawl(self, container_id=None, **kwargs):
def get_opt(self, kwargs):
password = "password"
user = "user"

Expand All @@ -26,32 +28,56 @@ def crawl(self, container_id=None, **kwargs):
if "user" in kwargs:
user = kwargs["user"]

return password, user

def crawl(self, container_id=None, **kwargs):

password, user = self.get_opt(kwargs)
c = dockercontainer.DockerContainer(container_id)

# check image name
if c.image_name.find(self.feature_key) == -1:
logger.error("%s is not %s container",
c.image_name,
self.feature_key)
raise CrawlError("%s does not have expected name for %s (name=%s)",
container_id,
self.feature_key,
c.image_name)

# extract IP and Port information
ip = c.get_container_ip()
ports = c.get_container_ports()

# crawl all candidate ports
port = None

if "annotation.io.kubernetes.container.ports" in\
c.inspect['Config']['Labels']:

ports = c.inspect['Config']['Labels'][
'annotation.io.kubernetes.container.ports']

ports = json.loads(ports)

else:
ports = c.get_container_ports()

for each_port in ports:
return liberty_crawler.retrieve_metrics(
host=ip,
port=each_port,
user=user,
password=password,
feature_type=self.feature_type
)

raise CrawlError("%s has no accessible endpoint for %s",
container_id,
self.feature_key)
tmp_port = None
if "containerPort" in each_port:
tmp_port = int(each_port['containerPort'])
else:
tmp_port = int(each_port)

if tmp_port == self.default_port:
port = tmp_port

if not port:
return

state = c.inspect['State']
pid = str(state['Pid'])
ips = run_as_another_namespace(
pid, ['net'], utils.misc.get_host_ip4_addresses)

for each_ip in ips:
if each_ip != "127.0.0.1":
ip = each_ip
break

try:
return liberty_crawler.retrieve_metrics(
host=ip,
port=port,
user=user,
password=password,
feature_type=self.feature_type)
except:
logger.info("liberty does not listen on port:%d", port)
raise ConnectionError("liberty does not listen on port:%d", port)
Loading