From d2c4a41dc9238715d15c61b78ee52f48bbcd59c7 Mon Sep 17 00:00:00 2001 From: Leandro Carneiro Date: Fri, 30 Jul 2021 10:04:57 -0300 Subject: [PATCH] feat: Include tags in metrics labels --- kong/plugins/prometheus/exporter.lua | 50 +- kong/plugins/prometheus/handler.lua | 6 + kong/plugins/prometheus/schema.lua | 2 + spec/02-access_spec.lua | 797 ++++++++++++++++++++++++++- spec/03-custom-serve_spec.lua | 2 +- spec/04-status_api_spec.lua | 8 +- 6 files changed, 834 insertions(+), 31 deletions(-) diff --git a/kong/plugins/prometheus/exporter.lua b/kong/plugins/prometheus/exporter.lua index f36369e..1ba5f83 100644 --- a/kong/plugins/prometheus/exporter.lua +++ b/kong/plugins/prometheus/exporter.lua @@ -84,25 +84,25 @@ local function init() if kong_subsystem == "http" then metrics.status = prometheus:counter("http_status", "HTTP status codes per service/route in Kong", - {"service", "route", "code"}) + {"service", "route", "code", "service_tags", "route_tags"}) else metrics.status = prometheus:counter("stream_status", "Stream status codes per service/route in Kong", - {"service", "route", "code"}) + {"service", "route", "code", "service_tags", "route_tags"}) end metrics.latency = prometheus:histogram("latency", "Latency added by Kong, total " .. "request time and upstream latency " .. "for each service/route in Kong", - {"service", "route", "type"}, + {"service", "route", "type", "service_tags", "route_tags"}, DEFAULT_BUCKETS) -- TODO make this configurable metrics.bandwidth = prometheus:counter("bandwidth", "Total bandwidth in bytes " .. "consumed per service/route in Kong", - {"service", "route", "type"}) + {"service", "route", "type", "service_tags", "route_tags"}) metrics.consumer_status = prometheus:counter("http_consumer_status", "HTTP status codes for customer per service/route in Kong", - {"service", "route", "code", "consumer"}) + {"service", "route", "code", "consumer", "service_tags", "route_tags"}) if enterprise then enterprise.init(prometheus) @@ -137,8 +137,8 @@ end -- Since in the prometheus library we create a new table for each diverged label -- so putting the "more dynamic" label at the end will save us some memory -local labels_table = {0, 0, 0} -local labels_table4 = {0, 0, 0, 0} +local labels_table = {0, 0, 0, 0, 0} +local labels_table6 = {0, 0, 0, 0, 0, 0} local upstream_target_addr_health_table = { { value = 0, labels = { 0, 0, 0, "healthchecks_off" } }, { value = 0, labels = { 0, 0, 0, "healthy" } }, @@ -168,22 +168,30 @@ if kong_subsystem == "http" then return end - local service_name + local service_name, service_tags if message and message.service then service_name = message.service.name or message.service.host + service_tags = ((message.service.tags) and (type(message.service.tags) == "table")) + and concat(message.service.tags, ",") + or "" else -- do not record any stats if the service is not present return end - local route_name + local route_name, route_tags if message and message.route then route_name = message.route.name or message.route.id + route_tags = ((message.route.tags) and (type(message.route.tags) == "table")) + and concat(message.route.tags, ",") + or "" end labels_table[1] = service_name labels_table[2] = route_name labels_table[3] = message.response.status + labels_table[4] = service_tags + labels_table[5] = route_tags metrics.status:inc(1, labels_table) local request_size = tonumber(message.request.size) @@ -217,11 +225,13 @@ if kong_subsystem == "http" then end if serialized.consumer ~= nil then - labels_table4[1] = labels_table[1] - labels_table4[2] = labels_table[2] - labels_table4[3] = message.response.status - labels_table4[4] = serialized.consumer - metrics.consumer_status:inc(1, labels_table4) + labels_table6[1] = labels_table[1] + labels_table6[2] = labels_table[2] + labels_table6[3] = message.response.status + labels_table6[4] = serialized.consumer + labels_table6[5] = labels_table[4] + labels_table6[6] = labels_table[5] + metrics.consumer_status:inc(1, labels_table6) end end @@ -234,22 +244,30 @@ else return end - local service_name + local service_name, service_tags if message and message.service then service_name = message.service.name or message.service.host + service_tags = ((message.service.tags) and (type(message.service.tags) == "table")) + and concat(message.service.tags, ",") + or "" else -- do not record any stats if the service is not present return end - local route_name + local route_name, route_tags if message and message.route then route_name = message.route.name or message.route.id + route_tags = ((message.route.tags) and (type(message.route.tags) == "table")) + and concat(message.route.tags, ",") + or "" end labels_table[1] = service_name labels_table[2] = route_name labels_table[3] = message.session.status + labels_table[4] = service_tags + labels_table[5] = route_tags metrics.status:inc(1, labels_table) local ingress_size = tonumber(message.session.received) diff --git a/kong/plugins/prometheus/handler.lua b/kong/plugins/prometheus/handler.lua index 6afa8b3..cfce763 100644 --- a/kong/plugins/prometheus/handler.lua +++ b/kong/plugins/prometheus/handler.lua @@ -22,6 +22,12 @@ function PrometheusHandler.log(self, conf) if conf.per_consumer and message.consumer ~= nil then serialized.consumer = message.consumer.username end + if not conf.expose_services_tags and message.service ~= nil then + message.service.tags = {} + end + if not conf.expose_routes_tags and message.route ~= nil then + message.route.tags = {} + end prometheus.log(message, serialized) end diff --git a/kong/plugins/prometheus/schema.lua b/kong/plugins/prometheus/schema.lua index d749077..d463cff 100644 --- a/kong/plugins/prometheus/schema.lua +++ b/kong/plugins/prometheus/schema.lua @@ -14,6 +14,8 @@ return { type = "record", fields = { { per_consumer = { type = "boolean", default = false }, }, + { expose_services_tags = { type = "boolean", default = false }, }, + { expose_routes_tags = { type = "boolean", default = false }, }, }, custom_validator = validate_shared_dict, }, }, diff --git a/spec/02-access_spec.lua b/spec/02-access_spec.lua index ad09512..7806afd 100644 --- a/spec/02-access_spec.lua +++ b/spec/02-access_spec.lua @@ -123,7 +123,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_status{service="mock-service",route="http-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) res = assert(proxy_client:send { @@ -143,7 +143,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_status{service="mock-service",route="http-route",code="400"} 1', nil, true) + return body:find('kong_http_status{service="mock-service",route="http-route",code="400",service_tags="",route_tags=""} 1', nil, true) end) end) @@ -168,7 +168,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) ok, resp = proxy_client_grpcs({ @@ -191,7 +191,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) end) @@ -212,7 +212,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_stream_status{service="tcp-service",route="tcp-route",code="200"} 1', nil, true) + return body:find('kong_stream_status{service="tcp-service",route="tcp-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) end) @@ -314,6 +314,645 @@ describe("Plugin: prometheus (access)", function() end) end) +describe("Plugin: prometheus (access) exposing tags", function() + local proxy_client + local admin_client + local proxy_client_grpc + local proxy_client_grpcs + + setup(function() + local bp = helpers.get_db_utils() + + local service = bp.services:insert { + name = "mock-service", + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_port, + protocol = helpers.mock_upstream_protocol, + tags = {"service-http-tag1", "service-http-tag2"}, + } + + bp.routes:insert { + protocols = { "http" }, + name = "http-route", + paths = { "/" }, + methods = { "GET" }, + service = service, + tags = {"route-http-tag1", "route-http-tag2"}, + } + + local grpc_service = bp.services:insert { + name = "mock-grpc-service", + url = "grpc://grpcbin:9000", + tags = {"service-grpc-tag1", "service-grpc-tag2"}, + } + + bp.routes:insert { + protocols = { "grpc" }, + name = "grpc-route", + hosts = { "grpc" }, + service = grpc_service, + tags = {"route-grpc-tag1", "route-grpc-tag2"}, + } + + local grpcs_service = bp.services:insert { + name = "mock-grpcs-service", + url = "grpcs://grpcbin:9001", + tags = {"service-grpcs-tag1", "service-grpcs-tag2"}, + } + + bp.routes:insert { + protocols = { "grpcs" }, + name = "grpcs-route", + hosts = { "grpcs" }, + service = grpcs_service, + tags = {"route-grpcs-tag1", "route-grpcs-tag2"}, + } + + local tcp_service = bp.services:insert { + name = "tcp-service", + url = "tcp://127.0.0.1:" .. TCP_SERVICE_PORT, + tags = {"service-tcp-tag1", "service-tcp-tag2"}, + } + + bp.routes:insert { + protocols = { "tcp" }, + name = "tcp-route", + service = tcp_service, + destinations = { { port = TCP_PROXY_PORT } }, + tags = {"route-tcp-tag1", "route-tcp-tag2"}, + } + + bp.plugins:insert { + protocols = { "http", "https", "grpc", "grpcs", "tcp", "tls" }, + name = "prometheus", + config = { + expose_services_tags = true, + expose_routes_tags = true, + }, + } + + helpers.tcp_server(TCP_SERVICE_PORT) + assert(helpers.start_kong { + nginx_conf = nginx_conf, + plugins = "bundled, prometheus", + stream_listen = "127.0.0.1:" .. TCP_PROXY_PORT, + }) + proxy_client = helpers.proxy_client() + admin_client = helpers.admin_client() + proxy_client_grpc = helpers.proxy_client_grpc() + proxy_client_grpcs = helpers.proxy_client_grpcs() + end) + + teardown(function() + helpers.kill_tcp_server(TCP_SERVICE_PORT) + if proxy_client then + proxy_client:close() + end + if admin_client then + admin_client:close() + end + + helpers.stop_kong() + end) + + it("increments the count for proxied requests", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/status/200", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(200, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="service-http-tag1,service-http-tag2",route_tags="route-http-tag1,route-http-tag2"} 1', nil, true) + end) + + res = assert(proxy_client:send { + method = "GET", + path = "/status/400", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(400, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="400",service_tags="service-http-tag1,service-http-tag2",route_tags="route-http-tag1,route-http-tag2"} 1', nil, true) + end) + end) + + it("increments the count for proxied grpc requests", function() + local ok, resp = proxy_client_grpc({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpc", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200",service_tags="service-grpc-tag1,service-grpc-tag2",route_tags="route-grpc-tag1,route-grpc-tag2"} 1', nil, true) + end) + + ok, resp = proxy_client_grpcs({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpcs", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200",service_tags="service-grpcs-tag1,service-grpcs-tag2",route_tags="route-grpcs-tag1,route-grpcs-tag2"} 1', nil, true) + end) + end) + + pending("increments the count for proxied TCP streams", function() + local conn = assert(ngx.socket.connect("127.0.0.1", TCP_PROXY_PORT)) + + assert(conn:send("hi there!\n")) + local gotback = assert(conn:receive("*a")) + assert.equal("hi there!\n", gotback) + + conn:close() + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_stream_status{service="tcp-service",route="tcp-route",code="200",service_tags="service-tcp-tag1,service-tcp-tag2",route_tags="route-tcp-tag1,route-tcp-tag2"} 1', nil, true) + end) + end) +end) + +describe("Plugin: prometheus (access) exposing only route tags", function() + local proxy_client + local admin_client + local proxy_client_grpc + local proxy_client_grpcs + + setup(function() + local bp = helpers.get_db_utils() + + local service = bp.services:insert { + name = "mock-service", + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_port, + protocol = helpers.mock_upstream_protocol, + tags = {"service-http-tag1", "service-http-tag2"}, + } + + bp.routes:insert { + protocols = { "http" }, + name = "http-route", + paths = { "/" }, + methods = { "GET" }, + service = service, + tags = {"route-http-tag1", "route-http-tag2"}, + } + + local grpc_service = bp.services:insert { + name = "mock-grpc-service", + url = "grpc://grpcbin:9000", + tags = {"service-grpc-tag1", "service-grpc-tag2"}, + } + + bp.routes:insert { + protocols = { "grpc" }, + name = "grpc-route", + hosts = { "grpc" }, + service = grpc_service, + tags = {"route-grpc-tag1", "route-grpc-tag2"}, + } + + local grpcs_service = bp.services:insert { + name = "mock-grpcs-service", + url = "grpcs://grpcbin:9001", + tags = {"service-grpcs-tag1", "service-grpcs-tag2"}, + } + + bp.routes:insert { + protocols = { "grpcs" }, + name = "grpcs-route", + hosts = { "grpcs" }, + service = grpcs_service, + tags = {"route-grpcs-tag1", "route-grpcs-tag2"}, + } + + local tcp_service = bp.services:insert { + name = "tcp-service", + url = "tcp://127.0.0.1:" .. TCP_SERVICE_PORT, + tags = {"service-tcp-tag1", "service-tcp-tag2"}, + } + + bp.routes:insert { + protocols = { "tcp" }, + name = "tcp-route", + service = tcp_service, + destinations = { { port = TCP_PROXY_PORT } }, + tags = {"route-tcp-tag1", "route-tcp-tag2"}, + } + + bp.plugins:insert { + protocols = { "http", "https", "grpc", "grpcs", "tcp", "tls" }, + name = "prometheus", + config = { + expose_services_tags = false, + expose_routes_tags = true, + }, + } + + helpers.tcp_server(TCP_SERVICE_PORT) + assert(helpers.start_kong { + nginx_conf = nginx_conf, + plugins = "bundled, prometheus", + stream_listen = "127.0.0.1:" .. TCP_PROXY_PORT, + }) + proxy_client = helpers.proxy_client() + admin_client = helpers.admin_client() + proxy_client_grpc = helpers.proxy_client_grpc() + proxy_client_grpcs = helpers.proxy_client_grpcs() + end) + + teardown(function() + helpers.kill_tcp_server(TCP_SERVICE_PORT) + if proxy_client then + proxy_client:close() + end + if admin_client then + admin_client:close() + end + + helpers.stop_kong() + end) + + it("increments the count for proxied requests", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/status/200", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(200, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="",route_tags="route-http-tag1,route-http-tag2"} 1', nil, true) + end) + + res = assert(proxy_client:send { + method = "GET", + path = "/status/400", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(400, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="400",service_tags="",route_tags="route-http-tag1,route-http-tag2"} 1', nil, true) + end) + end) + + it("increments the count for proxied grpc requests", function() + local ok, resp = proxy_client_grpc({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpc", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200",service_tags="",route_tags="route-grpc-tag1,route-grpc-tag2"} 1', nil, true) + end) + + ok, resp = proxy_client_grpcs({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpcs", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200",service_tags="",route_tags="route-grpcs-tag1,route-grpcs-tag2"} 1', nil, true) + end) + end) + + pending("increments the count for proxied TCP streams", function() + local conn = assert(ngx.socket.connect("127.0.0.1", TCP_PROXY_PORT)) + + assert(conn:send("hi there!\n")) + local gotback = assert(conn:receive("*a")) + assert.equal("hi there!\n", gotback) + + conn:close() + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_stream_status{service="tcp-service",route="tcp-route",code="200",service_tags="",route_tags="route-tcp-tag1,route-tcp-tag2"} 1', nil, true) + end) + end) +end) + +describe("Plugin: prometheus (access) exposing only service tags", function() + local proxy_client + local admin_client + local proxy_client_grpc + local proxy_client_grpcs + + setup(function() + local bp = helpers.get_db_utils() + + local service = bp.services:insert { + name = "mock-service", + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_port, + protocol = helpers.mock_upstream_protocol, + tags = {"service-http-tag1", "service-http-tag2"}, + } + + bp.routes:insert { + protocols = { "http" }, + name = "http-route", + paths = { "/" }, + methods = { "GET" }, + service = service, + tags = {"route-http-tag1", "route-http-tag2"}, + } + + local grpc_service = bp.services:insert { + name = "mock-grpc-service", + url = "grpc://grpcbin:9000", + tags = {"service-grpc-tag1", "service-grpc-tag2"}, + } + + bp.routes:insert { + protocols = { "grpc" }, + name = "grpc-route", + hosts = { "grpc" }, + service = grpc_service, + tags = {"route-grpc-tag1", "route-grpc-tag2"}, + } + + local grpcs_service = bp.services:insert { + name = "mock-grpcs-service", + url = "grpcs://grpcbin:9001", + tags = {"service-grpcs-tag1", "service-grpcs-tag2"}, + } + + bp.routes:insert { + protocols = { "grpcs" }, + name = "grpcs-route", + hosts = { "grpcs" }, + service = grpcs_service, + tags = {"route-grpcs-tag1", "route-grpcs-tag2"}, + } + + local tcp_service = bp.services:insert { + name = "tcp-service", + url = "tcp://127.0.0.1:" .. TCP_SERVICE_PORT, + tags = {"service-tcp-tag1", "service-tcp-tag2"}, + } + + bp.routes:insert { + protocols = { "tcp" }, + name = "tcp-route", + service = tcp_service, + destinations = { { port = TCP_PROXY_PORT } }, + tags = {"route-tcp-tag1", "route-tcp-tag2"}, + } + + bp.plugins:insert { + protocols = { "http", "https", "grpc", "grpcs", "tcp", "tls" }, + name = "prometheus", + config = { + expose_services_tags = true, + expose_routes_tags = false, + }, + } + + helpers.tcp_server(TCP_SERVICE_PORT) + assert(helpers.start_kong { + nginx_conf = nginx_conf, + plugins = "bundled, prometheus", + stream_listen = "127.0.0.1:" .. TCP_PROXY_PORT, + }) + proxy_client = helpers.proxy_client() + admin_client = helpers.admin_client() + proxy_client_grpc = helpers.proxy_client_grpc() + proxy_client_grpcs = helpers.proxy_client_grpcs() + end) + + teardown(function() + helpers.kill_tcp_server(TCP_SERVICE_PORT) + if proxy_client then + proxy_client:close() + end + if admin_client then + admin_client:close() + end + + helpers.stop_kong() + end) + + it("increments the count for proxied requests", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/status/200", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(200, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="service-http-tag1,service-http-tag2",route_tags=""} 1', nil, true) + end) + + res = assert(proxy_client:send { + method = "GET", + path = "/status/400", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(400, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-service",route="http-route",code="400",service_tags="service-http-tag1,service-http-tag2",route_tags=""} 1', nil, true) + end) + end) + + it("increments the count for proxied grpc requests", function() + local ok, resp = proxy_client_grpc({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpc", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200",service_tags="service-grpc-tag1,service-grpc-tag2",route_tags=""} 1', nil, true) + end) + + ok, resp = proxy_client_grpcs({ + service = "hello.HelloService.SayHello", + body = { + greeting = "world!" + }, + opts = { + ["-authority"] = "grpcs", + } + }) + assert(ok, resp) + assert.truthy(resp) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200",service_tags="service-grpcs-tag1,service-grpcs-tag2",route_tags=""} 1', nil, true) + end) + end) + + pending("increments the count for proxied TCP streams", function() + local conn = assert(ngx.socket.connect("127.0.0.1", TCP_PROXY_PORT)) + + assert(conn:send("hi there!\n")) + local gotback = assert(conn:receive("*a")) + assert.equal("hi there!\n", gotback) + + conn:close() + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_stream_status{service="tcp-service",route="tcp-route",code="200",service_tags="service-tcp-tag1,service-tcp-tag2",route_tags=""} 1', nil, true) + end) + end) +end) + local test_f if stream_available then test_f = describe @@ -455,7 +1094,145 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="200",consumer="alice"} 1', nil, true) + return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="200",consumer="alice",service_tags="",route_tags=""} 1', nil, true) + end) + + res = assert(proxy_client:send { + method = "GET", + path = "/status/400", + headers = { + host = helpers.mock_upstream_host, + apikey = 'alice-key', + } + }) + assert.res_status(400, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="400",consumer="alice",service_tags="",route_tags=""} 1', nil, true) + end) + end) + + it("behave correctly if consumer is not found", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/status/200", + headers = { + host = helpers.mock_upstream_host, + } + }) + assert.res_status(401, res) + + local body + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + body = assert.res_status(200, res) + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="",route_tags=""} 1', nil, true) + end) + + assert.not_match('kong_http_consumer_status{service="mock-service",route="http-route",code="401",consumer="alice",service_tags="",route_tags=""} 1', body, nil, true) + assert.matches('kong_http_status{service="mock-service",route="http-route",code="401",service_tags="",route_tags=""} 1', body, nil, true) + + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + end) +end) + +describe("Plugin: prometheus (access) per-consumer metrics and exposing tags", function() + local proxy_client + local admin_client + + setup(function() + local bp = helpers.get_db_utils() + + local service = bp.services:insert { + name = "mock-service", + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_port, + protocol = helpers.mock_upstream_protocol, + tags = {"service-tag1", "service-tag2"}, + } + + local route = bp.routes:insert { + protocols = { "http" }, + name = "http-route", + paths = { "/" }, + methods = { "GET" }, + service = service, + tags = {"route-tag1", "route-tag2"}, + } + + bp.plugins:insert { + protocols = { "http", "https", "grpc", "grpcs", "tcp", "tls" }, + name = "prometheus", + config = { + per_consumer = true, + expose_services_tags = true, + expose_routes_tags = true, + } + } + + bp.plugins:insert { + name = "key-auth", + route = route, + } + + local consumer = bp.consumers:insert { + username = "alice", + } + + bp.keyauth_credentials:insert { + key = "alice-key", + consumer = consumer, + } + + assert(helpers.start_kong { + nginx_conf = nginx_conf, + plugins = "bundled, prometheus", + }) + proxy_client = helpers.proxy_client() + admin_client = helpers.admin_client() + end) + + teardown(function() + if proxy_client then + proxy_client:close() + end + if admin_client then + admin_client:close() + end + + helpers.stop_kong() + end) + + it("increments the count for proxied requests", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/status/200", + headers = { + host = helpers.mock_upstream_host, + apikey = 'alice-key', + } + }) + assert.res_status(200, res) + + helpers.wait_until(function() + local res = assert(admin_client:send { + method = "GET", + path = "/metrics", + }) + local body = assert.res_status(200, res) + assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) + + return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="200",consumer="alice",service_tags="service-tag1,service-tag2",route_tags="route-tag1,route-tag2"} 1', nil, true) end) res = assert(proxy_client:send { @@ -476,7 +1253,7 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="400",consumer="alice"} 1', nil, true) + return body:find('kong_http_consumer_status{service="mock-service",route="http-route",code="400",consumer="alice",service_tags="service-tag1,service-tag2",route_tags="route-tag1,route-tag2"} 1', nil, true) end) end) @@ -497,11 +1274,11 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() path = "/metrics", }) body = assert.res_status(200, res) - return body:find('kong_http_status{service="mock-service",route="http-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="service-tag1,service-tag2",route_tags="route-tag1,route-tag2"} 1', nil, true) end) - assert.not_match('kong_http_consumer_status{service="mock-service",route="http-route",code="401",consumer="alice"} 1', body, nil, true) - assert.matches('kong_http_status{service="mock-service",route="http-route",code="401"} 1', body, nil, true) + assert.not_match('kong_http_consumer_status{service="mock-service",route="http-route",code="401",consumer="alice",service_tags="service-tag1,service-tag2",route_tags="route-tag1,route-tag2"} 1', body, nil, true) + assert.matches('kong_http_status{service="mock-service",route="http-route",code="401",service_tags="service-tag1,service-tag2",route_tags="route-tag1,route-tag2"} 1', body, nil, true) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) end) diff --git a/spec/03-custom-serve_spec.lua b/spec/03-custom-serve_spec.lua index 3bb0091..84a86ed 100644 --- a/spec/03-custom-serve_spec.lua +++ b/spec/03-custom-serve_spec.lua @@ -57,7 +57,7 @@ describe("Plugin: prometheus (custom server)",function() path = "/metrics", }) local body = assert.res_status(200, res) - assert.matches('kong_http_status{service="mock-service",route="http-route",code="200"} 1', body, nil, true) + assert.matches('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="",route_tags=""} 1', body, nil, true) end) it("custom port returns 404 for anything other than /metrics", function() local client = helpers.http_client("127.0.0.1", 9542) diff --git a/spec/04-status_api_spec.lua b/spec/04-status_api_spec.lua index c93c240..1227212 100644 --- a/spec/04-status_api_spec.lua +++ b/spec/04-status_api_spec.lua @@ -176,7 +176,7 @@ describe("Plugin: prometheus (access via status API)", function() path = "/metrics", }) local body = assert.res_status(200, res) - return body:find('kong_http_status{service="mock-service",route="http-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-service",route="http-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) res = assert(proxy_client:send { @@ -194,7 +194,7 @@ describe("Plugin: prometheus (access via status API)", function() path = "/metrics", }) local body = assert.res_status(200, res) - return body:find('kong_http_status{service="mock-service",route="http-route",code="400"} 1', nil, true) + return body:find('kong_http_status{service="mock-service",route="http-route",code="400",service_tags="",route_tags=""} 1', nil, true) end) end) @@ -217,7 +217,7 @@ describe("Plugin: prometheus (access via status API)", function() path = "/metrics", }) local body = assert.res_status(200, res) - return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-grpc-service",route="grpc-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) ok, resp = proxy_client_grpcs({ @@ -238,7 +238,7 @@ describe("Plugin: prometheus (access via status API)", function() path = "/metrics", }) local body = assert.res_status(200, res) - return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200"} 1', nil, true) + return body:find('kong_http_status{service="mock-grpcs-service",route="grpcs-route",code="200",service_tags="",route_tags=""} 1', nil, true) end) end)