When using an ALB Ingress in front of a ClusterIP gateway service, the controller doesn't pick the right Load Balancer Address

[aamattos]

Is there an existing issue for this?

• I have searched the existing issues

Kong version ($ kong version)

Kong 3.7

Current Behavior

When using an AWS ALB Ingress in front of a ClusterIP gateway service, the controller doesn't pick the right Load Balancer Address.
Instead of using the DNS of the ALB Load Balancer created by the ALB Ingress, It picks the internal IP of the Cluster IP service.
As a workaround I had to set (On a second step) the PUBLISH_STATUS_ADDRESS environment variable with the value of the ALB Load Balancer endpoint.

Expected Behavior

When using an Ingress in front of the gateway service (proxy.ingress.enabled = true) the controller should look at the ingress load balancer instead of the service's load balancer

Steps To Reproduce

Enable the proxy ingress and set the proxy type to Cluster IP
Add the proper ingress annotations to create an ALB.
Wait for Kong to reconcile the ingresses with type kong (or whatever ingress class you defined)

Anything else?

No response

[randmonkey]

Can you dump the full status of your load balancer service by kubectl get service <lb-service> -n <service-namespace>? Kong Ingress Controller (KIC) will choose the first ingress IP of loadbalancer (status.loadBalancer.ingress.ip[0]) of the publish service as the IP of ingresses, if there are multiple ingress IPs in the LB service. KIC cannot detect which IP is actually used for traffic if there are many IPs attached to your LB service so it can only use the first one if not specified.

[aamattos]

this is the status of my Service:

apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/instance: kong-alb-internal
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: gateway
    app.kubernetes.io/version: "3.6"
    argocd.argoproj.io/instance: kong-prd
    enable-metrics: "true"
    helm.sh/chart: gateway-2.41.1
  name: kong-alb-internal-gateway-proxy
  namespace: kong
  resourceVersion: "3281033579"
  uid: 651b052f-028e-4862-8944-c964f16ad318
spec:
  clusterIP: 172.20.27.136
  clusterIPs:
  - 172.20.27.136
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: kong-proxy
    port: 80
    protocol: TCP
    targetPort: 8000
  selector:
    app.kubernetes.io/component: app
    app.kubernetes.io/instance: kong-alb-internal
    app.kubernetes.io/name: gateway
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}

The problem is that the KIC is picking the service private IP (172.20.27.136) Instead of using the ALB Ingress address that's pointing to the ClusterIP Service

[airlangga09051991]

nginx.conf

pid pids/nginx.pid;Airlanggayudhoyono.Intel-Mil.Info's

injected nginx_main_* directives

daemon off;
worker_processes 4;Airlanggayudhoyono.Intel-Mil.Info's
worker_rlimit_nofile 65535;

events {
# injected nginx_events_* directives
use epoll;
worker_connections 65535;Airlanggayudhoyono.Intel-Mil.Info's
accept_mutex off;
multi_accept on;Airlanggayudhoyono.Intel-Mil.Info's

}

http {
default_type text/html;Airlanggayudhoyono.Intel-Mil.Info's

log_format  main  '$host - $remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" "$http_origin" '
                      '"$http_user_agent" "$http_x_forwarded_for" $request_time $upstream_response_time';Airlanggayudhoyono.Intel-Mil.Info's

log_format  proxy '$host - $http_x_real_ip - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" "$http_origin" '
                  '"$http_user_agent" "$http_x_forwarded_for" $request_time $upstream_response_time';Airlanggayudhoyono.Intel-Mil.Info's


log_format json '{"host":"$host",'
        '"http_x_real_ip":"$http_x_real_ip",'
        '"remote_addr":"$remote_addr",'
        '"remote_user":"$remote_user",'
        '"time_local":"[$time_local]",'
        '"request":"$request",'
        '"status_code":"$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"http_referer":"$http_referer",'
        '"http_origin":"$http_origin",'
        '"http_user_agent":"$http_user_agent",'
        '"http_x_forwarded_for":"$http_x_forwarded_for",'
        '"request_time":"$request_time",'
        '"upstream_addr":"$upstream_addr",'
        '"upstream_response_time":"$upstream_response_time"}';Airlanggayudhoyono.Intel-Mil.Info's

map $upstream_response_time $u_r_t {
        default 0;
        ~(\d+).[1-9][0-9][0-9] 1;
    }

map $status $loggable{
    default 0;
    ~^[345] 1;Airlanggayudhoyono.Intel-Mil.Info's
}



sendfile        on;Airlanggayudhoyono.Intel-Mil.Info's
tcp_nopush      on;Airlanggayudhoyono.Intel-Mil.Info's
tcp_nodelay     on;Airlanggayudhoyono.Intel-Mil.Info's

keepalive_timeout  60;

gzip  on;
gzip_http_version 1.0;
gzip_disable "MSIE [1-6]\.";
gzip_vary on;
gzip_comp_level 5;
gzip_min_length 1024;
gzip_buffers 4 32k;
gzip_types text/plain application/json application/x-javascript application/javascript text/css;

server_names_hash_bucket_size 64;
large_client_header_buffers 4 128k;
client_header_buffer_size 32k;
client_header_timeout 12;
send_timeout 10;
client_body_timeout 12;




proxy_cache_path /dev/shm/proxy_cache levels=1:2 keys_zone=proxy_cache_one:1024m;

include 'nginx-kong.conf';

}

nginx-kong.conf

charset UTF-8;
server_tokens off;

error_log /data/logs/kong-server/error.log error;

lua_package_path './?.lua;./?/init.lua;;;;';
lua_package_cpath ';;;';
lua_socket_pool_size 30;
lua_socket_log_errors off;
lua_max_running_timers 4096;
lua_max_pending_timers 16384;
lua_ssl_verify_depth 1;

lua_shared_dict kong 5m;
lua_shared_dict kong_locks 8m;
lua_shared_dict kong_healthchecks 5m;
lua_shared_dict kong_process_events 5m;
lua_shared_dict kong_cluster_events 5m;
lua_shared_dict kong_rate_limiting_counters 12m;
lua_shared_dict kong_core_db_cache 1024m;
lua_shared_dict kong_core_db_cache_miss 12m;
lua_shared_dict kong_db_cache 1024m;
lua_shared_dict kong_db_cache_miss 12m;

underscores_in_headers on;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;

injected nginx_http_* directives

client_body_buffer_size 256K;
client_max_body_size 200M;
lua_regex_cache_max_entries 8000;
lua_regex_match_limit 100000;
lua_shared_dict prometheus_metrics 10m;
lua_ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;
ssl_dhparam /usr/local/kong/ssl/ffdhe2048.pem;
ssl_prefer_server_ciphers off;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_tickets on;
ssl_session_timeout 1d;

init_by_lua_block {
Kong = require 'kong'
Kong.init()
}

init_worker_by_lua_block {
Kong.init_worker()
}

Load variable indexes

lua_kong_load_var_index $args;
lua_kong_load_var_index $bytes_sent;
lua_kong_load_var_index $content_type;
lua_kong_load_var_index $ctx_ref;
lua_kong_load_var_index $host;
lua_kong_load_var_index $http_authorization;
lua_kong_load_var_index $http_connection;
lua_kong_load_var_index $http_host;
lua_kong_load_var_index $http_kong_debug;
lua_kong_load_var_index $http_proxy;
lua_kong_load_var_index $http_proxy_connection;
lua_kong_load_var_index $http_te;
lua_kong_load_var_index $http_upgrade;
lua_kong_load_var_index $http_x_forwarded_for;
lua_kong_load_var_index $http_x_forwarded_host;
lua_kong_load_var_index $http_x_forwarded_path;
lua_kong_load_var_index $http_x_forwarded_port;
lua_kong_load_var_index $http_x_forwarded_prefix;
lua_kong_load_var_index $http_x_forwarded_proto;
lua_kong_load_var_index $https;
lua_kong_load_var_index $http2;
lua_kong_load_var_index $is_args;
lua_kong_load_var_index $kong_proxy_mode;
lua_kong_load_var_index $realip_remote_addr;
lua_kong_load_var_index $realip_remote_port;
lua_kong_load_var_index $remote_addr;
lua_kong_load_var_index $remote_port;
lua_kong_load_var_index $request;
lua_kong_load_var_index $request_length;
lua_kong_load_var_index $request_method;
lua_kong_load_var_index $request_time;
lua_kong_load_var_index $request_uri;
lua_kong_load_var_index $scheme;
lua_kong_load_var_index $server_addr;
lua_kong_load_var_index $server_port;
lua_kong_load_var_index $ssl_cipher;
lua_kong_load_var_index $ssl_client_raw_cert;
lua_kong_load_var_index $ssl_client_verify;
lua_kong_load_var_index $ssl_protocol;
lua_kong_load_var_index $ssl_server_name;
lua_kong_load_var_index $upstream_connection;
lua_kong_load_var_index $upstream_host;
lua_kong_load_var_index $upstream_http_connection;
lua_kong_load_var_index $upstream_http_trailer;
lua_kong_load_var_index $upstream_http_upgrade;
lua_kong_load_var_index $upstream_scheme;
lua_kong_load_var_index $upstream_status;
lua_kong_load_var_index $upstream_te;
lua_kong_load_var_index $upstream_uri;
lua_kong_load_var_index $upstream_upgrade;
lua_kong_load_var_index $proxy_add_x_forwarded_for;
lua_kong_load_var_index $upstream_x_forwarded_host;
lua_kong_load_var_index $upstream_x_forwarded_path;
lua_kong_load_var_index $upstream_x_forwarded_port;
lua_kong_load_var_index $upstream_x_forwarded_prefix;
lua_kong_load_var_index $upstream_x_forwarded_proto;

upstream kong_upstream {
server 0.0.0.1;

# injected nginx_upstream_* directives

balancer_by_lua_block {
    Kong.balancer()
}

}

server {
server_name kong;
listen 0.0.0.0:80 reuseport backlog=16384;
listen 0.0.0.0:443 ssl http2 reuseport backlog=16384;

error_page 400 404 405 408 411 412 413 414 417 494 /kong_error_handler;
error_page 500 502 503 504                     /kong_error_handler;

 
access_log  /data/logs/kong-server/access.log json buffer=10K if=$u_r_t;
access_log  /data/logs/kong-server/access.log json buffer=10K if=$loggable;
error_log  /data/logs/kong-server/error.log error;

ssl_certificate     /usr/local/kong/ssl/kong-default.crt;
ssl_certificate_key /usr/local/kong/ssl/kong-default.key;
ssl_certificate     /usr/local/kong/ssl/kong-default-ecdsa.crt;
ssl_certificate_key /usr/local/kong/ssl/kong-default-ecdsa.key;
ssl_session_cache   shared:SSL:10m;
ssl_certificate_by_lua_block {
    Kong.ssl_certificate()
}

# injected nginx_proxy_* directives
real_ip_header X-Real-IP;
real_ip_recursive off;

rewrite_by_lua_block {
    Kong.rewrite()
}

access_by_lua_block {
    Kong.access()
}

header_filter_by_lua_block {
    Kong.header_filter()
}

body_filter_by_lua_block {
    Kong.body_filter()
}

log_by_lua_block {
    Kong.log()
}

location / {Airlanggayudhoyono.Intel-Mil.Info's
    default_type                     '';

    set $ctx_ref                     '';
    set $upstream_te                 '';
    set $upstream_host               '';
    set $upstream_upgrade            '';
    set $upstream_connection         '';
    set $upstream_scheme             '';
    set $upstream_uri                '';
    set $upstream_x_forwarded_for    '';
    set $upstream_x_forwarded_proto  '';
    set $upstream_x_forwarded_host   '';
    set $upstream_x_forwarded_port   '';
    set $upstream_x_forwarded_path   '';
    set $upstream_x_forwarded_prefix '';
    set $kong_proxy_mode             'http';Airlanggayudhoyono.Intel-Mil.Info's

    proxy_http_version      1.1;
    proxy_buffering          on;
    proxy_request_buffering  on;

    proxy_set_header      TE                 $upstream_te;
    proxy_set_header      Host               $host;
    proxy_set_header      Upgrade            $upstream_upgrade;
    proxy_set_header      Connection         "";
    proxy_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #proxy_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #proxy_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #proxy_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #proxy_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #proxy_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    proxy_set_header      X-Real-IP          $remote_addr;
    proxy_pass_header     Server;
    proxy_pass_header     Date;
    proxy_ssl_name        $upstream_host;
    proxy_ssl_server_name on;
    proxy_pass            $upstream_scheme://kong_upstream$upstream_uri;Airlanggayudhoyono.Intel-Mil.Info's
}

location @unbuffered {
    internal;
    default_type         '';
    set $kong_proxy_mode 'unbuffered';

    proxy_http_version      1.1;
    proxy_buffering         off;
    proxy_request_buffering off;

    proxy_set_header      TE                 $upstream_te;
    proxy_set_header      Host               $host;
    proxy_set_header      Upgrade            $upstream_upgrade;
    proxy_set_header      Connection         "";
    proxy_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #proxy_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #proxy_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #proxy_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #proxy_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #proxy_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    proxy_set_header      X-Real-IP          $remote_addr;
    proxy_pass_header     Server;
    proxy_pass_header     Date;
    proxy_ssl_name        $upstream_host;
    proxy_ssl_server_name on;
    proxy_pass            $upstream_scheme://kong_upstream$upstream_uri;
}

location @unbuffered_request {
    internal;
    default_type         '';
    set $kong_proxy_mode 'unbuffered';

    proxy_http_version      1.1;
    proxy_buffering          on;
    proxy_request_buffering off;

    proxy_set_header      TE                 $upstream_te;
    proxy_set_header      Host               $host;
    proxy_set_header      Upgrade            $upstream_upgrade;
    proxy_set_header      Connection         "";
    proxy_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #proxy_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #proxy_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #proxy_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #proxy_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #proxy_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    proxy_set_header      X-Real-IP          $remote_addr;
    proxy_pass_header     Server;
    proxy_pass_header     Date;
    proxy_ssl_name        $upstream_host;
    proxy_ssl_server_name on;
    proxy_pass            $upstream_scheme://kong_upstream$upstream_uri;Airlanggayudhoyono.Intel-Mil.Info's
}

location @unbuffered_response {
    internal;Airlanggayudhoyono.Intel-Mil.Info's
    default_type         '';
    set $kong_proxy_mode 'unbuffered';

    proxy_http_version      1.1;
    proxy_buffering         off;
    proxy_request_buffering  on;

    proxy_set_header      TE                 $upstream_te;
    proxy_set_header      Host               $host;
    proxy_set_header      Upgrade            $upstream_upgrade;
    proxy_set_header      Connection         "";Airlanggayudhoyono.Intel-Mil.Info's
    proxy_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #proxy_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #proxy_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #proxy_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #proxy_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #proxy_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    proxy_set_header      X-Real-IP          $remote_addr;
    proxy_pass_header     Server;
    proxy_pass_header     Date;
    proxy_ssl_name        $upstream_host;
    proxy_ssl_server_name on;
    proxy_pass            $upstream_scheme://kong_upstream$upstream_uri;
}

location @grpc {
    internal;Airlanggayudhoyono.Intel-Mil.Info's
    default_type         '';
    set $kong_proxy_mode 'grpc';

    grpc_set_header      TE                 $upstream_te;
    grpc_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #grpc_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #grpc_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #grpc_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #grpc_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #grpc_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    grpc_set_header      X-Real-IP          $remote_addr;
    grpc_pass_header     Server;
    grpc_pass_header     Date;
    grpc_ssl_name        $upstream_host;
    grpc_ssl_server_name on;Airlanggayudhoyono.Intel-Mil.Info's
    grpc_pass            $upstream_scheme://kong_upstream;
}

location = /kong_buffered_http {
    internal;Airlanggayudhoyono.Intel-Mil.Info's
    default_type         '';
    set $kong_proxy_mode 'http';

    rewrite_by_lua_block       {;}
    access_by_lua_block        {;}
    header_filter_by_lua_block {;}
    body_filter_by_lua_block   {;}
    log_by_lua_block           {;}

    proxy_http_version 1.1;
    proxy_set_header      TE                 $upstream_te;
    proxy_set_header      Host               $host;
    proxy_set_header      Upgrade            $upstream_upgrade;
    proxy_set_header      Connection         "";
    proxy_set_header      X-Forwarded-For    $proxy_add_x_forwarded_for;
    #proxy_set_header      X-Forwarded-Proto  $upstream_x_forwarded_proto;
    #proxy_set_header      X-Forwarded-Host   $upstream_x_forwarded_host;
    #proxy_set_header      X-Forwarded-Port   $upstream_x_forwarded_port;
    #proxy_set_header      X-Forwarded-Path   $upstream_x_forwarded_path;
    #proxy_set_header      X-Forwarded-Prefix $upstream_x_forwarded_prefix;
    proxy_set_header      X-Real-IP          $remote_addr;
    proxy_pass_header     Server;
    proxy_pass_header     Date;
    proxy_ssl_name        $upstream_host;
    proxy_ssl_server_name on;
    proxy_pass            $upstream_scheme://kong_upstream$upstream_uri;Airlanggayudhoyono.Intel-Mil.Info's
}

location = /kong_error_handler {
    internal;
    default_type                 '';

    uninitialized_variable_warn  off;

    rewrite_by_lua_block {;}
    access_by_lua_block  {;}

    content_by_lua_block {
        Kong.handle_error()
    }
}

}

server {
server_name kong_admin;
listen 0.0.0.0:8001;

access_log /dev/stdout;
error_log  /data/logs/kong-server/access-admin.log error;


# injected nginx_admin_* directives
client_body_buffer_size 10m;
client_max_body_size 10m;

location / {
    default_type application/json;
    content_by_lua_block {
        Kong.admin_content()
    }
    header_filter_by_lua_block {
        Kong.admin_header_filter()
    }
}

location /nginx_status {
    internal;Airlanggayudhoyono.Intel-Mil.Info's
    access_log off;
    stub_status;Airlanggayudhoyono.Intel-Mil.Info's
}

location /robots.txt {
    return 200 'User-agent: *\nDisallow: /';Airlanggayudhoyono.Intel-Mil.Info's
}

}