Skip to content

Commit

Permalink
[#968] Replace HTTP test proxy with SSH tunnels
Browse files Browse the repository at this point in the history 
SSH tunnels support HTTPS and other tcp connections
  • Loading branch information
@brusdev @gaohoward
brusdev authored and gaohoward committed Jul 2, 2024
1 parent 6e7a9bb commit bcd2351
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 37 deletions.
116 changes: 91 additions & 25 deletions controllers/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@ import (
"strings"
"time"

"github.com/go-logr/logr"
configv1 "github.com/openshift/api/config/v1"
routev1 "github.com/openshift/api/route/v1"
"go.uber.org/zap/zapcore"
"golang.org/x/crypto/ssh"

"path/filepath"
"testing"
Expand Down Expand Up @@ -148,6 +150,7 @@ var (
defaultOperatorInstalled = true
defaultUid = int64(185)
watchClientList *list.List = nil
testProxyLog logr.Logger
)

func init() {
Expand Down Expand Up @@ -217,7 +220,7 @@ func setUpNamespace() {
}

err := k8sClient.Create(ctx, &testNamespace)
Expect(err == nil || errors.IsConflict(err)).To(BeTrue())
Expect(err == nil || errors.IsAlreadyExists(err)).To(BeTrue())

if isOpenshift {
Eventually(func(g Gomega) {
Expand Down Expand Up @@ -272,16 +275,20 @@ func setUpIngress() {
// Set up test-proxy for external http requests
func setUpTestProxy() {

var err error
testProxyPort := int32(3129)
testProxyPort := int32(44322)
testProxyDeploymentReplicas := int32(1)
testProxyName := "test-proxy"
testProxyNamespace := "default"
testProxyHost := testProxyName + ".tests.artemiscloud.io"
testProxyLabels := map[string]string{"app": "test-proxy"}
testProxyScript := fmt.Sprintf("openssl req -newkey rsa:2048 -nodes -keyout %[1]s -x509 -days 365 -out %[2]s -subj '/CN=test-proxy' && "+
"echo 'https_port %[3]d tls-cert=%[2]s tls-key=%[1]s' >> %[4]s && "+"entrypoint.sh -f %[4]s -NYC",
"/etc/squid/key.pem", "/etc/squid/certificate.pem", 3129, "/etc/squid/squid.conf")
testProxyScript := fmt.Sprintf("yum -y install openssh-server openssl stunnel && "+
"adduser --system -u 1000 tunnel && echo secret | passwd tunnel --stdin && "+
"sed -i 's/#Port.*$/Port 2022/' /etc/ssh/sshd_config && ssh-keygen -A && "+
"echo -e 'cert=%[1]s \n[ssh]\naccept=44322\nconnect=2022' > %[2]s && "+
"openssl req -new -x509 -days 365 -nodes -subj '/CN=test-proxy' -keyout %[1]s -out %[1]s && "+
"stunnel %[2]s && /usr/sbin/sshd -eD", "/etc/stunnel/stunnel.pem", "/etc/stunnel/stunnel.conf")

testProxyLog = ctrl.Log.WithName(testProxyName)

testProxyDeployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -300,24 +307,35 @@ func setUpTestProxy() {
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ningx",
Image: "docker.io/ubuntu/squid:edge",
Name: testProxyName + "-con",
Image: "registry.access.redhat.com/ubi8/ubi:8.9",
Command: []string{"sh", "-c", testProxyScript},
Env: []corev1.EnvVar{
{Name: "HTTP_PROXY", Value: os.Getenv("HTTP_PROXY")},
{Name: "HTTPS_PROXY", Value: os.Getenv("HTTPS_PROXY")},
{Name: "http_proxy", Value: os.Getenv("http_proxy")},
{Name: "https_proxy", Value: os.Getenv("https_proxy")},
},
Ports: []corev1.ContainerPort{
{
ContainerPort: testProxyPort,
Protocol: "TCP",
},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"SYS_CHROOT",
},
},
},
},
},
},
},
},
}

err = k8sClient.Create(ctx, &testProxyDeployment)
Expect(err == nil || errors.IsConflict(err)).To(BeTrue())
createOrOverwriteResource(&testProxyDeployment)

testProxyService := corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -334,33 +352,68 @@ func setUpTestProxy() {
},
},
}

err = k8sClient.Create(ctx, &testProxyService)
Expect(err == nil || errors.IsConflict(err)).To(BeTrue())
createOrOverwriteResource(&testProxyService)

testProxyIngress := ingresses.NewIngressForCRWithSSL(
nil, types.NamespacedName{Name: testProxyName, Namespace: testProxyNamespace},
map[string]string{}, testProxyName+"-dep-svc", strconv.FormatInt(int64(testProxyPort), 10),
true, "", testProxyHost, isOpenshift)

err = k8sClient.Create(ctx, testProxyIngress)
Expect(err == nil || errors.IsConflict(err)).To(BeTrue())

proxyUrl, err := url.Parse(fmt.Sprintf("https://%s:%d", testProxyHost, 443))
Expect(err).NotTo(HaveOccurred())
createOrOverwriteResource(testProxyIngress)

http.DefaultTransport = &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if strings.HasPrefix(addr, testProxyHost) {
addr = clusterIngressHost + ":443"
tlsConn, tlsErr := tls.Dial("tcp", clusterIngressHost+":443",
&tls.Config{ServerName: testProxyHost, InsecureSkipVerify: true})
if tlsErr != nil {
testProxyLog.V(1).Info("Error creating tls connection", "addr", addr, "error", tlsErr)
return nil, tlsErr
}
return (&net.Dialer{}).DialContext(ctx, network, addr)

sshConn, sshChans, sshReqs, sshErr := ssh.NewClientConn(tlsConn, "127.0.0.1:2022", &ssh.ClientConfig{
User: "tunnel",
Auth: []ssh.AuthMethod{ssh.Password("secret")},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
})
if sshErr != nil {
testProxyLog.V(1).Info("Error creating SSH connection", "addr", addr, "error", sshErr)
fmt.Printf("\nError creating SSH tunnel to %s: %v", addr, sshErr)
tlsConn.Close()
return nil, sshErr
}

sshClient := ssh.NewClient(sshConn, sshChans, sshReqs)

sshClientConn, sshClientErr := sshClient.DialContext(ctx, network, addr)
if sshClientErr != nil {
testProxyLog.V(1).Info("Error creating SSH tunnel", "addr", addr, "error", sshClientErr)
sshClient.Close()
sshConn.Close()
tlsConn.Close()
return nil, sshClientErr
}

testProxyLog.V(1).Info("Opened SSH tunnel", "addr", addr)
return &testProxyConn{sshClientConn, addr, sshClient, &sshConn, tlsConn}, nil
},
Proxy: http.ProxyURL(proxyUrl),
TLSClientConfig: &tls.Config{ServerName: testProxyHost, InsecureSkipVerify: true},
}
}

type testProxyConn struct {
net.Conn
addr string
sshClient *ssh.Client
sshConn *ssh.Conn
tlsConn *tls.Conn
}

func (w *testProxyConn) Close() error {
testProxyLog.V(1).Info("Closed SSH tunnel", "addr", w.addr)
w.Conn.Close()
(*w.sshClient).Close()
(*w.sshConn).Close()
return (*w.tlsConn).Close()
}

func cleanUpTestProxy() {
var err error

Expand Down Expand Up @@ -398,6 +451,19 @@ func cleanUpTestProxy() {
Expect(err == nil || errors.IsNotFound(err)).To(BeTrue())
}

func createOrOverwriteResource(res client.Object) {
err := k8sClient.Create(ctx, res)
if errors.IsAlreadyExists(err) {
k8sClient.Delete(ctx, res)

Eventually(func(g Gomega) {
g.Expect(k8sClient.Create(ctx, res)).To(Succeed())
}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
} else {
Expect(err).To(Succeed())
}
}

func createControllerManagerForSuite() {
createControllerManager(false, "")
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (

require (
github.com/blang/semver/v4 v4.0.0
golang.org/x/crypto v0.21.0
k8s.io/apiextensions-apiserver v0.28.3
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
)
Expand Down Expand Up @@ -66,7 +67,6 @@ require (
github.com/prometheus/procfs v0.10.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
Expand Down
22 changes: 11 additions & 11 deletions pkg/utils/jolokia/jolokia.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,20 +83,20 @@ func (j *Jolokia) GetProtocol() string {
}

func (j *Jolokia) getClient() *http.Client {
httpClient := http.Client{
Transport: http.DefaultTransport,
Timeout: time.Second * 2,
}

if j.protocol == "https" {
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
ServerName: j.ip,
},
},
Timeout: time.Second * 2, //Maximum of 2 seconds
httpClientTransport := httpClient.Transport.(*http.Transport)
httpClientTransport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
ServerName: j.ip,
}
}
return &http.Client{
Timeout: time.Second * 2, // Maximum of 2 seconds
}

return &httpClient
}

func (j *Jolokia) Read(_path string) (*ResponseData, error) {
Expand Down

0 comments on commit bcd2351

Please sign in to comment.