Merge pull request #62 from Aggouri/as/ssh-proxy

Add HTTP Proxy support for SSH

Author: xtremerui

README.md

@@ -72,6 +72,13 @@ filed named `.gitkeep`. Finally, create individual locks by making an empty file
   retrying to acquire a lock or release a lock. The default is 10 seconds.
   Valid values: `60s`, `90m`, `1h`.
 
+* `https_tunnel`: *Optional.* Information about an HTTPS proxy that will be used to tunnel SSH-based git commands over.
+  Has the following sub-properties:
+  * `proxy_host`: *Required.* The host name or IP of the proxy server
+  * `proxy_port`: *Required.* The proxy server's listening port
+  * `proxy_user`: *Optional.* If the proxy requires authentication, use this username
+  * `proxy_password`: *Optional.* If the proxy requires authentication, use this password
+
 ### Example
 
 Fetching a repo with only 100 commits of history:
@@ -234,6 +241,29 @@ example:
       params: {release: specific-aws-env}
 ```
 
+### Configuring resource to proxy SSH commands through an HTTP proxy
+
+```
+resources:
+- name: aws-environments
+  type: pool
+  source:
+    uri: [email protected]:concourse/locks.git
+    branch: master
+    pool: aws
+    private_key: |
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEowIBAAKCAQEAtCS10/f7W7lkQaSgD/mVeaSOvSF9ql4hf/zfMwfVGgHWjj+W
+      <Lots more text>
+      DWiJL+OFeg9kawcUL6hQ8JeXPhlImG6RTUffma9+iGQyyBMCGd1l
+      -----END RSA PRIVATE KEY-----
+    https_tunnel:
+      proxy_host: proxy-server.mycorp.com
+      proxy_port: 3128
+      proxy_user: myuser
+      proxy_password: myverysecurepassword
+```
+
 ## Development
 
 ### Prerequisites
@@ -258,6 +288,29 @@ docker build -t pool-resource --target tests -f dockerfiles/alpine/Dockerfile .
 docker build -t pool-resource --target tests -f dockerfiles/ubuntu/Dockerfile .
 ```
 
+#### Note about the integration tests
+
+If you want to run the integration tests, a bit more work is required. You will require
+an actual git repo to which you can push and pull, configured for SSH access. To do this,
+add two files to `integration-tests/ssh` (note that names **are** important):
+* `test_key`: This is the private key used to authenticate against your repo.
+* `test_repo`: This file contains one line of the form `test_repo_url[#test_branch]`.
+  If the branch is not specified, it defaults to `main`. For example,
+  `[email protected]:concourse-git-tester/git-resource-integration-tests.git` or
+  `[email protected]:concourse-git-tester/git-resource-integration-tests.git#testing`
+
+To set up or reset the contents of the repo, use the `integration-tests/ssh/init-repo.sh` script. 
+The script clones the configured repository, (re-)creates the relevant directories,
+commits and pushes the changes. If you'd rather execute the commands yourself, view the script
+contents to understand the directory structure expected by the integration tests. 
+
+Then run the tests for both `alpine` and `ubuntu` images:
+
+```sh
+docker build -t pool-resource --target integrationtests -f dockerfiles/alpine/Dockerfile .
+docker build -t pool-resource --build-arg base_image=concourse/golang-builder --target integrationtests -f dockerfiles/ubuntu/Dockerfile .
+```
+
 ### Contributing
 
 Please make all pull requests to the `master` branch and ensure tests pass

assets/check

@@ -16,6 +16,7 @@ payload=$TMPDIR/git-resource-request
 cat > $payload <&0
 
 load_pubkey $payload
+configure_https_tunnel $payload
 configure_credentials $payload
 
 uri=$(jq -r '.source.uri // ""' < $payload)

assets/common.sh

@@ -22,6 +22,31 @@ EOF
   fi
 }
 
+configure_https_tunnel() {
+  tunnel=$(jq -r '.source.https_tunnel // empty' < $1)
+
+  if [ ! -z "$tunnel" ]; then
+    host=$(echo "$tunnel" | jq -r '.proxy_host // empty')
+    port=$(echo "$tunnel" | jq -r '.proxy_port // empty')
+    user=$(echo "$tunnel" | jq -r '.proxy_user // empty')
+    password=$(echo "$tunnel" | jq -r '.proxy_password // empty')
+
+    pass_file=""
+    if [ ! -z "$user" ]; then
+      cat > ~/.ssh/tunnel_config <<EOF
+proxy_user = $user
+proxy_passwd = $password
+EOF
+      chmod 0600 ~/.ssh/tunnel_config
+      pass_file="-F ~/.ssh/tunnel_config"
+    fi
+
+    if [ -n "$host" ] && [ -n "$port" ]; then
+      echo "ProxyCommand /usr/bin/proxytunnel $pass_file -p $host:$port -d %h:%p" >> ~/.ssh/config
+    fi
+  fi
+}
+
 configure_credentials() {
   local username=$(jq -r '.source.username // ""' < $1)
   local password=$(jq -r '.source.password // ""' < $1)

assets/in

@@ -33,6 +33,7 @@ payload=$(mktemp $TMPDIR/pool-resource-request.XXXXXX)
 cat > $payload <&0
 
 load_pubkey $payload
+configure_https_tunnel $payload
 configure_credentials $payload
 
 uri=$(jq -r '.source.uri // ""' < $payload)

assets/out

@@ -10,6 +10,7 @@ exec 1>&2 # redirect all output to stderr for logging
 payload=$(mktemp $TMPDIR/pool-resource-request.XXXXXX)
 cat > $payload <&0
 load_pubkey $payload
+configure_https_tunnel $payload
 configure_credentials $payload
 
 /opt/go/out $1 >&3 < $payload

dockerfiles/alpine/Dockerfile

@@ -11,14 +11,23 @@ RUN set -e; for pkg in $(go list ./...); do \
 	done
 
 FROM ${base_image} AS resource
-RUN apk add --no-cache bash jq git git-daemon openssh
+RUN apk add --no-cache bash jq git git-daemon openssh make g++ libressl-dev
 RUN git config --global user.email "git@localhost"
 RUN git config --global user.name "git"
+
 ADD assets/ /opt/resource/
 RUN chmod +x /opt/resource/*
 COPY --from=builder /assets /opt/go
 RUN chmod +x /opt/go/out
 
+WORKDIR /root
+RUN git clone https://github.com/proxytunnel/proxytunnel.git && \
+    cd proxytunnel && \
+    make -j4 && \
+    install -c proxytunnel /usr/bin/proxytunnel && \
+    cd .. && \
+    rm -rf proxytunnel
+
 FROM resource AS tests
 COPY --from=builder /tests /go/resource-tests/
 RUN set -e; for test in /go/resource-tests/*.test; do \
@@ -27,4 +36,10 @@ RUN set -e; for test in /go/resource-tests/*.test; do \
 ADD test/ /opt/resource-tests
 RUN /opt/resource-tests/all.sh
 
+FROM resource AS integrationtests
+RUN apk --no-cache add squid
+ADD test/ /opt/resource-tests/test
+ADD integration-tests /opt/resource-tests/integration-tests
+RUN /opt/resource-tests/integration-tests/integration.sh
+
 FROM resource

dockerfiles/ubuntu/Dockerfile

@@ -11,7 +11,7 @@ RUN set -e; for pkg in $(go list ./...); do \
 	done
 
 FROM ${base_image} AS resource
-RUN apt-get update && apt-get install -y jq git
+RUN apt-get update && apt-get install -y jq git make g++ libssl-dev openssh-client
 RUN git config --global user.email "git@localhost"
 RUN git config --global user.name "git"
 
@@ -20,6 +20,14 @@ RUN chmod +x /opt/resource/*
 COPY --from=builder /assets /opt/go
 RUN chmod +x /opt/go/out
 
+WORKDIR /root
+RUN git clone https://github.com/proxytunnel/proxytunnel.git && \
+    cd proxytunnel && \
+    make -j4 && \
+    install -c proxytunnel /usr/bin/proxytunnel && \
+    cd .. && \
+    rm -rf proxytunnel
+
 FROM resource AS tests
 COPY --from=builder /tests /go/resource-tests/
 RUN set -e; for test in /go/resource-tests/*.test; do \
@@ -28,4 +36,10 @@ RUN set -e; for test in /go/resource-tests/*.test; do \
 ADD test/ /opt/resource-tests
 RUN /opt/resource-tests/all.sh
 
+FROM resource AS integrationtests
+RUN apt-get update && apt-get install -y squid net-tools
+ADD test/ /opt/resource-tests/test
+ADD integration-tests /opt/resource-tests/integration-tests
+RUN /opt/resource-tests/integration-tests/integration.sh
+
 FROM resource