Add EFS support

README.md

@@ -1,6 +1,23 @@
 # hl-component-sftp CfHighlander project
 ---
 
+## Important considerations
+
+If you want SFTP to be accessed over the internet, you will need to add an ACL rule for port 22 to the VPC config (note you will possibly then need to specify all of the ACL rules or else the default rules will be removed):
+
+```
+-
+  acl: public
+  number: 100
+  from: 22
+  ips:
+    - sftp
+
+ip_blocks:
+  sftp:
+    - 52.64.86.162/32   # base2 VPN for example
+```
+
 ## Config file options
 
 ### `dynamic_users`
@@ -32,6 +49,67 @@ A scheduled daily clean up lambda will automatically delete the temporary sftp u
 
 `VPC_ENDPOINT` - old way of doing things, it is recommended that you use the `VPC` option instead
 
+### `domain`
+
+`S3` - S3 bucket for storage
+
+`EFS` - EFS FileSystem for storage
+
+## EFS
+
+SFTP can be configured to point to an EFS FileSystem, to do so there are a few specific configurations needed to be made and the user setup is slightly different to when the domain is S3.
+
+### `sftp.config.yaml` file
+
+Set the `domain` to EFS:
+
+```
+domain: EFS
+```
+
+### cfhighlander file
+
+You will need to specify the parameter for the FileSystemId. Example:
+
+```
+Component name: 'sftp', template: 'sftp' do
+  parameter name: 'VpcId', value: cfout('vpcv2', 'VPCId')
+  parameter name: 'SubnetIds', value: cfout('vpcv2', 'PublicSubnets')
+  parameter name: 'DnsDomain', value: FnSub("${EnvironmentName}.#{root_domain}")
+  parameter name: 'FileSystemId', value: cfout('efsv2', 'FileSystem')
+end
+```
+
+### Example user config
+
+Here's how to configure a user for EFS, this is in the `sftp.config.yaml` file:
+
+```
+users:
+  - name: base2
+    # home_directory_type and home_directory_mappings go together, as if you want to use logical directories you need mappings, if you just want path directories then don't specify home_directory_type
+    home_directory_type: LOGICAL
+    home_directory_mappings:
+      - entry: /                                    # This is what the directory will be for the user connecting, specify / if you don't have any special requirements
+        target: /${FileSystemId}/ftp_home/base2     # This is where the directory will point to on the FileSystem, currently you need to specify /${FileSystemId} to tell it to put it on the root of the FileSystem. Currently you also need to create this directory on the FileSystem yourself
+    keys:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg6Q8ceKXLwe3kKUm7R9gEgekgdSUqD78umhWl11BtlYfGLJc3ZKTzBzANaOLyB1BO+xHw1QPDvK+LrfexLvO5WHQuFI5OtoFaPKZc3clL1PqeEAMvspddIElGtq0lEqMNHdrEryxMFNHd3lcwh45TGPOzY6y5GvOpq/Y5qUnVMfxGtW8G3AQ+Td0yd7swekLz13aUIg9U7aHJdwwukd8e9Hg+YHQrlKyGkq4gccMO8mUFMDaqyruZAhzWneJWJU4TvvK4gsaZHi+uO5e8PB/bIlzhSAlPrghuROgQye4+JanCMlW0QIL9IAF4wWuHmmIXrxxMTIg+4Qqthpav/9iX
+    access:
+      - read
+      - write
+    posix:
+      Uid: 0000                                     # Iterate the Uid for every new user
+      Gid: 1002                                     # Copy the Gid for every user (1002 was the Gid for the sftp group created on the example EFS FileSystem)
+      #SecondaryGids:                               # Currently not supported, this would allow users to be members of multiple groups
+      #  - 1003
+    filesystem: ${FileSystemId}                     # Reference the parameter, this is required for the user's IAM role
+```
+
+### EFS future improvements
+
+- Support `SecondaryGids` - just need to figure out the Ruby to pull the values out
+- Create a custom resource to create the specified target directory on the FileSystem if it doesn't already exist. Currently this is a manual process, which is less than ideal
+
 ## Cfhighlander Setup
 
 install cfhighlander [gem](https://github.com/theonestack/cfhighlander)

lambdas/efs_filesystem_policy_creator_cr/cr_response.py

@@ -0,0 +1,57 @@
+import logging
+from urllib.request import urlopen, Request, HTTPError, URLError
+import json
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+class CustomResourceResponse:
+    def __init__(self, request_payload):
+        self.payload = request_payload
+        self.response = {
+            "StackId": request_payload["StackId"],
+            "RequestId": request_payload["RequestId"],
+            "LogicalResourceId": request_payload["LogicalResourceId"],
+            "Status": 'SUCCESS',
+        }
+
+    def respond_error(self, message):
+        self.response['Status'] = 'FAILED'
+        self.response['Reason'] = message
+        self.respond()
+
+    def respond(self, resource_attributes=None):
+        event = self.payload
+        response = self.response
+        ####
+        #### copied from https://github.com/ryansb/cfn-wrapper-python/blob/master/cfn_resource.py
+        ####
+        if resource_attributes is not None:
+            response['Data'] = resource_attributes
+
+        if event.get("PhysicalResourceId", False):
+            response["PhysicalResourceId"] = event["PhysicalResourceId"]
+
+        logger.debug("Received %s request with event: %s" % (event['RequestType'], json.dumps(event)))
+
+        serialized = json.dumps(response)
+        logger.info(f"Responding to {event['RequestType']} request with: {serialized}")
+
+        req_data = serialized.encode('utf-8')
+
+        req = Request(
+            event['ResponseURL'],
+            data=req_data,
+            headers={'Content-Length': len(req_data), 'Content-Type': ''}
+        )
+        req.get_method = lambda: 'PUT'
+
+        try:
+            urlopen(req)
+            logger.debug("Request to CFN API succeeded, nothing to do here")
+        except HTTPError as e:
+            logger.error("Callback to CFN API failed with status %d" % e.code)
+            logger.error("Response: %s" % e.reason)
+        except URLError as e:
+            logger.error("Failed to reach the server - %s" % e.reason)

lambdas/efs_filesystem_policy_creator_cr/index.py

@@ -0,0 +1,95 @@
+import boto3
+import botocore.exceptions
+import json
+import sys
+import os
+
+sys.path.append(f"{os.environ['LAMBDA_TASK_ROOT']}/lib")
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+import cr_response
+
+efs = boto3.client('efs')
+
+def handler(event, context):
+    print(f"Received event:{json.dumps(event)}")
+    lambda_response = cr_response.CustomResourceResponse(event)
+    filesystem_id = event['ResourceProperties']['FileSystemId']
+    filesystem_policy = event['ResourceProperties']['Policy']
+
+    try:
+        if event['RequestType'] == 'Create':
+            if check_filesystem_policy(filesystem_id) == False:
+                event['PhysicalResourceId'] = context.log_stream_name     # Set the PhysicalResourceId to the name of the current log stream for the function as there is no physical resource being created
+                create_filesystem_policy(filesystem_id, filesystem_policy)
+                lambda_response.respond()
+            else:
+                lambda_response.respond_error("There is already a policy on this FileSystem, overwriting or modifying an existing FileSystem policy is not currently supported.")
+        elif event['RequestType'] == 'Update':
+            update_filesystem_policy(filesystem_id, filesystem_policy)
+            lambda_response.respond()
+        elif event['RequestType'] == 'Delete':
+            delete_filesystem_policy(filesystem_id)
+            lambda_response.respond()
+    except Exception as e:
+        message = str(e)
+        lambda_response.respond_error(message)
+    return 'OK'
+
+def create_filesystem_policy(filesystem_id, filesystem_policy):
+    print(f"Creating a FileSystem policy for {filesystem_id}")
+    try:
+        response = efs.put_file_system_policy(
+            FileSystemId=filesystem_id,
+            Policy=json.dumps(filesystem_policy)
+        )
+        print(response)
+        return response
+    except Exception as error:
+        print(f"error:{error}\n")
+        raise error
+
+def update_filesystem_policy(filesystem_id, filesystem_policy):
+    print(f"Updating the FileSystem policy for {filesystem_id}")
+    # There is no update FileSystem policy method so we have to delete and create again
+    try:
+        response = efs.delete_file_system_policy(
+            FileSystemId=filesystem_id
+        )
+        print(response)
+        response = efs.put_file_system_policy(
+            FileSystemId=filesystem_id,
+            Policy=json.dumps(filesystem_policy)
+        )
+        print(response)
+        return response
+    except Exception as error:
+        print(f"error:{error}\n")
+        raise error
+
+def delete_filesystem_policy(filesystem_id):
+    print(f"Deleting the FileSystem policy for {filesystem_id}")
+    try:
+        response = efs.delete_file_system_policy(
+            FileSystemId=filesystem_id
+        )
+        print(response)
+        return response
+    except Exception as error:
+        print(f"error:{error}\n")
+        raise error
+
+def check_filesystem_policy(filesystem_id):  
+    try:
+        print("Checking the FileSystem for a FileSystem policy...")
+        response = efs.describe_file_system_policy(
+            FileSystemId=filesystem_id
+        )
+        print('Filesystem policy found.')
+        return True   
+    except botocore.exceptions.ClientError as error:
+        if error.response['Error']['Code'] == 'PolicyNotFound':
+            print('No FileSystem policy found.')
+            return False
+        else:
+            raise error

sftp.cfhighlander.rb

@@ -15,15 +15,19 @@
     if endpoint.upcase == 'VPC' && vpc_public == true
       ComponentParam 'AvailabilityZones', max_availability_zones, 
         allowedValues: (1..max_availability_zones).to_a,
-        description: 'Set the Availabiltiy Zone count for the sftp server',
+        description: 'Set the Availability Zone count for the sftp server',
         isGlobal: true 
       ComponentParam 'EIPs', '', 
         type: 'CommaDelimitedList',
         description: 'List of EIP Ids, if none are provided they will be created'
     end
+    if domain.upcase == 'EFS'
+      ComponentParam 'FileSystemId', ''
+    end
   end
 
   LambdaFunctions 'apigateway_identity_provider' if identity_provider.upcase == 'API_GATEWAY'
   LambdaFunctions 'output_vpc_endpoint_ips_custom_resource' if output_vpc_endpoint_ips
   LambdaFunctions 'dynamic_users_create_and_cleanup' if identity_provider.upcase == 'API_GATEWAY' and dynamic_users
-end
+  LambdaFunctions 'sftp_custom_resources' if domain.upcase == 'EFS'
+end