fix: remove EFS from AWS stack - use S3 caching for storage instead

.github/workflows/cloud-runner-integrity.yml

@@ -611,28 +611,30 @@ jobs:
           rm -rf ./cloud-runner-cache/* || true
           docker system prune -f || true
           df -h
-      # SKIP: cloud-runner-environment test requires full CloudFormation support (VPC, EFS, ECS, IAM)
+      - name: Run cloud-runner-environment test (AWS provider)
-      # which LocalStack Community edition doesn't fully support. This test is validated by K8s provider tests.
+        timeout-minutes: 30
-      # Uncomment below to run with LocalStack Pro or real AWS credentials.
+        run: yarn run test "cloud-runner-environment" --detectOpenHandles --forceExit --runInBand
-      # - name: Run cloud-runner-environment test
+        env:
-      #   timeout-minutes: 30
+          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
-      #   run: yarn run test "cloud-runner-environment" --detectOpenHandles --forceExit --runInBand
+          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
-      #   env:
+          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
-      #     UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
+          PROJECT_PATH: test-project
-      #     UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
+          TARGET_PLATFORM: StandaloneWindows64
-      #     UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
+          cloudRunnerTests: true
-      #     PROJECT_PATH: test-project
+          versioning: None
-      #     TARGET_PLATFORM: StandaloneWindows64
+          KUBE_STORAGE_CLASS: local-path
-      #     cloudRunnerTests: true
+          PROVIDER_STRATEGY: aws
-      #     versioning: None
+          AWS_ACCESS_KEY_ID: test
-      #     KUBE_STORAGE_CLASS: local-path
+          AWS_SECRET_ACCESS_KEY: test
-      #     PROVIDER_STRATEGY: aws
+          AWS_ENDPOINT: http://localhost:4566
-      #     AWS_ACCESS_KEY_ID: test
+          AWS_ENDPOINT_URL: http://localhost:4566
-      #     AWS_SECRET_ACCESS_KEY: test
+          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
-      #     AWS_ENDPOINT: http://localhost:4566
+          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
-      #     AWS_ENDPOINT_URL: http://localhost:4566
+      - name: Clean up disk space
-      #     GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+        run: |
-      #     GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+          rm -rf ./cloud-runner-cache/* || true
+          docker system prune -f || true
+          df -h
       - name: Run cloud-runner-s3-steps test (AWS provider)
         timeout-minutes: 30
         run: yarn run test "cloud-runner-s3-steps" --detectOpenHandles --forceExit --runInBand

dist/index.js

@@ -1987,10 +1987,6 @@ class AWSJobStack {
                 ParameterKey: 'WorkingDirectory',
                 ParameterValue: workingdir,
             },
-            {
-                ParameterKey: 'EFSMountDirectory',
-                ParameterValue: mountdir,
-            },
             ...secretsMappedToCloudFormationParameters,
         ];
         cloud_runner_logger_1.default.log(`Starting AWS job with memory: ${cloud_runner_1.default.buildParameters.containerMemory} cpu: ${cloud_runner_1.default.buildParameters.containerCpu}`);
@@ -2438,18 +2434,6 @@ Resources:
     Properties:
       BucketName: !Ref EnvironmentName
 
-  EFSServerSecurityGroup:
-    Type: AWS::EC2::SecurityGroup
-    Properties:
-      GroupName: 'efs-server-endpoints'
-      GroupDescription: Which client ip addrs are allowed to access EFS server
-      VpcId: !Ref 'VPC'
-      SecurityGroupIngress:
-        - IpProtocol: tcp
-          FromPort: 2049
-          ToPort: 2049
-          SourceSecurityGroupId: !Ref ContainerSecurityGroup
-          #CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
   # A security group for the containers we will run in Fargate.
   # Rules are added to this security group based on what ingress you
   # add for the cluster.
@@ -2685,48 +2669,7 @@ Resources:
                   - 'kinesis:PutRecord'
                 Resource: '*'
 
-  #####################EFS#####################
-  EfsFileStorage:
-    Type: 'AWS::EFS::FileSystem'
-    Properties:
-      BackupPolicy:
-        Status: ENABLED
-      PerformanceMode: maxIO
-      Encrypted: false
-
-      FileSystemPolicy:
-        Version: '2012-10-17'
-        Statement:
-          - Effect: 'Allow'
-            Action:
-              - 'elasticfilesystem:ClientMount'
-              - 'elasticfilesystem:ClientWrite'
-              - 'elasticfilesystem:ClientRootAccess'
-            Principal:
-              AWS: '*'
-
-  MountTargetResource1:
-    Type: AWS::EFS::MountTarget
-    Properties:
-      FileSystemId: !Ref EfsFileStorage
-      SubnetId: !Ref PublicSubnetOne
-      SecurityGroups:
-        - !Ref EFSServerSecurityGroup
-
-  MountTargetResource2:
-    Type: AWS::EFS::MountTarget
-    Properties:
-      FileSystemId: !Ref EfsFileStorage
-      SubnetId: !Ref PublicSubnetTwo
-      SecurityGroups:
-        - !Ref EFSServerSecurityGroup
-
 Outputs:
-  EfsFileStorageId:
-    Description: 'The connection endpoint for the database.'
-    Value: !Ref EfsFileStorage
-    Export:
-      Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId
   ClusterName:
     Description: The name of the ECS cluster
     Value: !Ref 'ECSCluster'
@@ -3006,9 +2949,9 @@ Parameters:
     Description: >-
       (Optional) An IAM role to give the service's containers if the code within
       needs to access other AWS resources like S3 buckets, DynamoDB tables, etc
-  EFSMountDirectory:
+  WorkDir:
     Type: String
-    Default: '/efsdata'
+    Default: '/data'
   # template secrets p1 - input
 Mappings:
   SubnetConfig:
@@ -3042,12 +2985,6 @@ Resources:
       Cpu: !Ref ContainerCpu
       Memory: !Ref ContainerMemory
       NetworkMode: awsvpc
-      Volumes:
-        - Name: efs-data
-          EFSVolumeConfiguration:
-            FilesystemId:
-              'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId'
-            TransitEncryption: DISABLED
       RequiresCompatibilities:
         - FARGATE
       ExecutionRoleArn:
@@ -3075,10 +3012,6 @@ Resources:
             - Name: ALLOW_EMPTY_PASSWORD
               Value: 'yes'
             # template - env vars
-          MountPoints:
-            - SourceVolume: efs-data
-              ContainerPath: !Ref EFSMountDirectory
-              ReadOnly: false
           Secrets:
             # template secrets p3 - container def
           LogConfiguration:

src/model/cloud-runner/providers/aws/aws-job-stack.ts

@@ -126,10 +126,6 @@ export class AWSJobStack {
         ParameterKey: 'WorkingDirectory',
         ParameterValue: workingdir,
       },
-      {
-        ParameterKey: 'EFSMountDirectory',
-        ParameterValue: mountdir,
-      },
       ...secretsMappedToCloudFormationParameters,
     ];
     CloudRunnerLogger.log(

src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts

@@ -52,18 +52,6 @@ Resources:
     Properties:
       BucketName: !Ref EnvironmentName
 
-  EFSServerSecurityGroup:
-    Type: AWS::EC2::SecurityGroup
-    Properties:
-      GroupName: 'efs-server-endpoints'
-      GroupDescription: Which client ip addrs are allowed to access EFS server
-      VpcId: !Ref 'VPC'
-      SecurityGroupIngress:
-        - IpProtocol: tcp
-          FromPort: 2049
-          ToPort: 2049
-          SourceSecurityGroupId: !Ref ContainerSecurityGroup
-          #CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
   # A security group for the containers we will run in Fargate.
   # Rules are added to this security group based on what ingress you
   # add for the cluster.
@@ -299,48 +287,7 @@ Resources:
                   - 'kinesis:PutRecord'
                 Resource: '*'
 
-  #####################EFS#####################
-  EfsFileStorage:
-    Type: 'AWS::EFS::FileSystem'
-    Properties:
-      BackupPolicy:
-        Status: ENABLED
-      PerformanceMode: maxIO
-      Encrypted: false
-
-      FileSystemPolicy:
-        Version: '2012-10-17'
-        Statement:
-          - Effect: 'Allow'
-            Action:
-              - 'elasticfilesystem:ClientMount'
-              - 'elasticfilesystem:ClientWrite'
-              - 'elasticfilesystem:ClientRootAccess'
-            Principal:
-              AWS: '*'
-
-  MountTargetResource1:
-    Type: AWS::EFS::MountTarget
-    Properties:
-      FileSystemId: !Ref EfsFileStorage
-      SubnetId: !Ref PublicSubnetOne
-      SecurityGroups:
-        - !Ref EFSServerSecurityGroup
-
-  MountTargetResource2:
-    Type: AWS::EFS::MountTarget
-    Properties:
-      FileSystemId: !Ref EfsFileStorage
-      SubnetId: !Ref PublicSubnetTwo
-      SecurityGroups:
-        - !Ref EFSServerSecurityGroup
-
 Outputs:
-  EfsFileStorageId:
-    Description: 'The connection endpoint for the database.'
-    Value: !Ref EfsFileStorage
-    Export:
-      Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId
   ClusterName:
     Description: The name of the ECS cluster
     Value: !Ref 'ECSCluster'

src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts

@@ -54,9 +54,9 @@ Parameters:
     Description: >-
       (Optional) An IAM role to give the service's containers if the code within
       needs to access other AWS resources like S3 buckets, DynamoDB tables, etc
-  EFSMountDirectory:
+  WorkDir:
     Type: String
-    Default: '/efsdata'
+    Default: '/data'
   # template secrets p1 - input
 Mappings:
   SubnetConfig:
@@ -90,12 +90,6 @@ Resources:
       Cpu: !Ref ContainerCpu
       Memory: !Ref ContainerMemory
       NetworkMode: awsvpc
-      Volumes:
-        - Name: efs-data
-          EFSVolumeConfiguration:
-            FilesystemId:
-              'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId'
-            TransitEncryption: DISABLED
       RequiresCompatibilities:
         - FARGATE
       ExecutionRoleArn:
@@ -123,10 +117,6 @@ Resources:
             - Name: ALLOW_EMPTY_PASSWORD
               Value: 'yes'
             # template - env vars
-          MountPoints:
-            - SourceVolume: efs-data
-              ContainerPath: !Ref EFSMountDirectory
-              ReadOnly: false
           Secrets:
             # template secrets p3 - container def
           LogConfiguration: