Cloud runner develop - latest fixes (#524)

2026-06-11 16:34:03 -07:00 · 2023-03-27 12:14:23 +01:00
parent 309d668d63
commit 7abb3a409d
91 changed files with 4943 additions and 3599 deletions
@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as core from '@actions/core';
 import * as SDK from 'aws-sdk';
 import { BaseStackFormation } from './cloud-formations/base-stack-formation';
@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as SDK from 'aws-sdk';
 import * as core from '@actions/core';
 import CloudRunner from '../../cloud-runner';
@@ -1,12 +1,12 @@
 import * as SDK from 'aws-sdk';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSError } from './aws-error';
 import CloudRunner from '../../cloud-runner';
 import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';

 export class AWSJobStack {
@@ -27,21 +27,19 @@ export class AWSJobStack {
  ): Promise<CloudRunnerAWSTaskDef> {
    const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
    let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
-    const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024';
-    const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072';
    taskDefCloudFormation = taskDefCloudFormation.replace(
      `ContainerCpu:
    Default: 1024`,
      `ContainerCpu:
-    Default: ${Number.parseInt(cpu)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
    );
    taskDefCloudFormation = taskDefCloudFormation.replace(
      `ContainerMemory:
    Default: 2048`,
      `ContainerMemory:
-    Default: ${Number.parseInt(memory)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
    );
-    if (CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (!CloudRunnerOptions.asyncCloudRunner) {
      taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
        taskDefCloudFormation,
        '# template resources logstream',
@@ -116,7 +114,7 @@ export class AWSJobStack {
      ...secretsMappedToCloudFormationParameters,
    ];
    CloudRunnerLogger.log(
-      `Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`,
+      `Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
    );
    let previousStackExists = true;
    while (previousStackExists) {
@@ -140,11 +138,16 @@ export class AWSJobStack {
      Capabilities: ['CAPABILITY_IAM'],
      Parameters: parameters,
    };
-
    try {
      CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
      await CF.createStack(createStackInput).promise();
      await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
+      const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
+      for (const parameter of parameters) {
+        if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
+          throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
+        }
+      }
    } catch (error) {
      await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
      throw error;
@@ -180,7 +183,7 @@ export class AWSJobStack {
    if (CloudRunnerOptions.useCleanupCron) {
      try {
        CloudRunnerLogger.log(`Creating job cleanup formation`);
-        CF.createStack(createCleanupStackInput).promise();
+        await CF.createStack(createCleanupStackInput).promise();

        // await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
      } catch (error) {
@@ -1,14 +1,14 @@
 import * as AWS from 'aws-sdk';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import * as core from '@actions/core';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
 import * as zlib from 'node:zlib';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { Input } from '../../..';
 import CloudRunner from '../../cloud-runner';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import GitHub from '../../../github';

 class AWSTaskRunner {
@@ -32,7 +32,7 @@ class AWSTaskRunner {
    const streamName =
      taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';

-    const task = await AWSTaskRunner.ECS.runTask({
+    const runParameters = {
      cluster,
      taskDefinition,
      platformVersion: '1.4.0',
@@ -41,7 +41,7 @@ class AWSTaskRunner {
          {
            name: taskDef.taskDefStackName,
            environment,
-            command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
+            command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
          },
        ],
      },
@@ -53,16 +53,23 @@ class AWSTaskRunner {
          securityGroups: [ContainerSecurityGroup],
        },
      },
-    }).promise();
+    };
+
+    if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
+      CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
+      throw new Error(`Container Overrides length must be at most 8192`);
+    }
+
+    const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
    const taskArn = task.tasks?.[0].taskArn || '';
    CloudRunnerLogger.log('Cloud runner job is starting');
    await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
    CloudRunnerLogger.log(
-      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
-        CloudRunnerOptions.watchCloudRunnerToEnd
-      } Async:${CloudRunnerOptions.asyncCloudRunner}`,
+      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
+        CloudRunnerOptions.asyncCloudRunner
+      }`,
    );
-    if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (CloudRunnerOptions.asyncCloudRunner) {
      const shouldCleanup: boolean = false;
      const output: string = '';
      CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
@@ -72,26 +79,31 @@ class AWSTaskRunner {

    CloudRunnerLogger.log(`Streaming...`);
    const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
-    await new Promise((resolve) => resolve(5000));
-    const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
-    const containerState = taskData.containers?.[0];
-    const exitCode = containerState?.exitCode || undefined;
+    let exitCode;
+    let containerState;
+    let taskData;
+    while (exitCode === undefined) {
+      await new Promise((resolve) => resolve(10000));
+      taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
+      containerState = taskData.containers?.[0];
+      exitCode = containerState?.exitCode;
+    }
    CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
-    const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
+    if (exitCode === undefined) {
+      CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
+    }
+    const wasSuccessful = exitCode === 0;
    if (wasSuccessful) {
      CloudRunnerLogger.log(`Cloud runner job has finished successfully`);

      return { output, shouldCleanup };
-    } else {
-      if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
-        throw new Error('Container exited with code 1');
-      }
-      const message = `Cloud runner job exit code ${exitCode}`;
-      taskData.overrides = undefined;
-      taskData.attachments = undefined;
-      CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
-      throw new Error(message);
    }
+
+    if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
+      throw new Error('Container exited with code 1');
+    }
+
+    throw new Error(`Task failed`);
  }

  private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
@@ -129,7 +141,7 @@ class AWSTaskRunner {
    const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
    let iterator = await AWSTaskRunner.getLogIterator(stream);

-    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
+    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
    CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
    await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch:  ${logBaseUrl}`, ``);
    let shouldReadLogs = true;
@@ -30,7 +30,7 @@ Parameters:
    Type: Number
    Description: How much CPU to give the container. 1024 is 1 CPU
  ContainerMemory:
-    Default: 2048
+    Default: 4096
    Type: Number
    Description: How much memory in megabytes to give the container
  BUILDGUID:
@@ -1,11 +1,11 @@
 import * as SDK from 'aws-sdk';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
 import AwsTaskRunner from './aws-task-runner';
 import { ProviderInterface } from '../provider-interface';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSJobStack as AwsJobStack } from './aws-job-stack';
 import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
 import { Input } from '../../..';
@@ -13,13 +13,13 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { TaskService } from './services/task-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';

 class AWSBuildEnvironment implements ProviderInterface {
  private baseStackName: string;

  constructor(buildParameters: BuildParameters) {
-    this.baseStackName = buildParameters.awsBaseStackName;
+    this.baseStackName = buildParameters.awsStackName;
  }
  async listResources(): Promise<ProviderResource[]> {
    await TaskService.getCloudFormationJobStacks();
@@ -75,7 +75,11 @@ class AWSBuildEnvironment implements ProviderInterface {
    branchName: string,
    // eslint-disable-next-line no-unused-vars
    defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
-  ) {}
+  ) {
+    process.env.AWS_REGION = Input.region;
+    const CF = new SDK.CloudFormation();
+    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
+  }

  async runTaskInWorkflow(
    buildGuid: string,
@@ -94,8 +98,6 @@ class AWSBuildEnvironment implements ProviderInterface {
    CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
    const entrypoint = ['/bin/sh'];
    const startTimeMs = Date.now();
-
-    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
    const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
      CF,
      buildGuid,
@@ -143,6 +145,9 @@ class AWSBuildEnvironment implements ProviderInterface {
    await CF.waitFor('stackDeleteComplete', {
      StackName: taskDef.taskDefStackName,
    }).promise();
+    await CF.waitFor('stackDeleteComplete', {
+      StackName: `${taskDef.taskDefStackName}-cleanup`,
+    }).promise();
    CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
    CloudRunnerLogger.log('Cleanup complete');
  }
@@ -1,6 +1,6 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { TaskService } from './task-service';

 export class GarbageCollectionService {
@@ -1,6 +1,6 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
 import AwsTaskRunner from '../aws-task-runner';
 import { ListObjectsRequest } from 'aws-sdk/clients/s3';
@@ -161,7 +161,7 @@ export class TaskService {
    process.env.AWS_REGION = Input.region;
    const s3 = new AWS.S3();
    const listRequest: ListObjectsRequest = {
-      Bucket: CloudRunner.buildParameters.awsBaseStackName,
+      Bucket: CloudRunner.buildParameters.awsStackName,
    };
    const results = await s3.listObjects(listRequest).promise();