mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-12 17:03:55 -07:00
Rename Cloud Runner to Orchestrator (#775)
* Rename "Cloud Runner" to "Orchestrator" across entire codebase Breaking change: All CloudRunner classes, options, environment variables, and action.yml inputs have been renamed to Orchestrator equivalents. - Renamed src/model/cloud-runner/ directory to src/model/orchestrator/ - Renamed all cloud-runner-* files to orchestrator-* - Renamed all CloudRunner* classes to Orchestrator* (15+ classes) - Renamed all cloudRunner* properties to orchestrator* equivalents - Renamed CLOUD_RUNNER_* env vars to ORCHESTRATOR_* - Updated action.yml [CloudRunner] markers to [Orchestrator] - Updated workflow files and package.json test scripts - Updated all runtime strings (cache paths, log messages, branch refs) - Rebuilt dist/index.js No backward compatibility layer is provided. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Remove tracked log/temp files and add to .gitignore Remove $LOG_FILE and temp/job-log.txt debug artifacts that should not be in the repository. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frostebite
@@ -0,0 +1,222 @@
|
||||
# Provider Loader Dynamic Imports
|
||||
|
||||
## What is a Provider?
|
||||
|
||||
A **provider** is a pluggable backend that Orchestrator uses to run builds and workflows. Examples include **AWS**, **Kubernetes**, or local execution. Each provider implements the [ProviderInterface](https://github.com/game-ci/unity-builder/blob/main/src/model/orchestrator/providers/provider-interface.ts), which defines the common lifecycle methods (setup, run, cleanup, garbage collection, etc.).
|
||||
|
||||
This abstraction makes Orchestrator flexible: you can switch execution environments or add your own provider (via npm package, GitHub repo, or local path) without changing the rest of your pipeline.
|
||||
|
||||
## Dynamic Provider Loading
|
||||
|
||||
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub repositories, and NPM packages.
|
||||
|
||||
## Features
|
||||
|
||||
- **Local File Paths**: Load providers from relative or absolute file paths
|
||||
- **GitHub URLs**: Clone and load providers from GitHub repositories with automatic updates
|
||||
- **NPM Packages**: Load providers from installed NPM packages
|
||||
- **Automatic Updates**: GitHub repositories are automatically updated when changes are available
|
||||
- **Caching**: Local caching of cloned repositories for improved performance
|
||||
- **Fallback Support**: Graceful fallback to local provider if loading fails
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Loading Built-in Providers
|
||||
|
||||
```typescript
|
||||
import { ProviderLoader } from './provider-loader';
|
||||
|
||||
// Load built-in providers
|
||||
const awsProvider = await ProviderLoader.loadProvider('aws', buildParameters);
|
||||
const k8sProvider = await ProviderLoader.loadProvider('k8s', buildParameters);
|
||||
```
|
||||
|
||||
### Loading Local Providers
|
||||
|
||||
```typescript
|
||||
// Load from relative path
|
||||
const localProvider = await ProviderLoader.loadProvider('./my-local-provider', buildParameters);
|
||||
|
||||
// Load from absolute path
|
||||
const absoluteProvider = await ProviderLoader.loadProvider('/path/to/provider', buildParameters);
|
||||
```
|
||||
|
||||
### Loading GitHub Providers
|
||||
|
||||
```typescript
|
||||
// Load from GitHub URL
|
||||
const githubProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Load from specific branch
|
||||
const branchProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider/tree/develop',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Load from specific path in repository
|
||||
const pathProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider/tree/main/src/providers',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Shorthand notation
|
||||
const shorthandProvider = await ProviderLoader.loadProvider('user/repo', buildParameters);
|
||||
const branchShorthand = await ProviderLoader.loadProvider('user/repo@develop', buildParameters);
|
||||
```
|
||||
|
||||
### Loading NPM Packages
|
||||
|
||||
```typescript
|
||||
// Load from NPM package
|
||||
const npmProvider = await ProviderLoader.loadProvider('my-provider-package', buildParameters);
|
||||
|
||||
// Load from scoped NPM package
|
||||
const scopedProvider = await ProviderLoader.loadProvider('@scope/my-provider', buildParameters);
|
||||
```
|
||||
|
||||
## Provider Interface
|
||||
|
||||
All providers must implement the `ProviderInterface`:
|
||||
|
||||
```typescript
|
||||
interface ProviderInterface {
|
||||
cleanupWorkflow(): Promise<void>;
|
||||
setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void>;
|
||||
runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string>;
|
||||
garbageCollect(): Promise<void>;
|
||||
listResources(): Promise<ProviderResource[]>;
|
||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||
watchWorkflow(): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
## Example Provider Implementation
|
||||
|
||||
```typescript
|
||||
// my-provider.ts
|
||||
import { ProviderInterface } from './provider-interface';
|
||||
import BuildParameters from './build-parameters';
|
||||
|
||||
export default class MyProvider implements ProviderInterface {
|
||||
constructor(private buildParameters: BuildParameters) {}
|
||||
|
||||
async cleanupWorkflow(): Promise<void> {
|
||||
// Cleanup logic
|
||||
}
|
||||
|
||||
async setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void> {
|
||||
// Setup logic
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string> {
|
||||
// Task execution logic
|
||||
return 'Task completed';
|
||||
}
|
||||
|
||||
async garbageCollect(): Promise<void> {
|
||||
// Garbage collection logic
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<void> {
|
||||
// Watch logic
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Utility Methods
|
||||
|
||||
### Analyze Provider Source
|
||||
|
||||
```typescript
|
||||
// Analyze a provider source without loading it
|
||||
const sourceInfo = ProviderLoader.analyzeProviderSource('https://github.com/user/repo');
|
||||
console.log(sourceInfo.type); // 'github'
|
||||
console.log(sourceInfo.owner); // 'user'
|
||||
console.log(sourceInfo.repo); // 'repo'
|
||||
```
|
||||
|
||||
### Clean Up Cache
|
||||
|
||||
```typescript
|
||||
// Clean up old cached repositories (older than 30 days)
|
||||
await ProviderLoader.cleanupCache();
|
||||
|
||||
// Clean up repositories older than 7 days
|
||||
await ProviderLoader.cleanupCache(7);
|
||||
```
|
||||
|
||||
### Get Available Providers
|
||||
|
||||
```typescript
|
||||
// Get list of built-in providers
|
||||
const providers = ProviderLoader.getAvailableProviders();
|
||||
console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local']
|
||||
```
|
||||
|
||||
## Supported URL Formats
|
||||
|
||||
### GitHub URLs
|
||||
- `https://github.com/user/repo`
|
||||
- `https://github.com/user/repo.git`
|
||||
- `https://github.com/user/repo/tree/branch`
|
||||
- `https://github.com/user/repo/tree/branch/path/to/provider`
|
||||
- `git@github.com:user/repo.git`
|
||||
|
||||
### Shorthand GitHub References
|
||||
- `user/repo`
|
||||
- `user/repo@branch`
|
||||
- `user/repo@branch/path/to/provider`
|
||||
|
||||
### Local Paths
|
||||
- `./relative/path`
|
||||
- `../relative/path`
|
||||
- `/absolute/path`
|
||||
- `C:\\path\\to\\provider` (Windows)
|
||||
|
||||
### NPM Packages
|
||||
- `package-name`
|
||||
- `@scope/package-name`
|
||||
|
||||
## Caching
|
||||
|
||||
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the repository owner, name, and branch. This ensures that:
|
||||
|
||||
1. Repositories are only cloned once
|
||||
2. Updates are checked and applied automatically
|
||||
3. Performance is improved for repeated loads
|
||||
4. Storage is managed efficiently
|
||||
|
||||
## Error Handling
|
||||
|
||||
The provider loader includes comprehensive error handling:
|
||||
|
||||
- **Missing packages**: Clear error messages when providers cannot be found
|
||||
- **Interface validation**: Ensures providers implement the required interface
|
||||
- **Git operations**: Handles network issues and repository access problems
|
||||
- **Fallback mechanism**: Falls back to local provider if loading fails
|
||||
|
||||
## Configuration
|
||||
|
||||
The provider loader can be configured through environment variables:
|
||||
|
||||
- `PROVIDER_CACHE_DIR`: Custom cache directory (default: `.provider-cache`)
|
||||
- `GIT_TIMEOUT`: Git operation timeout in milliseconds (default: 30000)
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use specific branches or tags**: Always specify the branch or specific tag when loading from GitHub
|
||||
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
||||
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
||||
4. **Test locally first**: Test providers locally before deploying
|
||||
5. **Use semantic versioning**: Tag your provider repositories for stable versions
|
||||
@@ -0,0 +1,170 @@
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
import {
|
||||
CloudFormation,
|
||||
CreateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
CreateStackCommandInput,
|
||||
DescribeStacksCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
DescribeStacksCommandInput,
|
||||
ListStacksCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
Parameter,
|
||||
UpdateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
UpdateStackCommandInput,
|
||||
waitUntilStackCreateComplete,
|
||||
waitUntilStackUpdateComplete,
|
||||
} from '@aws-sdk/client-cloudformation';
|
||||
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
export class AWSBaseStack {
|
||||
constructor(baseStackName: string) {
|
||||
this.baseStackName = baseStackName;
|
||||
}
|
||||
private baseStackName: string;
|
||||
|
||||
async setupBaseStack(CF: CloudFormation) {
|
||||
const baseStackName = this.baseStackName;
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
|
||||
const baseStack = BaseStackFormation.formation;
|
||||
|
||||
// Cloud Formation Input
|
||||
const describeStackInput: DescribeStacksCommandInput = {
|
||||
StackName: baseStackName,
|
||||
};
|
||||
const parametersWithoutHash: Parameter[] = [{ ParameterKey: 'EnvironmentName', ParameterValue: baseStackName }];
|
||||
const parametersHash = crypto
|
||||
.createHash('md5')
|
||||
.update(baseStack + JSON.stringify(parametersWithoutHash))
|
||||
.digest('hex');
|
||||
const parameters: Parameter[] = [
|
||||
...parametersWithoutHash,
|
||||
...[{ ParameterKey: 'Version', ParameterValue: parametersHash }],
|
||||
];
|
||||
const updateInput: UpdateStackCommandInput = {
|
||||
StackName: baseStackName,
|
||||
TemplateBody: baseStack,
|
||||
Parameters: parameters,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
};
|
||||
const createStackInput: CreateStackCommandInput = {
|
||||
StackName: baseStackName,
|
||||
TemplateBody: baseStack,
|
||||
Parameters: parameters,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
};
|
||||
|
||||
const stacks = await CF.send(
|
||||
new ListStacksCommand({
|
||||
StackStatusFilter: [
|
||||
'CREATE_IN_PROGRESS',
|
||||
'UPDATE_IN_PROGRESS',
|
||||
'UPDATE_COMPLETE',
|
||||
'CREATE_COMPLETE',
|
||||
'ROLLBACK_COMPLETE',
|
||||
],
|
||||
}),
|
||||
);
|
||||
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
|
||||
const stackExists: boolean = stackNames.includes(baseStackName);
|
||||
const describeStack = async () => {
|
||||
return await CF.send(new DescribeStacksCommand(describeStackInput));
|
||||
};
|
||||
try {
|
||||
if (!stackExists) {
|
||||
OrchestratorLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
||||
let created = false;
|
||||
try {
|
||||
await CF.send(new CreateStackCommand(createStackInput));
|
||||
created = true;
|
||||
} catch (error: any) {
|
||||
const message = `${error?.name ?? ''} ${error?.message ?? ''}`;
|
||||
if (message.includes('AlreadyExistsException')) {
|
||||
OrchestratorLogger.log(`Base stack already exists, continuing with describe`);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (created) {
|
||||
OrchestratorLogger.log(`created stack (version: ${parametersHash})`);
|
||||
}
|
||||
}
|
||||
const CFState = await describeStack();
|
||||
let stack = CFState.Stacks?.[0];
|
||||
if (!stack) {
|
||||
throw new Error(`Base stack doesn't exist, even after creation, stackExists check: ${stackExists}`);
|
||||
}
|
||||
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
|
||||
|
||||
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
|
||||
OrchestratorLogger.log(
|
||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`,
|
||||
);
|
||||
await waitUntilStackCreateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
describeStackInput,
|
||||
);
|
||||
}
|
||||
|
||||
if (stackExists) {
|
||||
OrchestratorLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
||||
if (parametersHash !== stackVersion) {
|
||||
OrchestratorLogger.log(`Attempting update of base stack`);
|
||||
try {
|
||||
await CF.send(new UpdateStackCommand(updateInput));
|
||||
} catch (error: any) {
|
||||
if (error['message'].includes('No updates are to be performed')) {
|
||||
OrchestratorLogger.log(`No updates are to be performed`);
|
||||
} else {
|
||||
OrchestratorLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
||||
OrchestratorLogger.log(error['message']);
|
||||
}
|
||||
OrchestratorLogger.log(`Continuing...`);
|
||||
}
|
||||
} else {
|
||||
OrchestratorLogger.log(`No update required`);
|
||||
}
|
||||
stack = (await describeStack()).Stacks?.[0];
|
||||
if (!stack) {
|
||||
throw new Error(
|
||||
`Base stack doesn't exist, even after updating and creation, stackExists check: ${stackExists}`,
|
||||
);
|
||||
}
|
||||
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
|
||||
OrchestratorLogger.log(
|
||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`,
|
||||
);
|
||||
await waitUntilStackUpdateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
describeStackInput,
|
||||
);
|
||||
}
|
||||
}
|
||||
OrchestratorLogger.log('base stack is now ready');
|
||||
} catch (error) {
|
||||
core.error(JSON.stringify(await describeStack(), undefined, 4));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { CloudFormation } from '@aws-sdk/client-cloudformation';
|
||||
import { ECS } from '@aws-sdk/client-ecs';
|
||||
import { Kinesis } from '@aws-sdk/client-kinesis';
|
||||
import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { S3 } from '@aws-sdk/client-s3';
|
||||
import { Input } from '../../..';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
|
||||
export class AwsClientFactory {
|
||||
private static cloudFormation: CloudFormation;
|
||||
private static ecs: ECS;
|
||||
private static kinesis: Kinesis;
|
||||
private static cloudWatchLogs: CloudWatchLogs;
|
||||
private static s3: S3;
|
||||
|
||||
private static getCredentials() {
|
||||
// Explicitly provide credentials from environment variables for LocalStack compatibility
|
||||
// LocalStack accepts any credentials, but the AWS SDK needs them to be explicitly set
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
|
||||
if (accessKeyId && secretAccessKey) {
|
||||
return {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
}
|
||||
|
||||
// Return undefined to let AWS SDK use default credential chain
|
||||
return;
|
||||
}
|
||||
|
||||
static getCloudFormation(): CloudFormation {
|
||||
if (!this.cloudFormation) {
|
||||
this.cloudFormation = new CloudFormation({
|
||||
region: Input.region,
|
||||
endpoint: OrchestratorOptions.awsCloudFormationEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return this.cloudFormation;
|
||||
}
|
||||
|
||||
static getECS(): ECS {
|
||||
if (!this.ecs) {
|
||||
this.ecs = new ECS({
|
||||
region: Input.region,
|
||||
endpoint: OrchestratorOptions.awsEcsEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return this.ecs;
|
||||
}
|
||||
|
||||
static getKinesis(): Kinesis {
|
||||
if (!this.kinesis) {
|
||||
this.kinesis = new Kinesis({
|
||||
region: Input.region,
|
||||
endpoint: OrchestratorOptions.awsKinesisEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return this.kinesis;
|
||||
}
|
||||
|
||||
static getCloudWatchLogs(): CloudWatchLogs {
|
||||
if (!this.cloudWatchLogs) {
|
||||
this.cloudWatchLogs = new CloudWatchLogs({
|
||||
region: Input.region,
|
||||
endpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return this.cloudWatchLogs;
|
||||
}
|
||||
|
||||
static getS3(): S3 {
|
||||
if (!this.s3) {
|
||||
this.s3 = new S3({
|
||||
region: Input.region,
|
||||
endpoint: OrchestratorOptions.awsS3Endpoint,
|
||||
forcePathStyle: true,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return this.s3;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
||||
|
||||
export class AWSCloudFormationTemplates {
|
||||
public static getParameterTemplate(p1: string) {
|
||||
return `
|
||||
${p1}:
|
||||
Type: String
|
||||
Default: ''
|
||||
`;
|
||||
}
|
||||
|
||||
public static getSecretTemplate(p1: string) {
|
||||
return `
|
||||
${p1}Secret:
|
||||
Type: AWS::SecretsManager::Secret
|
||||
Properties:
|
||||
Name: '${p1}'
|
||||
SecretString: !Ref ${p1}
|
||||
`;
|
||||
}
|
||||
|
||||
public static getSecretDefinitionTemplate(p1: string, p2: string) {
|
||||
return `
|
||||
Secrets:
|
||||
- Name: '${p1}'
|
||||
ValueFrom: !Ref ${p2}Secret
|
||||
`;
|
||||
}
|
||||
|
||||
public static insertAtTemplate(template: string, insertionKey: string, insertion: string) {
|
||||
const index = template.search(insertionKey) + insertionKey.length + '\n'.length;
|
||||
template = [template.slice(0, index), insertion, template.slice(index)].join('');
|
||||
|
||||
return template;
|
||||
}
|
||||
|
||||
public static readTaskCloudFormationTemplate(): string {
|
||||
return TaskDefinitionFormation.formation;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
|
||||
import * as core from '@actions/core';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
|
||||
export class AWSError {
|
||||
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
|
||||
OrchestratorLogger.log('aws error: ');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
||||
OrchestratorLogger.log('Getting events and resources for task stack');
|
||||
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
|
||||
OrchestratorLogger.log(JSON.stringify(events, undefined, 4));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
import {
|
||||
CloudFormation,
|
||||
CreateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
CreateStackCommandInput,
|
||||
DescribeStackResourcesCommand,
|
||||
DescribeStacksCommand,
|
||||
ListStacksCommand,
|
||||
waitUntilStackCreateComplete,
|
||||
} from '@aws-sdk/client-cloudformation';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { AWSError } from './aws-error';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
export class AWSJobStack {
|
||||
private baseStackName: string;
|
||||
constructor(baseStackName: string) {
|
||||
this.baseStackName = baseStackName;
|
||||
}
|
||||
|
||||
public async setupCloudFormations(
|
||||
CF: CloudFormation,
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
entrypoint: string[],
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<OrchestratorAWSTaskDef> {
|
||||
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
||||
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||
`ContainerCpu:
|
||||
Default: 1024`,
|
||||
`ContainerCpu:
|
||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerCpu)}`,
|
||||
);
|
||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||
`ContainerMemory:
|
||||
Default: 2048`,
|
||||
`ContainerMemory:
|
||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerMemory)}`,
|
||||
);
|
||||
if (!OrchestratorOptions.asyncOrchestrator) {
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'# template resources logstream',
|
||||
TaskDefinitionFormation.streamLogs,
|
||||
);
|
||||
}
|
||||
for (const secret of secrets) {
|
||||
secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace(
|
||||
/[^\dA-Za-z]/g,
|
||||
'',
|
||||
)}`;
|
||||
if (typeof secret.ParameterValue == 'number') {
|
||||
secret.ParameterValue = `${secret.ParameterValue}`;
|
||||
}
|
||||
if (!secret.ParameterValue || secret.ParameterValue === '') {
|
||||
secrets = secrets.filter((x) => x !== secret);
|
||||
continue;
|
||||
}
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'p1 - input',
|
||||
AWSCloudFormationTemplates.getParameterTemplate(secret.ParameterKey),
|
||||
);
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'# template resources secrets',
|
||||
AWSCloudFormationTemplates.getSecretTemplate(`${secret.ParameterKey}`),
|
||||
);
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'p3 - container def',
|
||||
AWSCloudFormationTemplates.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey),
|
||||
);
|
||||
}
|
||||
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
|
||||
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
|
||||
});
|
||||
const logGroupName = `${this.baseStackName}/${taskDefStackName}`;
|
||||
const parameters = [
|
||||
{
|
||||
ParameterKey: 'EnvironmentName',
|
||||
ParameterValue: this.baseStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'ImageUrl',
|
||||
ParameterValue: image,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'ServiceName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'LogGroupName',
|
||||
ParameterValue: logGroupName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'Command',
|
||||
ParameterValue: 'echo "this template should be overwritten when running a task"',
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EntryPoint',
|
||||
ParameterValue: entrypoint.join(','),
|
||||
},
|
||||
{
|
||||
ParameterKey: 'WorkingDirectory',
|
||||
ParameterValue: workingdir,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EFSMountDirectory',
|
||||
ParameterValue: mountdir,
|
||||
},
|
||||
...secretsMappedToCloudFormationParameters,
|
||||
];
|
||||
OrchestratorLogger.log(
|
||||
`Starting AWS job with memory: ${Orchestrator.buildParameters.containerMemory} cpu: ${Orchestrator.buildParameters.containerCpu}`,
|
||||
);
|
||||
let previousStackExists = true;
|
||||
while (previousStackExists) {
|
||||
previousStackExists = false;
|
||||
const stacks = await CF.send(new ListStacksCommand({}));
|
||||
if (!stacks.StackSummaries) {
|
||||
throw new Error('Faild to get stacks');
|
||||
}
|
||||
for (let index = 0; index < stacks.StackSummaries.length; index++) {
|
||||
const element = stacks.StackSummaries[index];
|
||||
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
||||
previousStackExists = true;
|
||||
OrchestratorLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
}
|
||||
}
|
||||
}
|
||||
const createStackInput: CreateStackCommandInput = {
|
||||
StackName: taskDefStackName,
|
||||
TemplateBody: taskDefCloudFormation,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
Parameters: parameters,
|
||||
};
|
||||
try {
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
OrchestratorLogger.log(
|
||||
`Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`,
|
||||
);
|
||||
await CF.send(new CreateStackCommand(createStackInput));
|
||||
await waitUntilStackCreateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{ StackName: taskDefStackName },
|
||||
);
|
||||
const describeStack = await CF.send(new DescribeStacksCommand({ StackName: taskDefStackName }));
|
||||
for (const parameter of parameters) {
|
||||
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
|
||||
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const createCleanupStackInput: CreateStackCommandInput = {
|
||||
StackName: `${taskDefStackName}-cleanup`,
|
||||
TemplateBody: CleanupCronFormation.formation,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
Parameters: [
|
||||
{
|
||||
ParameterKey: 'StackName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'DeleteStackName',
|
||||
ParameterValue: `${taskDefStackName}-cleanup`,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'TTL',
|
||||
ParameterValue: `1080`,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDGUID',
|
||||
ParameterValue: Orchestrator.buildParameters.buildGuid,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EnvironmentName',
|
||||
ParameterValue: this.baseStackName,
|
||||
},
|
||||
],
|
||||
};
|
||||
if (OrchestratorOptions.useCleanupCron) {
|
||||
try {
|
||||
OrchestratorLogger.log(`Creating job cleanup formation`);
|
||||
await CF.send(new CreateStackCommand(createCleanupStackInput));
|
||||
|
||||
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
|
||||
} catch (error) {
|
||||
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const taskDefResources = (
|
||||
await CF.send(
|
||||
new DescribeStackResourcesCommand({
|
||||
StackName: taskDefStackName,
|
||||
}),
|
||||
)
|
||||
).StackResources;
|
||||
|
||||
const baseResources = (await CF.send(new DescribeStackResourcesCommand({ StackName: this.baseStackName })))
|
||||
.StackResources;
|
||||
|
||||
return {
|
||||
taskDefStackName,
|
||||
taskDefCloudFormation,
|
||||
taskDefResources,
|
||||
baseResources,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
|
||||
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import * as core from '@actions/core';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import * as zlib from 'node:zlib';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { Input } from '../../..';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import GitHub from '../../../github';
|
||||
import { AwsClientFactory } from './aws-client-factory';
|
||||
|
||||
class AWSTaskRunner {
|
||||
private static readonly encodedUnderscore = `$252F`;
|
||||
|
||||
/**
|
||||
* Transform localhost endpoints to host.docker.internal for container environments.
|
||||
* When LocalStack is used, ECS tasks run in Docker containers that need to reach
|
||||
* LocalStack on the host machine via host.docker.internal.
|
||||
*/
|
||||
private static transformEndpointsForContainer(
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
): OrchestratorEnvironmentVariable[] {
|
||||
const endpointEnvironmentNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
|
||||
return environment.map((x) => {
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvironmentNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
// Replace localhost with host.docker.internal so ECS containers can access host services
|
||||
value = value
|
||||
.replace('http://localhost', 'http://host.docker.internal')
|
||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
||||
OrchestratorLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
||||
}
|
||||
|
||||
return { name: x.name, value };
|
||||
});
|
||||
}
|
||||
|
||||
static async runTask(
|
||||
taskDef: OrchestratorAWSTaskDef,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
commands: string,
|
||||
): Promise<{ output: string; shouldCleanup: boolean }> {
|
||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
const taskDefinition =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
|
||||
const SubnetOne =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
|
||||
const SubnetTwo =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
|
||||
const ContainerSecurityGroup =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
|
||||
const streamName =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||
|
||||
// Transform localhost endpoints for container environment
|
||||
const transformedEnvironment = AWSTaskRunner.transformEndpointsForContainer(environment);
|
||||
|
||||
const runParameters = {
|
||||
cluster,
|
||||
taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment: transformedEnvironment,
|
||||
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, Orchestrator.buildParameters)],
|
||||
},
|
||||
],
|
||||
},
|
||||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [SubnetOne, SubnetTwo],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [ContainerSecurityGroup],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
|
||||
OrchestratorLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
||||
throw new Error(`Container Overrides length must be at most 8192`);
|
||||
}
|
||||
|
||||
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
OrchestratorLogger.log('Orchestrator job is starting');
|
||||
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
||||
OrchestratorLogger.log(
|
||||
`Orchestrator job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
||||
OrchestratorOptions.asyncOrchestrator
|
||||
}`,
|
||||
);
|
||||
if (OrchestratorOptions.asyncOrchestrator) {
|
||||
const shouldCleanup: boolean = false;
|
||||
const output: string = '';
|
||||
OrchestratorLogger.log(`Watch Orchestrator To End: false`);
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Streaming...`);
|
||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
||||
let exitCode;
|
||||
let containerState;
|
||||
let taskData;
|
||||
while (exitCode === undefined) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10000));
|
||||
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
const containers = taskData?.containers as any[] | undefined;
|
||||
if (!containers || containers.length === 0) {
|
||||
continue;
|
||||
}
|
||||
containerState = containers[0];
|
||||
exitCode = containerState?.exitCode;
|
||||
}
|
||||
OrchestratorLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||
if (exitCode === undefined) {
|
||||
OrchestratorLogger.logWarning(`Undefined exitcode for container`);
|
||||
}
|
||||
const wasSuccessful = exitCode === 0;
|
||||
if (wasSuccessful) {
|
||||
OrchestratorLogger.log(`Orchestrator job has finished successfully`);
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
|
||||
if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
|
||||
throw new Error('Container exited with code 1');
|
||||
}
|
||||
|
||||
throw new Error(`Task failed`);
|
||||
}
|
||||
|
||||
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
|
||||
try {
|
||||
await waitUntilTasksRunning(
|
||||
{
|
||||
client: AwsClientFactory.getECS(),
|
||||
maxWaitTime: 300,
|
||||
minDelay: 5,
|
||||
maxDelay: 30,
|
||||
},
|
||||
{ tasks: [taskArn], cluster },
|
||||
);
|
||||
} catch (error_) {
|
||||
const error = error_ as Error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
OrchestratorLogger.log(`Orchestrator job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
|
||||
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
static async describeTasks(clusterName: string, taskArn: string) {
|
||||
const maxAttempts = 10;
|
||||
let delayMs = 1000;
|
||||
const maxDelayMs = 60000;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const tasks = await AwsClientFactory.getECS().send(
|
||||
new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }),
|
||||
);
|
||||
if (tasks.tasks?.[0]) {
|
||||
return tasks.tasks?.[0];
|
||||
}
|
||||
throw new Error('No task found');
|
||||
} catch (error: any) {
|
||||
const isThrottle = error?.name === 'ThrottlingException' || /rate exceeded/i.test(String(error?.message));
|
||||
if (!isThrottle || attempt === maxAttempts) {
|
||||
throw error;
|
||||
}
|
||||
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
|
||||
const sleepMs = delayMs + jitterMs;
|
||||
OrchestratorLogger.log(
|
||||
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, sleepMs));
|
||||
delayMs = Math.min(delayMs * 2, maxDelayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
OrchestratorLogger.log(`Streaming...`);
|
||||
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
||||
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
||||
|
||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${Orchestrator.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${Orchestrator.buildParameters.awsStackName}-${Orchestrator.buildParameters.buildGuid}`;
|
||||
OrchestratorLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
||||
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
let timestamp: number = 0;
|
||||
let output = '';
|
||||
while (shouldReadLogs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
|
||||
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3500));
|
||||
}
|
||||
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
|
||||
iterator,
|
||||
shouldReadLogs,
|
||||
output,
|
||||
shouldCleanup,
|
||||
));
|
||||
}
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
|
||||
private static async handleLogStreamIteration(
|
||||
iterator: string,
|
||||
shouldReadLogs: boolean,
|
||||
output: string,
|
||||
shouldCleanup: boolean,
|
||||
) {
|
||||
let records: any;
|
||||
try {
|
||||
records = await AwsClientFactory.getKinesis().send(new GetRecordsCommand({ ShardIterator: iterator }));
|
||||
} catch (error: any) {
|
||||
const isThrottle = error?.name === 'ThrottlingException' || /rate exceeded/i.test(String(error?.message));
|
||||
if (isThrottle) {
|
||||
const baseBackoffMs = 1000;
|
||||
const jitterMs = Math.floor(Math.random() * 1000);
|
||||
const sleepMs = baseBackoffMs + jitterMs;
|
||||
OrchestratorLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
|
||||
await new Promise((r) => setTimeout(r, sleepMs));
|
||||
|
||||
return { iterator, shouldReadLogs, output, shouldCleanup };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
iterator = records.NextShardIterator || '';
|
||||
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
|
||||
records,
|
||||
iterator,
|
||||
shouldReadLogs,
|
||||
output,
|
||||
shouldCleanup,
|
||||
));
|
||||
|
||||
return { iterator, shouldReadLogs, output, shouldCleanup };
|
||||
}
|
||||
|
||||
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
|
||||
if (taskData?.lastStatus === 'UNKNOWN') {
|
||||
OrchestratorLogger.log('## Orchestrator job unknwon');
|
||||
}
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
if (timestamp === 0) {
|
||||
OrchestratorLogger.log('## Orchestrator job stopped, streaming end of logs');
|
||||
timestamp = Date.now();
|
||||
}
|
||||
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
|
||||
OrchestratorLogger.log('## Orchestrator status is not RUNNING for 30 seconds, last query for logs');
|
||||
shouldReadLogs = false;
|
||||
}
|
||||
OrchestratorLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
||||
}
|
||||
|
||||
return { timestamp, shouldReadLogs };
|
||||
}
|
||||
|
||||
private static logRecords(
|
||||
records: any,
|
||||
iterator: string,
|
||||
shouldReadLogs: boolean,
|
||||
output: string,
|
||||
shouldCleanup: boolean,
|
||||
) {
|
||||
if ((records.Records ?? []).length > 0 && iterator) {
|
||||
for (const record of records.Records ?? []) {
|
||||
const json = JSON.parse(
|
||||
zlib.gunzipSync(Buffer.from(record.Data as unknown as string, 'base64')).toString('utf8'),
|
||||
);
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (const logEvent of json.logEvents) {
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
logEvent.message,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { shouldReadLogs, output, shouldCleanup };
|
||||
}
|
||||
|
||||
private static async getLogStream(kinesisStreamName: string) {
|
||||
return await AwsClientFactory.getKinesis().send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
|
||||
}
|
||||
|
||||
private static async getLogIterator(stream: any) {
|
||||
return (
|
||||
(
|
||||
await AwsClientFactory.getKinesis().send(
|
||||
new GetShardIteratorCommand({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription?.StreamName ?? '',
|
||||
ShardId: stream.StreamDescription?.Shards?.[0]?.ShardId || '',
|
||||
}),
|
||||
)
|
||||
).ShardIterator || ''
|
||||
);
|
||||
}
|
||||
}
|
||||
export default AWSTaskRunner;
|
||||
@@ -0,0 +1,397 @@
|
||||
export class BaseStackFormation {
|
||||
public static readonly baseStackDecription = `Game-CI base stack`;
|
||||
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
|
||||
Description: ${BaseStackFormation.baseStackDecription}
|
||||
Parameters:
|
||||
EnvironmentName:
|
||||
Type: String
|
||||
Default: development
|
||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
||||
Version:
|
||||
Type: String
|
||||
Description: 'hash of template'
|
||||
|
||||
# ContainerPort:
|
||||
# Type: Number
|
||||
# Default: 80
|
||||
# Description: What port number the application inside the docker container is binding to
|
||||
|
||||
Mappings:
|
||||
# Hard values for the subnet masks. These masks define
|
||||
# the range of internal IP addresses that can be assigned.
|
||||
# The VPC can have all IP's from 10.0.0.0 to 10.0.255.255
|
||||
# There are four subnets which cover the ranges:
|
||||
#
|
||||
# 10.0.0.0 - 10.0.0.255
|
||||
# 10.0.1.0 - 10.0.1.255
|
||||
# 10.0.2.0 - 10.0.2.255
|
||||
# 10.0.3.0 - 10.0.3.255
|
||||
|
||||
SubnetConfig:
|
||||
VPC:
|
||||
CIDR: '10.0.0.0/16'
|
||||
PublicOne:
|
||||
CIDR: '10.0.0.0/24'
|
||||
PublicTwo:
|
||||
CIDR: '10.0.1.0/24'
|
||||
|
||||
Resources:
|
||||
# VPC in which containers will be networked.
|
||||
# It has two public subnets, and two private subnets.
|
||||
# We distribute the subnets across the first two available subnets
|
||||
# for the region, for high availability.
|
||||
VPC:
|
||||
Type: AWS::EC2::VPC
|
||||
Properties:
|
||||
EnableDnsSupport: true
|
||||
EnableDnsHostnames: true
|
||||
CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
|
||||
|
||||
MainBucket:
|
||||
Type: "AWS::S3::Bucket"
|
||||
Properties:
|
||||
BucketName: !Ref EnvironmentName
|
||||
|
||||
EFSServerSecurityGroup:
|
||||
Type: AWS::EC2::SecurityGroup
|
||||
Properties:
|
||||
GroupName: 'efs-server-endpoints'
|
||||
GroupDescription: Which client ip addrs are allowed to access EFS server
|
||||
VpcId: !Ref 'VPC'
|
||||
SecurityGroupIngress:
|
||||
- IpProtocol: tcp
|
||||
FromPort: 2049
|
||||
ToPort: 2049
|
||||
SourceSecurityGroupId: !Ref ContainerSecurityGroup
|
||||
#CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
|
||||
# A security group for the containers we will run in Fargate.
|
||||
# Rules are added to this security group based on what ingress you
|
||||
# add for the cluster.
|
||||
ContainerSecurityGroup:
|
||||
Type: AWS::EC2::SecurityGroup
|
||||
Properties:
|
||||
GroupName: 'task security group'
|
||||
GroupDescription: Access to the Fargate containers
|
||||
VpcId: !Ref 'VPC'
|
||||
# SecurityGroupIngress:
|
||||
# - IpProtocol: tcp
|
||||
# FromPort: !Ref ContainerPort
|
||||
# ToPort: !Ref ContainerPort
|
||||
# CidrIp: 0.0.0.0/0
|
||||
SecurityGroupEgress:
|
||||
- IpProtocol: -1
|
||||
FromPort: 2049
|
||||
ToPort: 2049
|
||||
CidrIp: '0.0.0.0/0'
|
||||
|
||||
# Two public subnets, where containers can have public IP addresses
|
||||
PublicSubnetOne:
|
||||
Type: AWS::EC2::Subnet
|
||||
Properties:
|
||||
AvailabilityZone: !Select
|
||||
- 0
|
||||
- Fn::GetAZs: !Ref 'AWS::Region'
|
||||
VpcId: !Ref 'VPC'
|
||||
CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR']
|
||||
# MapPublicIpOnLaunch: true
|
||||
|
||||
PublicSubnetTwo:
|
||||
Type: AWS::EC2::Subnet
|
||||
Properties:
|
||||
AvailabilityZone: !Select
|
||||
- 1
|
||||
- Fn::GetAZs: !Ref 'AWS::Region'
|
||||
VpcId: !Ref 'VPC'
|
||||
CidrBlock: !FindInMap ['SubnetConfig', 'PublicTwo', 'CIDR']
|
||||
# MapPublicIpOnLaunch: true
|
||||
|
||||
# Setup networking resources for the public subnets. Containers
|
||||
# in the public subnets have public IP addresses and the routing table
|
||||
# sends network traffic via the internet gateway.
|
||||
InternetGateway:
|
||||
Type: AWS::EC2::InternetGateway
|
||||
GatewayAttachement:
|
||||
Type: AWS::EC2::VPCGatewayAttachment
|
||||
Properties:
|
||||
VpcId: !Ref 'VPC'
|
||||
InternetGatewayId: !Ref 'InternetGateway'
|
||||
|
||||
# Attaching a Internet Gateway to route table makes it public.
|
||||
PublicRouteTable:
|
||||
Type: AWS::EC2::RouteTable
|
||||
Properties:
|
||||
VpcId: !Ref 'VPC'
|
||||
PublicRoute:
|
||||
Type: AWS::EC2::Route
|
||||
DependsOn: GatewayAttachement
|
||||
Properties:
|
||||
RouteTableId: !Ref 'PublicRouteTable'
|
||||
DestinationCidrBlock: '0.0.0.0/0'
|
||||
GatewayId: !Ref 'InternetGateway'
|
||||
|
||||
# Attaching a public route table makes a subnet public.
|
||||
PublicSubnetOneRouteTableAssociation:
|
||||
Type: AWS::EC2::SubnetRouteTableAssociation
|
||||
Properties:
|
||||
SubnetId: !Ref PublicSubnetOne
|
||||
RouteTableId: !Ref PublicRouteTable
|
||||
PublicSubnetTwoRouteTableAssociation:
|
||||
Type: AWS::EC2::SubnetRouteTableAssociation
|
||||
Properties:
|
||||
SubnetId: !Ref PublicSubnetTwo
|
||||
RouteTableId: !Ref PublicRouteTable
|
||||
|
||||
# ECS Resources
|
||||
ECSCluster:
|
||||
Type: AWS::ECS::Cluster
|
||||
|
||||
# A role used to allow AWS Autoscaling to inspect stats and adjust scaleable targets
|
||||
# on your AWS account
|
||||
AutoscalingRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: [application-autoscaling.amazonaws.com]
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: service-autoscaling
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- 'application-autoscaling:*'
|
||||
- 'cloudwatch:DescribeAlarms'
|
||||
- 'cloudwatch:PutMetricAlarm'
|
||||
- 'ecs:DescribeServices'
|
||||
- 'ecs:UpdateService'
|
||||
Resource: '*'
|
||||
|
||||
# This is an IAM role which authorizes ECS to manage resources on your
|
||||
# account on your behalf, such as updating your load balancer with the
|
||||
# details of where your containers are, so that traffic can reach your
|
||||
# containers.
|
||||
ECSRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: [ecs.amazonaws.com]
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: ecs-service
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
# Rules which allow ECS to attach network interfaces to instances
|
||||
# on your behalf in order for awsvpc networking mode to work right
|
||||
- 'ec2:AttachNetworkInterface'
|
||||
- 'ec2:CreateNetworkInterface'
|
||||
- 'ec2:CreateNetworkInterfacePermission'
|
||||
- 'ec2:DeleteNetworkInterface'
|
||||
- 'ec2:DeleteNetworkInterfacePermission'
|
||||
- 'ec2:Describe*'
|
||||
- 'ec2:DetachNetworkInterface'
|
||||
|
||||
# Rules which allow ECS to update load balancers on your behalf
|
||||
# with the information sabout how to send traffic to your containers
|
||||
- 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
|
||||
- 'elasticloadbalancing:DeregisterTargets'
|
||||
- 'elasticloadbalancing:Describe*'
|
||||
- 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
|
||||
- 'elasticloadbalancing:RegisterTargets'
|
||||
Resource: '*'
|
||||
|
||||
# This is a role which is used by the ECS tasks themselves.
|
||||
ECSTaskExecutionRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: [ecs-tasks.amazonaws.com]
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: AmazonECSTaskExecutionRolePolicy
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
# Allow the use of secret manager
|
||||
- 'secretsmanager:GetSecretValue'
|
||||
- 'kms:Decrypt'
|
||||
|
||||
# Allow the ECS Tasks to download images from ECR
|
||||
- 'ecr:GetAuthorizationToken'
|
||||
- 'ecr:BatchCheckLayerAvailability'
|
||||
- 'ecr:GetDownloadUrlForLayer'
|
||||
- 'ecr:BatchGetImage'
|
||||
|
||||
# Allow the ECS tasks to upload logs to CloudWatch
|
||||
- 'logs:CreateLogStream'
|
||||
- 'logs:PutLogEvents'
|
||||
Resource: '*'
|
||||
|
||||
DeleteCFNLambdaExecutionRole:
|
||||
Type: 'AWS::IAM::Role'
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: 'Allow'
|
||||
Principal:
|
||||
Service: ['lambda.amazonaws.com']
|
||||
Action: 'sts:AssumeRole'
|
||||
Path: '/'
|
||||
Policies:
|
||||
- PolicyName: DeleteCFNLambdaExecutionRole
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: 'Allow'
|
||||
Action:
|
||||
- 'logs:CreateLogGroup'
|
||||
- 'logs:CreateLogStream'
|
||||
- 'logs:PutLogEvents'
|
||||
Resource: 'arn:aws:logs:*:*:*'
|
||||
- Effect: 'Allow'
|
||||
Action:
|
||||
- 'cloudformation:DeleteStack'
|
||||
- 'kinesis:DeleteStream'
|
||||
- 'secretsmanager:DeleteSecret'
|
||||
- 'kinesis:DescribeStreamSummary'
|
||||
- 'logs:DeleteLogGroup'
|
||||
- 'logs:DeleteSubscriptionFilter'
|
||||
- 'ecs:DeregisterTaskDefinition'
|
||||
- 'lambda:DeleteFunction'
|
||||
- 'lambda:InvokeFunction'
|
||||
- 'events:RemoveTargets'
|
||||
- 'events:DeleteRule'
|
||||
- 'lambda:RemovePermission'
|
||||
Resource: '*'
|
||||
|
||||
### cloud watch to kinesis role
|
||||
CloudWatchIAMRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: [logs.amazonaws.com]
|
||||
Action: ['sts:AssumeRole']
|
||||
Path: /
|
||||
Policies:
|
||||
- PolicyName: service-autoscaling
|
||||
PolicyDocument:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- 'kinesis:PutRecord'
|
||||
Resource: '*'
|
||||
|
||||
#####################EFS#####################
|
||||
EfsFileStorage:
|
||||
Type: 'AWS::EFS::FileSystem'
|
||||
Properties:
|
||||
BackupPolicy:
|
||||
Status: ENABLED
|
||||
PerformanceMode: maxIO
|
||||
Encrypted: false
|
||||
|
||||
FileSystemPolicy:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: 'Allow'
|
||||
Action:
|
||||
- 'elasticfilesystem:ClientMount'
|
||||
- 'elasticfilesystem:ClientWrite'
|
||||
- 'elasticfilesystem:ClientRootAccess'
|
||||
Principal:
|
||||
AWS: '*'
|
||||
|
||||
MountTargetResource1:
|
||||
Type: AWS::EFS::MountTarget
|
||||
Properties:
|
||||
FileSystemId: !Ref EfsFileStorage
|
||||
SubnetId: !Ref PublicSubnetOne
|
||||
SecurityGroups:
|
||||
- !Ref EFSServerSecurityGroup
|
||||
|
||||
MountTargetResource2:
|
||||
Type: AWS::EFS::MountTarget
|
||||
Properties:
|
||||
FileSystemId: !Ref EfsFileStorage
|
||||
SubnetId: !Ref PublicSubnetTwo
|
||||
SecurityGroups:
|
||||
- !Ref EFSServerSecurityGroup
|
||||
|
||||
Outputs:
|
||||
EfsFileStorageId:
|
||||
Description: 'The connection endpoint for the database.'
|
||||
Value: !Ref EfsFileStorage
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId
|
||||
ClusterName:
|
||||
Description: The name of the ECS cluster
|
||||
Value: !Ref 'ECSCluster'
|
||||
Export:
|
||||
Name: !Sub${' ${EnvironmentName}'}:ClusterName
|
||||
AutoscalingRole:
|
||||
Description: The ARN of the role used for autoscaling
|
||||
Value: !GetAtt 'AutoscalingRole.Arn'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:AutoscalingRole
|
||||
ECSRole:
|
||||
Description: The ARN of the ECS role
|
||||
Value: !GetAtt 'ECSRole.Arn'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:ECSRole
|
||||
ECSTaskExecutionRole:
|
||||
Description: The ARN of the ECS role tsk execution role
|
||||
Value: !GetAtt 'ECSTaskExecutionRole.Arn'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:ECSTaskExecutionRole
|
||||
|
||||
DeleteCFNLambdaExecutionRole:
|
||||
Description: Lambda execution role for cleaning up cloud formations
|
||||
Value: !GetAtt 'DeleteCFNLambdaExecutionRole.Arn'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:DeleteCFNLambdaExecutionRole
|
||||
|
||||
CloudWatchIAMRole:
|
||||
Description: The ARN of the CloudWatch role for subscription filter
|
||||
Value: !GetAtt 'CloudWatchIAMRole.Arn'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:CloudWatchIAMRole
|
||||
VpcId:
|
||||
Description: The ID of the VPC that this stack is deployed in
|
||||
Value: !Ref 'VPC'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:VpcId
|
||||
PublicSubnetOne:
|
||||
Description: Public subnet one
|
||||
Value: !Ref 'PublicSubnetOne'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetOne
|
||||
PublicSubnetTwo:
|
||||
Description: Public subnet two
|
||||
Value: !Ref 'PublicSubnetTwo'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetTwo
|
||||
ContainerSecurityGroup:
|
||||
Description: A security group used to allow Fargate containers to receive traffic
|
||||
Value: !Ref 'ContainerSecurityGroup'
|
||||
Export:
|
||||
Name: !Sub ${'${EnvironmentName}'}:ContainerSecurityGroup
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
export class CleanupCronFormation {
|
||||
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
|
||||
Description: Schedule automatic deletion of CloudFormation stacks
|
||||
Metadata:
|
||||
AWS::CloudFormation::Interface:
|
||||
ParameterGroups:
|
||||
- Label:
|
||||
default: Input configuration
|
||||
Parameters:
|
||||
- StackName
|
||||
- TTL
|
||||
ParameterLabels:
|
||||
StackName:
|
||||
default: Stack name
|
||||
TTL:
|
||||
default: Time-to-live
|
||||
Parameters:
|
||||
EnvironmentName:
|
||||
Type: String
|
||||
Default: development
|
||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
||||
BUILDGUID:
|
||||
Type: String
|
||||
Default: ''
|
||||
StackName:
|
||||
Type: String
|
||||
Description: Stack name that will be deleted.
|
||||
DeleteStackName:
|
||||
Type: String
|
||||
Description: Stack name that will be deleted.
|
||||
TTL:
|
||||
Type: Number
|
||||
Description: Time-to-live in minutes for the stack.
|
||||
Resources:
|
||||
DeleteCFNLambda:
|
||||
Type: "AWS::Lambda::Function"
|
||||
Properties:
|
||||
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
|
||||
Code:
|
||||
ZipFile: |
|
||||
import boto3
|
||||
import os
|
||||
import json
|
||||
|
||||
stack_name = os.environ['stackName']
|
||||
delete_stack_name = os.environ['deleteStackName']
|
||||
|
||||
def delete_cfn(stack_name):
|
||||
try:
|
||||
cfn = boto3.resource('cloudformation')
|
||||
stack = cfn.Stack(stack_name)
|
||||
stack.delete()
|
||||
return "SUCCESS"
|
||||
except:
|
||||
return "ERROR"
|
||||
|
||||
def handler(event, context):
|
||||
print("Received event:")
|
||||
print(json.dumps(event))
|
||||
result = delete_cfn(stack_name)
|
||||
delete_cfn(delete_stack_name)
|
||||
return result
|
||||
Environment:
|
||||
Variables:
|
||||
stackName: !Ref 'StackName'
|
||||
deleteStackName: !Ref 'DeleteStackName'
|
||||
Handler: "index.handler"
|
||||
Runtime: "python3.9"
|
||||
Timeout: "5"
|
||||
Role:
|
||||
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
|
||||
DeleteStackEventRule:
|
||||
DependsOn:
|
||||
- DeleteCFNLambda
|
||||
- GenerateCronExpression
|
||||
Type: "AWS::Events::Rule"
|
||||
Properties:
|
||||
Name: !Join [ "", [ 'DeleteStackEventRule', !Ref BUILDGUID ] ]
|
||||
Description: Delete stack event
|
||||
ScheduleExpression: !GetAtt GenerateCronExpression.cron_exp
|
||||
State: "ENABLED"
|
||||
Targets:
|
||||
-
|
||||
Arn: !GetAtt DeleteCFNLambda.Arn
|
||||
Id: 'DeleteCFNLambda'
|
||||
PermissionForDeleteCFNLambda:
|
||||
Type: "AWS::Lambda::Permission"
|
||||
DependsOn:
|
||||
- DeleteStackEventRule
|
||||
Properties:
|
||||
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
|
||||
Action: "lambda:InvokeFunction"
|
||||
Principal: "events.amazonaws.com"
|
||||
SourceArn: !GetAtt DeleteStackEventRule.Arn
|
||||
GenerateCronExpLambda:
|
||||
Type: "AWS::Lambda::Function"
|
||||
Properties:
|
||||
FunctionName: !Join [ "", [ 'GenerateCronExpressionLambda', !Ref BUILDGUID ] ]
|
||||
Code:
|
||||
ZipFile: |
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
import cfnresponse
|
||||
|
||||
def deletion_time(ttl):
|
||||
delete_at_time = datetime.now() + timedelta(minutes=int(ttl))
|
||||
hh = delete_at_time.hour
|
||||
mm = delete_at_time.minute
|
||||
yyyy = delete_at_time.year
|
||||
month = delete_at_time.month
|
||||
dd = delete_at_time.day
|
||||
# minutes hours day month day-of-week year
|
||||
cron_exp = "cron({} {} {} {} ? {})".format(mm, hh, dd, month, yyyy)
|
||||
return cron_exp
|
||||
|
||||
def handler(event, context):
|
||||
print('Received event: %s' % json.dumps(event))
|
||||
status = cfnresponse.SUCCESS
|
||||
try:
|
||||
if event['RequestType'] == 'Delete':
|
||||
cfnresponse.send(event, context, status, {})
|
||||
else:
|
||||
ttl = event['ResourceProperties']['ttl']
|
||||
responseData = {}
|
||||
responseData['cron_exp'] = deletion_time(ttl)
|
||||
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
|
||||
except Exception as e:
|
||||
logging.error('Exception: %s' % e, exc_info=True)
|
||||
status = cfnresponse.FAILED
|
||||
cfnresponse.send(event, context, status, {}, None)
|
||||
Handler: "index.handler"
|
||||
Runtime: "python3.9"
|
||||
Timeout: "5"
|
||||
Role:
|
||||
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
|
||||
GenerateCronExpression:
|
||||
Type: "Custom::GenerateCronExpression"
|
||||
Version: "1.0"
|
||||
Properties:
|
||||
Name: !Join [ "", [ 'GenerateCronExpression', !Ref BUILDGUID ] ]
|
||||
ServiceToken: !GetAtt GenerateCronExpLambda.Arn
|
||||
ttl: !Ref 'TTL'
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import Orchestrator from '../../../orchestrator';
|
||||
|
||||
export class TaskDefinitionFormation {
|
||||
public static readonly description: string = `Game CI Orchestrator Task Stack`;
|
||||
public static get formation(): string {
|
||||
return `AWSTemplateFormatVersion: 2010-09-09
|
||||
Description: ${TaskDefinitionFormation.description}
|
||||
Parameters:
|
||||
EnvironmentName:
|
||||
Type: String
|
||||
Default: development
|
||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
||||
ServiceName:
|
||||
Type: String
|
||||
Default: example
|
||||
Description: A name for the service
|
||||
LogGroupName:
|
||||
Type: String
|
||||
Default: example
|
||||
Description: Name to use for the log group created for this task
|
||||
ImageUrl:
|
||||
Type: String
|
||||
Default: nginx
|
||||
Description: >-
|
||||
The url of a docker image that contains the application process that will
|
||||
handle the traffic for this service
|
||||
ContainerPort:
|
||||
Type: Number
|
||||
Default: 80
|
||||
Description: What port number the application inside the docker container is binding to
|
||||
ContainerCpu:
|
||||
Default: ${Orchestrator.buildParameters.containerCpu}
|
||||
Type: Number
|
||||
Description: How much CPU to give the container. 1024 is 1 CPU
|
||||
ContainerMemory:
|
||||
Default: ${Orchestrator.buildParameters.containerMemory}
|
||||
Type: Number
|
||||
Description: How much memory in megabytes to give the container
|
||||
BUILDGUID:
|
||||
Type: String
|
||||
Default: ''
|
||||
Command:
|
||||
Type: String
|
||||
Default: 'ls'
|
||||
EntryPoint:
|
||||
Type: String
|
||||
Default: '/bin/sh'
|
||||
WorkingDirectory:
|
||||
Type: String
|
||||
Default: '/efsdata/'
|
||||
Role:
|
||||
Type: String
|
||||
Default: ''
|
||||
Description: >-
|
||||
(Optional) An IAM role to give the service's containers if the code within
|
||||
needs to access other AWS resources like S3 buckets, DynamoDB tables, etc
|
||||
EFSMountDirectory:
|
||||
Type: String
|
||||
Default: '/efsdata'
|
||||
# template secrets p1 - input
|
||||
Mappings:
|
||||
SubnetConfig:
|
||||
VPC:
|
||||
CIDR: 10.0.0.0/16
|
||||
PublicOne:
|
||||
CIDR: 10.0.0.0/24
|
||||
PublicTwo:
|
||||
CIDR: 10.0.1.0/24
|
||||
Conditions:
|
||||
HasCustomRole: !Not
|
||||
- !Equals
|
||||
- Ref: Role
|
||||
- ''
|
||||
Resources:
|
||||
LogGroup:
|
||||
Type: 'AWS::Logs::LogGroup'
|
||||
Properties:
|
||||
LogGroupName: !Ref LogGroupName
|
||||
Metadata:
|
||||
'AWS::CloudFormation::Designer':
|
||||
id: aece53ae-b82d-4267-bc16-ed964b05db27
|
||||
# template resources secrets
|
||||
|
||||
# template resources logstream
|
||||
|
||||
TaskDefinition:
|
||||
Type: 'AWS::ECS::TaskDefinition'
|
||||
Properties:
|
||||
Family: !Ref ServiceName
|
||||
Cpu: !Ref ContainerCpu
|
||||
Memory: !Ref ContainerMemory
|
||||
NetworkMode: awsvpc
|
||||
Volumes:
|
||||
- Name: efs-data
|
||||
EFSVolumeConfiguration:
|
||||
FilesystemId:
|
||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId'
|
||||
TransitEncryption: DISABLED
|
||||
RequiresCompatibilities:
|
||||
- FARGATE
|
||||
ExecutionRoleArn:
|
||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:ECSTaskExecutionRole'
|
||||
TaskRoleArn:
|
||||
'Fn::If':
|
||||
- HasCustomRole
|
||||
- !Ref Role
|
||||
- !Ref 'AWS::NoValue'
|
||||
ContainerDefinitions:
|
||||
- Name: !Ref ServiceName
|
||||
Cpu: !Ref ContainerCpu
|
||||
Memory: !Ref ContainerMemory
|
||||
Image: !Ref ImageUrl
|
||||
EntryPoint:
|
||||
Fn::Split:
|
||||
- ','
|
||||
- !Ref EntryPoint
|
||||
Command:
|
||||
Fn::Split:
|
||||
- ','
|
||||
- !Ref Command
|
||||
WorkingDirectory: !Ref WorkingDirectory
|
||||
Environment:
|
||||
- Name: ALLOW_EMPTY_PASSWORD
|
||||
Value: 'yes'
|
||||
# template - env vars
|
||||
MountPoints:
|
||||
- SourceVolume: efs-data
|
||||
ContainerPath: !Ref EFSMountDirectory
|
||||
ReadOnly: false
|
||||
# template secrets p3 - container def
|
||||
LogConfiguration:
|
||||
LogDriver: awslogs
|
||||
Options:
|
||||
awslogs-group: !Ref LogGroupName
|
||||
awslogs-region: !Ref 'AWS::Region'
|
||||
awslogs-stream-prefix: !Ref ServiceName
|
||||
DependsOn:
|
||||
- LogGroup
|
||||
`;
|
||||
}
|
||||
public static streamLogs = `
|
||||
SubscriptionFilter:
|
||||
Type: 'AWS::Logs::SubscriptionFilter'
|
||||
Properties:
|
||||
FilterPattern: ''
|
||||
RoleArn:
|
||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
|
||||
LogGroupName: !Ref LogGroupName
|
||||
DestinationArn:
|
||||
'Fn::GetAtt':
|
||||
- KinesisStream
|
||||
- Arn
|
||||
Metadata:
|
||||
'AWS::CloudFormation::Designer':
|
||||
id: 7f809e91-9e5d-4678-98c1-c5085956c480
|
||||
DependsOn:
|
||||
- LogGroup
|
||||
- KinesisStream
|
||||
KinesisStream:
|
||||
Type: 'AWS::Kinesis::Stream'
|
||||
Properties:
|
||||
Name: !Ref ServiceName
|
||||
ShardCount: 1
|
||||
Metadata:
|
||||
'AWS::CloudFormation::Designer':
|
||||
id: c6f18447-b879-4696-8873-f981b2cedd2b
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import AwsTaskRunner from './aws-task-runner';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
||||
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
||||
import { Input } from '../../..';
|
||||
import { GarbageCollectionService } from './services/garbage-collection-service';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { TaskService } from './services/task-service';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { AwsClientFactory } from './aws-client-factory';
|
||||
import ResourceTracking from '../../services/core/resource-tracking';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
class AWSBuildEnvironment implements ProviderInterface {
|
||||
private baseStackName: string;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.baseStackName = buildParameters.awsStackName;
|
||||
}
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
await TaskService.getCloudFormationJobStacks();
|
||||
await TaskService.getLogGroups();
|
||||
await TaskService.getTasks();
|
||||
|
||||
return [];
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
async watchWorkflow(): Promise<string> {
|
||||
return await TaskService.watch();
|
||||
}
|
||||
|
||||
async listOtherResources(): Promise<string> {
|
||||
await TaskService.getLogGroups();
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
filter: string,
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
await GarbageCollectionService.cleanup(!previewOnly);
|
||||
|
||||
return ``;
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
async setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
ResourceTracking.logAllocationSummary('aws workflow');
|
||||
await ResourceTracking.logDiskUsageSnapshot('aws workflow (host)');
|
||||
AwsClientFactory.getECS();
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
AwsClientFactory.getKinesis();
|
||||
OrchestratorLogger.log(`AWS Region: ${CF.config.region}`);
|
||||
const entrypoint = ['/bin/sh'];
|
||||
const startTimeMs = Date.now();
|
||||
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
||||
CF,
|
||||
buildGuid,
|
||||
image,
|
||||
entrypoint,
|
||||
commands,
|
||||
mountdir,
|
||||
workingdir,
|
||||
secrets,
|
||||
);
|
||||
|
||||
let postRunTaskTimeMs;
|
||||
try {
|
||||
const postSetupStacksTimeMs = Date.now();
|
||||
OrchestratorLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
||||
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
||||
postRunTaskTimeMs = Date.now();
|
||||
OrchestratorLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
||||
if (shouldCleanup) {
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
const postCleanupTimeMs = Date.now();
|
||||
if (postRunTaskTimeMs !== undefined)
|
||||
OrchestratorLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
OrchestratorLogger.log(`error running task ${error}`);
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupResources(CF: CloudFormation, taskDef: OrchestratorAWSTaskDef) {
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
OrchestratorLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`);
|
||||
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
|
||||
if (OrchestratorOptions.useCleanupCron) {
|
||||
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
|
||||
}
|
||||
|
||||
await waitUntilStackDeleteComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{
|
||||
StackName: taskDef.taskDefStackName,
|
||||
},
|
||||
);
|
||||
await waitUntilStackDeleteComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{
|
||||
StackName: `${taskDef.taskDefStackName}-cleanup`,
|
||||
},
|
||||
);
|
||||
OrchestratorLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
||||
OrchestratorLogger.log('Cleanup complete');
|
||||
}
|
||||
}
|
||||
export default AWSBuildEnvironment;
|
||||
@@ -0,0 +1,10 @@
|
||||
// eslint-disable-next-line import/named
|
||||
import { StackResource } from '@aws-sdk/client-cloudformation';
|
||||
|
||||
class OrchestratorAWSTaskDef {
|
||||
public taskDefStackName!: string;
|
||||
public taskDefCloudFormation!: string;
|
||||
public taskDefResources: StackResource[] | undefined;
|
||||
public baseResources: StackResource[] | undefined;
|
||||
}
|
||||
export default OrchestratorAWSTaskDef;
|
||||
@@ -0,0 +1,75 @@
|
||||
import { DeleteStackCommand, DescribeStackResourcesCommand } from '@aws-sdk/client-cloudformation';
|
||||
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { StopTaskCommand } from '@aws-sdk/client-ecs';
|
||||
import Input from '../../../../input';
|
||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
||||
import { TaskService } from './task-service';
|
||||
import { AwsClientFactory } from '../aws-client-factory';
|
||||
|
||||
export class GarbageCollectionService {
|
||||
static isOlderThan1day(date: Date) {
|
||||
const ageDate = new Date(date.getTime() - Date.now());
|
||||
|
||||
return ageDate.getDay() > 0;
|
||||
}
|
||||
|
||||
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
const ecs = AwsClientFactory.getECS();
|
||||
const cwl = AwsClientFactory.getCloudWatchLogs();
|
||||
const taskDefinitionsInUse = new Array();
|
||||
const tasks = await TaskService.getTasks();
|
||||
|
||||
for (const task of tasks) {
|
||||
const { taskElement, element } = task;
|
||||
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
|
||||
OrchestratorLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
|
||||
}
|
||||
}
|
||||
|
||||
const jobStacks = await TaskService.getCloudFormationJobStacks();
|
||||
for (const element of jobStacks) {
|
||||
if (
|
||||
(await CF.send(new DescribeStackResourcesCommand({ StackName: element.StackName }))).StackResources?.some(
|
||||
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
||||
)
|
||||
) {
|
||||
OrchestratorLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
deleteResources &&
|
||||
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
|
||||
) {
|
||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
||||
OrchestratorLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Deleting ${element.StackName}`);
|
||||
await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
|
||||
}
|
||||
}
|
||||
const logGroups = await TaskService.getLogGroups();
|
||||
for (const element of logGroups) {
|
||||
if (
|
||||
deleteResources &&
|
||||
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
|
||||
) {
|
||||
OrchestratorLogger.log(`Deleting ${element.logGroupName}`);
|
||||
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
|
||||
}
|
||||
}
|
||||
|
||||
const locks = await TaskService.getLocks();
|
||||
for (const element of locks) {
|
||||
OrchestratorLogger.log(`Lock: ${element.Key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import {
|
||||
DescribeStackResourcesCommand,
|
||||
DescribeStacksCommand,
|
||||
ListStacksCommand,
|
||||
} from '@aws-sdk/client-cloudformation';
|
||||
import type { StackSummary } from '@aws-sdk/client-cloudformation';
|
||||
// eslint-disable-next-line import/named
|
||||
import { DescribeLogGroupsCommand, DescribeLogGroupsCommandInput } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import type { LogGroup } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
||||
import type { Task } from '@aws-sdk/client-ecs';
|
||||
import { ListObjectsV2Command } from '@aws-sdk/client-s3';
|
||||
import Input from '../../../../input';
|
||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
||||
import AwsTaskRunner from '../aws-task-runner';
|
||||
import Orchestrator from '../../../orchestrator';
|
||||
import { AwsClientFactory } from '../aws-client-factory';
|
||||
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
||||
|
||||
export class TaskService {
|
||||
static async watch() {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
const { output, shouldCleanup } = await AwsTaskRunner.streamLogsUntilTaskStops(
|
||||
process.env.cluster || ``,
|
||||
process.env.taskArn || ``,
|
||||
process.env.streamName || ``,
|
||||
);
|
||||
|
||||
return output;
|
||||
}
|
||||
public static async getCloudFormationJobStacks(): Promise<StackSummary[]> {
|
||||
const result: StackSummary[] = [];
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`List Cloud Formation Stacks`);
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
const stacks =
|
||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
||||
for (const element of stacks) {
|
||||
if (!element.CreationTime) {
|
||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
|
||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
result.push(element);
|
||||
}
|
||||
const baseStacks =
|
||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||
for (const element of baseStacks) {
|
||||
if (!element.CreationTime) {
|
||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
|
||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
result.push(element);
|
||||
}
|
||||
OrchestratorLogger.log(``);
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> {
|
||||
const result: { taskElement: Task; element: string }[] = [];
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`List Tasks`);
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = AwsClientFactory.getECS();
|
||||
const clusters: string[] = [];
|
||||
{
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const clusterResponse = await ecs.send(new ListClustersCommand({ nextToken }));
|
||||
clusters.push(...(clusterResponse.clusterArns ?? []));
|
||||
nextToken = clusterResponse.nextToken;
|
||||
} while (nextToken);
|
||||
}
|
||||
OrchestratorLogger.log(`Task Clusters ${clusters.length}`);
|
||||
for (const element of clusters) {
|
||||
const taskArns: string[] = [];
|
||||
{
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const taskResponse = await ecs.send(new ListTasksCommand({ cluster: element, nextToken }));
|
||||
taskArns.push(...(taskResponse.taskArns ?? []));
|
||||
nextToken = taskResponse.nextToken;
|
||||
} while (nextToken);
|
||||
}
|
||||
if (taskArns.length > 0) {
|
||||
const describeInput = { tasks: taskArns, cluster: element };
|
||||
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
|
||||
if (describeList.length === 0) {
|
||||
OrchestratorLogger.log(`No Tasks`);
|
||||
continue;
|
||||
}
|
||||
OrchestratorLogger.log(`Tasks ${describeList.length}`);
|
||||
for (const taskElement of describeList) {
|
||||
if (taskElement === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (taskElement.createdAt === undefined) {
|
||||
OrchestratorLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
result.push({ taskElement, element });
|
||||
}
|
||||
}
|
||||
}
|
||||
OrchestratorLogger.log(``);
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async awsDescribeJob(job: string) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
try {
|
||||
const stack =
|
||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
|
||||
const stackInfo = (await CF.send(new DescribeStackResourcesCommand({ StackName: job }))) || undefined;
|
||||
const stackInfo2 = (await CF.send(new DescribeStacksCommand({ StackName: job }))) || undefined;
|
||||
if (stack === undefined) {
|
||||
throw new Error('stack not defined');
|
||||
}
|
||||
if (!stack.CreationTime) {
|
||||
OrchestratorLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
|
||||
const message = `
|
||||
Task Stack ${stack.StackName}
|
||||
Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}
|
||||
${JSON.stringify(stack, undefined, 4)}
|
||||
${JSON.stringify(stackInfo, undefined, 4)}
|
||||
${JSON.stringify(stackInfo2, undefined, 4)}
|
||||
`;
|
||||
OrchestratorLogger.log(message);
|
||||
|
||||
return message;
|
||||
} catch (error) {
|
||||
OrchestratorLogger.error(
|
||||
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
public static async getLogGroups(): Promise<LogGroup[]> {
|
||||
const result: LogGroup[] = [];
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const cwl = AwsClientFactory.getCloudWatchLogs();
|
||||
let logStreamInput: DescribeLogGroupsCommandInput = {
|
||||
/* logGroupNamePrefix: 'game-ci' */
|
||||
};
|
||||
let logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
const logGroups = logGroupsDescribe.logGroups || [];
|
||||
while (logGroupsDescribe.nextToken) {
|
||||
logStreamInput = {
|
||||
/* logGroupNamePrefix: 'game-ci',*/
|
||||
nextToken: logGroupsDescribe.nextToken,
|
||||
};
|
||||
logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Log Groups ${logGroups.length}`);
|
||||
for (const element of logGroups) {
|
||||
if (element.creationTime === undefined) {
|
||||
OrchestratorLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
||||
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
result.push(element);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async getLocks(): Promise<Array<{ Key: string }>> {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
if (Orchestrator.buildParameters.storageProvider === 'rclone') {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
type ListObjectsFunction = (prefix: string) => Promise<string[]>;
|
||||
const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects('');
|
||||
|
||||
return objects.map((x: string) => ({ Key: x }));
|
||||
}
|
||||
const s3 = AwsClientFactory.getS3();
|
||||
const listRequest = {
|
||||
Bucket: Orchestrator.buildParameters.awsStackName,
|
||||
};
|
||||
|
||||
const results = await s3.send(new ListObjectsV2Command(listRequest));
|
||||
|
||||
return (results.Contents || []).map((object) => ({ Key: object.Key || '' }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import Docker from '../../../docker';
|
||||
import { Action } from '../../..';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import * as fs from 'node:fs';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import { StringKeyValuePair } from '../../../shared-types';
|
||||
|
||||
class LocalDockerOrchestrator implements ProviderInterface {
|
||||
public buildParameters!: BuildParameters;
|
||||
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
return new Promise((resolve) => resolve([]));
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
async cleanupWorkflow(
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
const { workspace } = Action;
|
||||
if (
|
||||
fs.existsSync(
|
||||
`${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
)
|
||||
) {
|
||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache/cache/build/`);
|
||||
await OrchestratorSystem.Run(
|
||||
`rm -r ${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
this.buildParameters = buildParameters;
|
||||
}
|
||||
|
||||
public async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(buildGuid);
|
||||
OrchestratorLogger.log(commands);
|
||||
|
||||
const { workspace, actionFolder } = Action;
|
||||
const content: StringKeyValuePair[] = [];
|
||||
for (const x of secrets) {
|
||||
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
||||
}
|
||||
|
||||
// Replace localhost with host.docker.internal for LocalStack endpoints (similar to K8s)
|
||||
// This allows Docker containers to access LocalStack running on the host
|
||||
const endpointEnvironmentNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
for (const x of environment) {
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvironmentNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
// Replace localhost with host.docker.internal so containers can access host services
|
||||
value = value
|
||||
.replace('http://localhost', 'http://host.docker.internal')
|
||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
||||
OrchestratorLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
||||
}
|
||||
content.push({ name: x.name, value });
|
||||
}
|
||||
|
||||
// if (this.buildParameters?.orchestratorIntegrationTests) {
|
||||
// core.info(JSON.stringify(content, undefined, 4));
|
||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||
// core.info(JSON.stringify(environment, undefined, 4));
|
||||
// }
|
||||
|
||||
// eslint-disable-next-line unicorn/no-for-loop
|
||||
for (let index = 0; index < content.length; index++) {
|
||||
if (content[index] === undefined) {
|
||||
delete content[index];
|
||||
}
|
||||
}
|
||||
let myOutput = '';
|
||||
const sharedFolder = `/data/`;
|
||||
|
||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||
const entrypointFilePath = `start.sh`;
|
||||
|
||||
// Use #!/bin/sh for POSIX compatibility (Alpine-based images like rclone/rclone don't have bash)
|
||||
const fileContents = `#!/bin/sh
|
||||
set -e
|
||||
|
||||
mkdir -p /github/workspace/orchestrator-cache
|
||||
mkdir -p /data/cache
|
||||
cp -a /github/workspace/orchestrator-cache/. ${sharedFolder}
|
||||
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
|
||||
# Only copy cache directory, exclude retained workspaces to avoid running out of disk space
|
||||
if [ -d "${sharedFolder}cache" ]; then
|
||||
cp -a ${sharedFolder}cache/. /github/workspace/orchestrator-cache/cache/ || true
|
||||
fi
|
||||
# Copy test files from /data/ root to workspace for test assertions
|
||||
# This allows tests to write files to /data/ and have them available in the workspace
|
||||
find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/orchestrator-cache/ \\; || true
|
||||
`;
|
||||
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
||||
flag: 'w',
|
||||
});
|
||||
|
||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
||||
OrchestratorLogger.log(`Running local-docker: \n ${fileContents}`);
|
||||
}
|
||||
|
||||
if (fs.existsSync(`${workspace}/orchestrator-cache`)) {
|
||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache && du -sh ${workspace}/orchestrator-cache`);
|
||||
}
|
||||
const exitCode = await Docker.run(
|
||||
image,
|
||||
{ workspace, actionFolder, ...this.buildParameters },
|
||||
false,
|
||||
`chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`,
|
||||
content,
|
||||
{
|
||||
listeners: {
|
||||
stdout: (data: Buffer) => {
|
||||
myOutput += data.toString();
|
||||
},
|
||||
stderr: (data: Buffer) => {
|
||||
myOutput += `[LOCAL-DOCKER-ERROR]${data.toString()}`;
|
||||
},
|
||||
},
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
// Docker doesn't exit on fail now so adding this to ensure behavior is unchanged
|
||||
// TODO: Is there a helpful way to consume the exit code or is it best to except
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`Build failed with exit code ${exitCode}`);
|
||||
}
|
||||
|
||||
return myOutput;
|
||||
}
|
||||
}
|
||||
export default LocalDockerOrchestrator;
|
||||
@@ -0,0 +1,460 @@
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import { BuildParameters } from '../../..';
|
||||
import * as core from '@actions/core';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import KubernetesStorage from './kubernetes-storage';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
||||
import KubernetesSecret from './kubernetes-secret';
|
||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
import { KubernetesRole } from './kubernetes-role';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import ResourceTracking from '../../services/core/resource-tracking';
|
||||
|
||||
class Kubernetes implements ProviderInterface {
|
||||
public static Instance: Kubernetes;
|
||||
public kubeConfig!: k8s.KubeConfig;
|
||||
public kubeClient!: k8s.CoreV1Api;
|
||||
public kubeClientApps!: k8s.AppsV1Api;
|
||||
public kubeClientBatch!: k8s.BatchV1Api;
|
||||
public rbacAuthorizationV1Api!: k8s.RbacAuthorizationV1Api;
|
||||
public buildGuid: string = '';
|
||||
public buildParameters!: BuildParameters;
|
||||
public pvcName: string = '';
|
||||
public secretName: string = '';
|
||||
public jobName: string = '';
|
||||
public namespace!: string;
|
||||
public podName: string = '';
|
||||
public containerName: string = '';
|
||||
public cleanupCronJobName: string = '';
|
||||
public serviceAccountName: string = '';
|
||||
public ip: string = '';
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
Kubernetes.Instance = this;
|
||||
this.kubeConfig = new k8s.KubeConfig();
|
||||
this.kubeConfig.loadFromDefault();
|
||||
this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api);
|
||||
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
|
||||
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
||||
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
|
||||
this.namespace = buildParameters.containerNamespace ? buildParameters.containerNamespace : 'default';
|
||||
OrchestratorLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||
}
|
||||
|
||||
async PushLogUpdate(logs: string) {
|
||||
// push logs to nginx file server via 'LOG_SERVICE_IP' env var
|
||||
const ip = process.env[`LOG_SERVICE_IP`];
|
||||
if (ip === undefined) {
|
||||
RemoteClientLogger.logWarning(`LOG_SERVICE_IP not set, skipping log push`);
|
||||
|
||||
return;
|
||||
}
|
||||
const url = `http://${ip}/api/log`;
|
||||
RemoteClientLogger.log(`Pushing logs to ${url}`);
|
||||
|
||||
// logs to base64
|
||||
logs = Buffer.from(logs).toString('base64');
|
||||
const response = await OrchestratorSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
|
||||
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
const pods = await this.kubeClient.listNamespacedPod(this.namespace);
|
||||
const serviceAccounts = await this.kubeClient.listNamespacedServiceAccount(this.namespace);
|
||||
const secrets = await this.kubeClient.listNamespacedSecret(this.namespace);
|
||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
||||
|
||||
return [
|
||||
...pods.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...serviceAccounts.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...secrets.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...jobs.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
];
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
public async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
try {
|
||||
this.buildParameters = buildParameters;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
|
||||
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
|
||||
|
||||
await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
try {
|
||||
OrchestratorLogger.log('Orchestrator K8s workflow!');
|
||||
ResourceTracking.logAllocationSummary('k8s workflow');
|
||||
await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)');
|
||||
await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)');
|
||||
|
||||
// Setup
|
||||
const id =
|
||||
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
|
||||
? Orchestrator.lockedWorkspace
|
||||
: this.buildParameters.buildGuid;
|
||||
this.pvcName = `unity-builder-pvc-${id}`;
|
||||
await KubernetesStorage.createPersistentVolumeClaim(
|
||||
this.buildParameters,
|
||||
this.pvcName,
|
||||
this.kubeClient,
|
||||
this.namespace,
|
||||
);
|
||||
this.buildGuid = buildGuid;
|
||||
this.secretName = `build-credentials-${this.buildGuid}`;
|
||||
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
||||
this.containerName = `main`;
|
||||
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
||||
|
||||
// For tests, clean up old images before creating job to free space for image pull
|
||||
// IMPORTANT: Preserve the Unity image to avoid re-pulling it
|
||||
if (process.env['orchestratorTests'] === 'true') {
|
||||
try {
|
||||
OrchestratorLogger.log('Cleaning up old images in k3d node before pulling new image...');
|
||||
const { OrchestratorSystem: OrchestratorSystemModule } = await import(
|
||||
'../../services/core/orchestrator-system'
|
||||
);
|
||||
|
||||
// Aggressive cleanup: remove stopped containers and non-Unity images
|
||||
// IMPORTANT: Preserve Unity images (unityci/editor) to avoid re-pulling the 3.9GB image
|
||||
const K3D_NODE_CONTAINERS = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
|
||||
const cleanupCommands: string[] = [];
|
||||
|
||||
for (const NODE of K3D_NODE_CONTAINERS) {
|
||||
// Remove all stopped containers (this frees runtime space but keeps images)
|
||||
cleanupCommands.push(
|
||||
`docker exec ${NODE} sh -c "crictl rm --all 2>/dev/null || true" || true`,
|
||||
`docker exec ${NODE} sh -c "for img in $(crictl images -q 2>/dev/null); do repo=$(crictl inspecti $img --format '{{.repo}}' 2>/dev/null || echo ''); if echo "$repo" | grep -qvE 'unityci/editor|unity'; then crictl rmi $img 2>/dev/null || true; fi; done" || true`,
|
||||
`docker exec ${NODE} sh -c "crictl rmi --prune 2>/dev/null || true" || true`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const cmd of cleanupCommands) {
|
||||
try {
|
||||
await OrchestratorSystemModule.Run(cmd, true, true);
|
||||
} catch (cmdError) {
|
||||
// Ignore individual command failures - cleanup is best effort
|
||||
OrchestratorLogger.log(`Cleanup command failed (non-fatal): ${cmdError}`);
|
||||
}
|
||||
}
|
||||
OrchestratorLogger.log('Cleanup completed (containers and non-Unity images removed, Unity images preserved)');
|
||||
} catch (cleanupError) {
|
||||
OrchestratorLogger.logWarning(`Failed to cleanup images before job creation: ${cleanupError}`);
|
||||
|
||||
// Continue anyway - image might already be cached
|
||||
}
|
||||
}
|
||||
|
||||
let output = '';
|
||||
try {
|
||||
// Before creating the job, verify we have the Unity image cached on the agent node
|
||||
// If not cached, try to ensure it's available to avoid disk pressure during pull
|
||||
if (process.env['orchestratorTests'] === 'true' && image.includes('unityci/editor')) {
|
||||
try {
|
||||
const { OrchestratorSystem: OrchestratorSystemModule2 } = await import(
|
||||
'../../services/core/orchestrator-system'
|
||||
);
|
||||
|
||||
// Check if image is cached on agent node (where pods run)
|
||||
const agentImageCheck = await OrchestratorSystemModule2.Run(
|
||||
`docker exec k3d-unity-builder-agent-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
|
||||
if (agentImageCheck.includes('not_cached')) {
|
||||
// Check if image is on server node
|
||||
const serverImageCheck = await OrchestratorSystemModule2.Run(
|
||||
`docker exec k3d-unity-builder-server-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
|
||||
// Check available disk space on agent node
|
||||
const diskInfo = await OrchestratorSystemModule2.Run(
|
||||
'docker exec k3d-unity-builder-agent-0 sh -c "df -h /var/lib/rancher/k3s 2>/dev/null | tail -1 || df -h / 2>/dev/null | tail -1 || echo unknown" || echo unknown',
|
||||
true,
|
||||
true,
|
||||
);
|
||||
|
||||
OrchestratorLogger.logWarning(
|
||||
`Unity image not cached on agent node (where pods run). Server node: ${
|
||||
serverImageCheck.includes('cached') ? 'has image' : 'no image'
|
||||
}. Disk info: ${diskInfo.trim()}. Pod will attempt to pull image (3.9GB) which may fail due to disk pressure.`,
|
||||
);
|
||||
|
||||
// If image is on server but not agent, log a warning
|
||||
// NOTE: We don't attempt to pull here because:
|
||||
// 1. Pulling a 3.9GB image can take several minutes and block the test
|
||||
// 2. If there's not enough disk space, the pull will hang indefinitely
|
||||
// 3. The pod will attempt to pull during scheduling anyway
|
||||
// 4. If the pull fails, Kubernetes will provide proper error messages
|
||||
if (serverImageCheck.includes('cached')) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'Unity image exists on server node but not agent node. Pod will attempt to pull during scheduling. If pull fails due to disk pressure, ensure cleanup runs before this test.',
|
||||
);
|
||||
} else {
|
||||
// Image not on either node - check if we have enough space to pull
|
||||
// Extract available space from disk info
|
||||
const availableSpaceMatch = diskInfo.match(/(\d+(?:\.\d+)?)\s*([gkm]?i?b)/i);
|
||||
if (availableSpaceMatch) {
|
||||
const availableValue = Number.parseFloat(availableSpaceMatch[1]);
|
||||
const availableUnit = availableSpaceMatch[2].toUpperCase();
|
||||
let availableGB = availableValue;
|
||||
|
||||
if (availableUnit.includes('M')) {
|
||||
availableGB = availableValue / 1024;
|
||||
} else if (availableUnit.includes('K')) {
|
||||
availableGB = availableValue / (1024 * 1024);
|
||||
}
|
||||
|
||||
// Unity image is ~3.9GB, need at least 4.5GB to be safe
|
||||
if (availableGB < 4.5) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`CRITICAL: Unity image not cached and only ${availableGB.toFixed(
|
||||
2,
|
||||
)}GB available. Image pull (3.9GB) will likely fail. Consider running cleanup or ensuring pre-pull step succeeds.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
OrchestratorLogger.log('Unity image is cached on agent node - pod should start without pulling');
|
||||
}
|
||||
} catch (checkError) {
|
||||
// Ignore check errors - continue with job creation
|
||||
OrchestratorLogger.logWarning(`Failed to verify Unity image cache: ${checkError}`);
|
||||
}
|
||||
}
|
||||
|
||||
OrchestratorLogger.log('Job does not exist');
|
||||
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||
OrchestratorLogger.log('Watching pod until running');
|
||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||
|
||||
OrchestratorLogger.log('Pod is running');
|
||||
output += await KubernetesTaskRunner.runTask(
|
||||
this.kubeConfig,
|
||||
this.kubeClient,
|
||||
this.jobName,
|
||||
this.podName,
|
||||
this.containerName,
|
||||
this.namespace,
|
||||
);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`error running k8s workflow ${error}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
OrchestratorLogger.log(
|
||||
JSON.stringify(
|
||||
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
|
||||
.map((x) => {
|
||||
return {
|
||||
message: x.message || ``,
|
||||
name: x.metadata.name || ``,
|
||||
reason: x.reason || ``,
|
||||
};
|
||||
})
|
||||
.filter((x) => x.name.includes(this.podName)),
|
||||
undefined,
|
||||
4,
|
||||
),
|
||||
);
|
||||
await this.cleanupTaskResources();
|
||||
throw error;
|
||||
}
|
||||
|
||||
await this.cleanupTaskResources();
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
OrchestratorLogger.log('Running job failed');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
|
||||
// await this.cleanupTaskResources();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async createJob(
|
||||
commands: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
) {
|
||||
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
|
||||
this.setPodNameAndContainerName(find);
|
||||
}
|
||||
|
||||
private async doesJobExist(name: string) {
|
||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
||||
|
||||
return jobs.body.items.some((x) => x.metadata?.name === name);
|
||||
}
|
||||
|
||||
private async doesFailedJobExist() {
|
||||
const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
|
||||
|
||||
return podStatus.body.status?.phase === `Failed`;
|
||||
}
|
||||
|
||||
private async createNamespacedJob(
|
||||
commands: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
) {
|
||||
for (let index = 0; index < 3; index++) {
|
||||
try {
|
||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
||||
commands,
|
||||
image,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets,
|
||||
this.buildGuid,
|
||||
this.buildParameters,
|
||||
this.secretName,
|
||||
this.pvcName,
|
||||
this.jobName,
|
||||
k8s,
|
||||
this.containerName,
|
||||
this.ip,
|
||||
);
|
||||
await new Promise((promise) => setTimeout(promise, 15000));
|
||||
|
||||
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
||||
|
||||
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||
OrchestratorLogger.log(`Build job created`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
OrchestratorLogger.log('Job created');
|
||||
|
||||
return result.body.metadata?.name;
|
||||
} catch (error) {
|
||||
OrchestratorLogger.log(`Error occured creating job: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setPodNameAndContainerName(pod: k8s.V1Pod) {
|
||||
this.podName = pod.metadata?.name || '';
|
||||
this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
|
||||
}
|
||||
|
||||
async cleanupTaskResources() {
|
||||
OrchestratorLogger.log('cleaning up');
|
||||
try {
|
||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
||||
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`Failed to cleanup`);
|
||||
if (error.response.body.reason !== `NotFound`) {
|
||||
OrchestratorLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`Failed to cleanup secret`);
|
||||
OrchestratorLogger.log(error.response.body.reason);
|
||||
}
|
||||
OrchestratorLogger.log('cleaned up Secret, Job and Pod');
|
||||
OrchestratorLogger.log('cleaning up finished');
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
||||
return;
|
||||
}
|
||||
OrchestratorLogger.log(`deleting PVC`);
|
||||
|
||||
try {
|
||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
||||
OrchestratorLogger.log('cleaned up PVC and Service Account');
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {
|
||||
const namespacedPods = await kubeClient.listNamespacedPod(namespace);
|
||||
const pod = namespacedPods.body.items.find((x) => x.metadata?.labels?.['job-name'] === jobName);
|
||||
if (pod === undefined) {
|
||||
throw new Error("pod with job-name label doesn't exist");
|
||||
}
|
||||
|
||||
return pod;
|
||||
}
|
||||
}
|
||||
export default Kubernetes;
|
||||
@@ -0,0 +1,208 @@
|
||||
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
|
||||
class KubernetesJobSpecFactory {
|
||||
static getJobSpec(
|
||||
command: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingDirectory: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
secretName: string,
|
||||
pvcName: string,
|
||||
jobName: string,
|
||||
k8s: any,
|
||||
containerName: string,
|
||||
ip: string = '',
|
||||
) {
|
||||
const endpointEnvironmentNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
|
||||
// Determine the LocalStack hostname to use for K8s pods
|
||||
// Priority: K8S_LOCALSTACK_HOST env var > localstack-main (container name on shared network)
|
||||
// Note: Using K8S_LOCALSTACK_HOST instead of LOCALSTACK_HOST to avoid conflict with awslocal CLI
|
||||
const localstackHost = process.env['K8S_LOCALSTACK_HOST'] || 'localstack-main';
|
||||
OrchestratorLogger.log(`K8s pods will use LocalStack host: ${localstackHost}`);
|
||||
|
||||
const adjustedEnvironment = environment.map((x) => {
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvironmentNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
// Replace localhost with the LocalStack container hostname
|
||||
// When k3d and LocalStack are on the same Docker network, pods can reach LocalStack by container name
|
||||
value = value
|
||||
.replace('http://localhost', `http://${localstackHost}`)
|
||||
.replace('http://127.0.0.1', `http://${localstackHost}`);
|
||||
OrchestratorLogger.log(`Replaced localhost with ${localstackHost} for ${x.name}: ${value}`);
|
||||
}
|
||||
|
||||
return { name: x.name, value } as OrchestratorEnvironmentVariable;
|
||||
});
|
||||
|
||||
const job = new k8s.V1Job();
|
||||
job.apiVersion = 'batch/v1';
|
||||
job.kind = 'Job';
|
||||
job.metadata = {
|
||||
name: jobName,
|
||||
labels: {
|
||||
app: 'unity-builder',
|
||||
buildGuid,
|
||||
},
|
||||
};
|
||||
|
||||
// Reduce TTL for tests to free up resources faster (default 9999s = ~2.8 hours)
|
||||
// For CI/test environments, use shorter TTL (300s = 5 minutes) to prevent disk pressure
|
||||
const jobTTL = process.env['orchestratorTests'] === 'true' ? 300 : 9999;
|
||||
job.spec = {
|
||||
ttlSecondsAfterFinished: jobTTL,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
terminationGracePeriodSeconds: 90, // Give PreStopHook (60s sleep) time to complete
|
||||
volumes: [
|
||||
{
|
||||
name: 'build-mount',
|
||||
persistentVolumeClaim: {
|
||||
claimName: pvcName,
|
||||
},
|
||||
},
|
||||
],
|
||||
containers: [
|
||||
{
|
||||
ttlSecondsAfterFinished: 9999,
|
||||
name: containerName,
|
||||
image,
|
||||
imagePullPolicy: process.env['orchestratorTests'] === 'true' ? 'IfNotPresent' : 'Always',
|
||||
command: ['/bin/sh'],
|
||||
args: [
|
||||
'-c',
|
||||
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, Orchestrator.buildParameters)}`,
|
||||
],
|
||||
|
||||
workingDir: `${workingDirectory}`,
|
||||
resources: {
|
||||
requests: (() => {
|
||||
// Use smaller resource requests for lightweight hook containers
|
||||
// Hook containers typically use utility images like aws-cli, rclone, etc.
|
||||
const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu'];
|
||||
const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage));
|
||||
|
||||
if (isLightweightContainer && process.env['orchestratorTests'] === 'true') {
|
||||
// For test environments, use minimal resources for hook containers
|
||||
return {
|
||||
memory: '128Mi',
|
||||
cpu: '100m', // 0.1 CPU
|
||||
};
|
||||
}
|
||||
|
||||
// For main build containers, use the configured resources
|
||||
const memoryMB = Number.parseInt(buildParameters.containerMemory);
|
||||
const cpuMB = Number.parseInt(buildParameters.containerCpu);
|
||||
|
||||
return {
|
||||
memory: !Number.isNaN(memoryMB) && memoryMB > 0 ? `${memoryMB / 1024}G` : '750M',
|
||||
cpu: !Number.isNaN(cpuMB) && cpuMB > 0 ? `${cpuMB / 1024}` : '1',
|
||||
};
|
||||
})(),
|
||||
},
|
||||
env: [
|
||||
...adjustedEnvironment.map((x) => {
|
||||
const environmentVariable = new V1EnvVar();
|
||||
environmentVariable.name = x.name;
|
||||
environmentVariable.value = x.value;
|
||||
|
||||
return environmentVariable;
|
||||
}),
|
||||
...secrets.map((x) => {
|
||||
const secret = new V1EnvVarSource();
|
||||
secret.secretKeyRef = new V1SecretKeySelector();
|
||||
secret.secretKeyRef.key = x.ParameterKey;
|
||||
secret.secretKeyRef.name = secretName;
|
||||
const environmentVariable = new V1EnvVar();
|
||||
environmentVariable.name = x.EnvironmentVariable;
|
||||
environmentVariable.valueFrom = secret;
|
||||
|
||||
return environmentVariable;
|
||||
}),
|
||||
{ name: 'LOG_SERVICE_IP', value: ip },
|
||||
],
|
||||
volumeMounts: [
|
||||
{
|
||||
name: 'build-mount',
|
||||
mountPath: `${mountdir}`,
|
||||
},
|
||||
],
|
||||
lifecycle: {
|
||||
preStop: {
|
||||
exec: {
|
||||
command: [
|
||||
'/bin/sh',
|
||||
'-c',
|
||||
'sleep 60; cd /data/builder/action/steps && chmod +x /steps/return_license.sh 2>/dev/null || true; /steps/return_license.sh 2>/dev/null || true',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
restartPolicy: 'Never',
|
||||
|
||||
// Add tolerations for CI/test environments to allow scheduling even with disk pressure
|
||||
// This is acceptable for CI where we aggressively clean up disk space
|
||||
tolerations: [
|
||||
{
|
||||
key: 'node.kubernetes.io/disk-pressure',
|
||||
operator: 'Exists',
|
||||
effect: 'NoSchedule',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (process.env['ORCHESTRATOR_MINIKUBE']) {
|
||||
job.spec.template.spec.volumes[0] = {
|
||||
name: 'build-mount',
|
||||
hostPath: {
|
||||
path: `/data`,
|
||||
type: `Directory`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Set ephemeral-storage request to a reasonable value to prevent evictions
|
||||
// For tests, don't set a request (or use minimal 128Mi) since k3d nodes have very limited disk space
|
||||
// Kubernetes will use whatever is available without a request, which is better for constrained environments
|
||||
// For production, use 2Gi to allow for larger builds
|
||||
// The node needs some free space headroom, so requesting too much causes evictions
|
||||
// With node at 96% usage and only ~2.7GB free, we can't request much without triggering evictions
|
||||
if (process.env['orchestratorTests'] !== 'true') {
|
||||
// Only set ephemeral-storage request for production builds
|
||||
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '2Gi';
|
||||
}
|
||||
|
||||
// For tests, don't set ephemeral-storage request - let Kubernetes use available space
|
||||
|
||||
return job;
|
||||
}
|
||||
}
|
||||
export default KubernetesJobSpecFactory;
|
||||
@@ -0,0 +1,194 @@
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
class KubernetesPods {
|
||||
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
|
||||
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
|
||||
const phase = pods[0]?.status?.phase || 'undefined status';
|
||||
OrchestratorLogger.log(`Getting pod status: ${phase}`);
|
||||
if (phase === `Failed`) {
|
||||
const pod = pods[0];
|
||||
const containerStatuses = pod.status?.containerStatuses || [];
|
||||
const conditions = pod.status?.conditions || [];
|
||||
const events = (await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
.filter((x) => x.involvedObject?.name === podName)
|
||||
.map((x) => ({
|
||||
message: x.message || '',
|
||||
reason: x.reason || '',
|
||||
type: x.type || '',
|
||||
}));
|
||||
|
||||
const errorDetails: string[] = [];
|
||||
errorDetails.push(`Pod: ${podName}`, `Phase: ${phase}`);
|
||||
|
||||
if (conditions.length > 0) {
|
||||
errorDetails.push(
|
||||
`Conditions: ${JSON.stringify(
|
||||
conditions.map((c) => ({ type: c.type, status: c.status, reason: c.reason, message: c.message })),
|
||||
undefined,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
let containerExitCode: number | undefined;
|
||||
let containerSucceeded = false;
|
||||
|
||||
if (containerStatuses.length > 0) {
|
||||
for (const [index, cs] of containerStatuses.entries()) {
|
||||
if (cs.state?.waiting) {
|
||||
errorDetails.push(
|
||||
`Container ${index} (${cs.name}) waiting: ${cs.state.waiting.reason} - ${cs.state.waiting.message || ''}`,
|
||||
);
|
||||
}
|
||||
if (cs.state?.terminated) {
|
||||
const exitCode = cs.state.terminated.exitCode;
|
||||
containerExitCode = exitCode;
|
||||
if (exitCode === 0) {
|
||||
containerSucceeded = true;
|
||||
}
|
||||
errorDetails.push(
|
||||
`Container ${index} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${
|
||||
cs.state.terminated.message || ''
|
||||
} (exit code: ${exitCode})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (events.length > 0) {
|
||||
errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
|
||||
}
|
||||
|
||||
// Check if only PreStopHook failed but container succeeded
|
||||
const hasPreStopHookFailure = events.some((event) => event.reason === 'FailedPreStopHook');
|
||||
const wasKilled = events.some((event) => event.reason === 'Killing');
|
||||
const hasExceededGracePeriod = events.some((event) => event.reason === 'ExceededGracePeriod');
|
||||
|
||||
// If container succeeded (exit code 0), PreStopHook failure is non-critical
|
||||
// Also check if pod was killed but container might have succeeded
|
||||
if (containerSucceeded && containerExitCode === 0) {
|
||||
// Container succeeded - PreStopHook failure is non-critical
|
||||
if (hasPreStopHookFailure) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`,
|
||||
);
|
||||
} else {
|
||||
OrchestratorLogger.log(
|
||||
`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`,
|
||||
);
|
||||
}
|
||||
OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
|
||||
|
||||
// Don't throw error - container succeeded, PreStopHook failure is non-critical
|
||||
return false; // Pod is not running, but we don't treat it as a failure
|
||||
}
|
||||
|
||||
// If pod was killed and we have PreStopHook failure, wait for container status
|
||||
// The container might have succeeded but status hasn't been updated yet
|
||||
if (wasKilled && hasPreStopHookFailure && (containerExitCode === undefined || !containerSucceeded)) {
|
||||
OrchestratorLogger.log(
|
||||
`Pod ${podName} was killed with PreStopHook failure. Waiting for container status to determine if container succeeded...`,
|
||||
);
|
||||
|
||||
// Wait a bit for container status to become available (up to 30 seconds)
|
||||
for (let index = 0; index < 6; index++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
try {
|
||||
const updatedPod = (await kubeClient.listNamespacedPod(namespace)).body.items.find(
|
||||
(x) => podName === x.metadata?.name,
|
||||
);
|
||||
if (updatedPod?.status?.containerStatuses && updatedPod.status.containerStatuses.length > 0) {
|
||||
const updatedContainerStatus = updatedPod.status.containerStatuses[0];
|
||||
if (updatedContainerStatus.state?.terminated) {
|
||||
const updatedExitCode = updatedContainerStatus.state.terminated.exitCode;
|
||||
if (updatedExitCode === 0) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} container succeeded (exit code 0) after waiting. PreStopHook failure is non-fatal.`,
|
||||
);
|
||||
|
||||
return false; // Pod is not running, but container succeeded
|
||||
} else {
|
||||
OrchestratorLogger.log(
|
||||
`Pod ${podName} container failed with exit code ${updatedExitCode} after waiting.`,
|
||||
);
|
||||
errorDetails.push(`Container terminated after wait: exit code ${updatedExitCode}`);
|
||||
containerExitCode = updatedExitCode;
|
||||
containerSucceeded = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (waitError) {
|
||||
OrchestratorLogger.log(`Error while waiting for container status: ${waitError}`);
|
||||
}
|
||||
}
|
||||
|
||||
// If we still don't have container status after waiting, but only PreStopHook failed,
|
||||
// be lenient - the container might have succeeded but status wasn't updated
|
||||
if (containerExitCode === undefined && hasPreStopHookFailure && !hasExceededGracePeriod) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} container status not available after waiting, but only PreStopHook failed (no ExceededGracePeriod). Assuming container may have succeeded.`,
|
||||
);
|
||||
|
||||
return false; // Be lenient - PreStopHook failure alone is not fatal
|
||||
}
|
||||
OrchestratorLogger.log(
|
||||
`Container status check completed. Exit code: ${containerExitCode}, PreStopHook failure: ${hasPreStopHookFailure}`,
|
||||
);
|
||||
}
|
||||
|
||||
// If we only have PreStopHook failure and no actual container failure, be lenient
|
||||
if (hasPreStopHookFailure && !hasExceededGracePeriod && containerExitCode === undefined) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} has PreStopHook failure but no container failure detected. Treating as non-fatal.`,
|
||||
);
|
||||
|
||||
return false; // PreStopHook failure alone is not fatal if container status is unclear
|
||||
}
|
||||
|
||||
// Check if pod was evicted due to disk pressure - this is an infrastructure issue
|
||||
const wasEvicted = errorDetails.some(
|
||||
(detail) => detail.toLowerCase().includes('evicted') || detail.toLowerCase().includes('diskpressure'),
|
||||
);
|
||||
if (wasEvicted) {
|
||||
const evictionMessage = `Pod ${podName} was evicted due to disk pressure. This is a test infrastructure issue - the cluster doesn't have enough disk space.`;
|
||||
OrchestratorLogger.logWarning(evictionMessage);
|
||||
OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
|
||||
throw new Error(
|
||||
`${evictionMessage}\nThis indicates the test environment needs more disk space or better cleanup.\n${errorDetails.join(
|
||||
'\n',
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Exit code 137 (128 + 9) means SIGKILL - container was killed by system (often OOM)
|
||||
// If this happened with PreStopHook failure, it might be a resource issue, not a real failure
|
||||
// Be lenient if we only have PreStopHook/ExceededGracePeriod issues
|
||||
if (containerExitCode === 137 && (hasPreStopHookFailure || hasExceededGracePeriod)) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} was killed (exit code 137 - likely OOM or resource limit) with PreStopHook/grace period issues. This may be a resource constraint issue rather than a build failure.`,
|
||||
);
|
||||
|
||||
// Still log the details but don't fail the test - the build might have succeeded before being killed
|
||||
OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
|
||||
|
||||
return false; // Don't treat system kills as test failures if only PreStopHook issues
|
||||
}
|
||||
|
||||
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
|
||||
OrchestratorLogger.log(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
return running;
|
||||
}
|
||||
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
|
||||
const phase = pods?.status?.phase || 'undefined status';
|
||||
|
||||
return phase;
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesPods;
|
||||
@@ -0,0 +1,53 @@
|
||||
import { RbacAuthorizationV1Api } from '@kubernetes/client-node';
|
||||
|
||||
class KubernetesRole {
|
||||
static async createRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
|
||||
// create admin kubernetes role and role binding
|
||||
const roleBinding = {
|
||||
apiVersion: 'rbac.authorization.k8s.io/v1',
|
||||
kind: 'RoleBinding',
|
||||
metadata: {
|
||||
name: `${serviceAccountName}-admin`,
|
||||
namespace,
|
||||
},
|
||||
subjects: [
|
||||
{
|
||||
kind: 'ServiceAccount',
|
||||
name: serviceAccountName,
|
||||
namespace,
|
||||
},
|
||||
],
|
||||
roleRef: {
|
||||
apiGroup: 'rbac.authorization.k8s.io',
|
||||
kind: 'Role',
|
||||
name: `${serviceAccountName}-admin`,
|
||||
},
|
||||
};
|
||||
|
||||
const role = {
|
||||
apiVersion: 'rbac.authorization.k8s.io/v1',
|
||||
kind: 'Role',
|
||||
metadata: {
|
||||
name: `${serviceAccountName}-admin`,
|
||||
namespace,
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
apiGroups: ['*'],
|
||||
resources: ['*'],
|
||||
verbs: ['*'],
|
||||
},
|
||||
],
|
||||
};
|
||||
const roleBindingResponse = await rbac.createNamespacedRoleBinding(namespace, roleBinding);
|
||||
const roleResponse = await rbac.createNamespacedRole(namespace, role);
|
||||
|
||||
return { roleBindingResponse, roleResponse };
|
||||
}
|
||||
|
||||
public static async deleteRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
|
||||
await rbac.deleteNamespacedRoleBinding(`${serviceAccountName}-admin`, namespace);
|
||||
await rbac.deleteNamespacedRole(`${serviceAccountName}-admin`, namespace);
|
||||
}
|
||||
}
|
||||
export { KubernetesRole };
|
||||
@@ -0,0 +1,45 @@
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as base64 from 'base-64';
|
||||
|
||||
class KubernetesSecret {
|
||||
static async createSecret(
|
||||
secrets: OrchestratorSecret[],
|
||||
secretName: string,
|
||||
namespace: string,
|
||||
kubeClient: CoreV1Api,
|
||||
) {
|
||||
try {
|
||||
const secret = new k8s.V1Secret();
|
||||
secret.apiVersion = 'v1';
|
||||
secret.kind = 'Secret';
|
||||
secret.type = 'Opaque';
|
||||
secret.metadata = {
|
||||
name: secretName,
|
||||
};
|
||||
secret.data = {};
|
||||
for (const buildSecret of secrets) {
|
||||
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
||||
}
|
||||
OrchestratorLogger.log(`Creating secret: ${secretName}`);
|
||||
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
|
||||
const mappedSecrets = existingSecrets.body.items.map((x) => {
|
||||
return x.metadata?.name || `no name`;
|
||||
});
|
||||
|
||||
OrchestratorLogger.log(
|
||||
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
|
||||
);
|
||||
await new Promise((promise) => setTimeout(promise, 15000));
|
||||
await kubeClient.createNamespacedSecret(namespace, secret);
|
||||
OrchestratorLogger.log('Created secret');
|
||||
} catch (error) {
|
||||
OrchestratorLogger.log(`Created secret failed ${error}`);
|
||||
throw new Error(`Failed to create kubernetes secret`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesSecret;
|
||||
@@ -0,0 +1,18 @@
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
|
||||
class KubernetesServiceAccount {
|
||||
static async createServiceAccount(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const serviceAccount = new k8s.V1ServiceAccount();
|
||||
serviceAccount.apiVersion = 'v1';
|
||||
serviceAccount.kind = 'ServiceAccount';
|
||||
serviceAccount.metadata = {
|
||||
name: serviceAccountName,
|
||||
};
|
||||
serviceAccount.automountServiceAccountToken = true;
|
||||
|
||||
return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesServiceAccount;
|
||||
@@ -0,0 +1,276 @@
|
||||
import { waitUntil } from 'async-wait-until';
|
||||
import * as core from '@actions/core';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { IncomingMessage } from 'node:http';
|
||||
import GitHub from '../../../github';
|
||||
|
||||
class KubernetesStorage {
|
||||
public static async createPersistentVolumeClaim(
|
||||
buildParameters: BuildParameters,
|
||||
pvcName: string,
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
) {
|
||||
if (buildParameters.kubeVolume !== ``) {
|
||||
OrchestratorLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
|
||||
pvcName = buildParameters.kubeVolume;
|
||||
|
||||
return;
|
||||
}
|
||||
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
|
||||
const pvcList = allPvc.map((x) => x.metadata?.name);
|
||||
OrchestratorLogger.log(`Current PVCs in namespace ${namespace}`);
|
||||
OrchestratorLogger.log(JSON.stringify(pvcList, undefined, 4));
|
||||
if (pvcList.includes(pvcName)) {
|
||||
OrchestratorLogger.log(`pvc ${pvcName} already exists`);
|
||||
if (GitHub.githubInputEnabled) {
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
OrchestratorLogger.log(`Creating PVC ${pvcName} (does not exist)`);
|
||||
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
|
||||
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
|
||||
}
|
||||
|
||||
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||
try {
|
||||
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
|
||||
} catch (error) {
|
||||
core.error('Failed to get PVC phase');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||
let checkCount = 0;
|
||||
try {
|
||||
OrchestratorLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
||||
|
||||
// Check if storage class uses WaitForFirstConsumer binding mode
|
||||
// If so, skip waiting - PVC will bind when pod is created
|
||||
let shouldSkipWait = false;
|
||||
try {
|
||||
const pvcBody = (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body;
|
||||
const storageClassName = pvcBody.spec?.storageClassName;
|
||||
|
||||
if (storageClassName) {
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
kubeConfig.loadFromDefault();
|
||||
const storageV1Api = kubeConfig.makeApiClient(k8s.StorageV1Api);
|
||||
|
||||
try {
|
||||
const sc = await storageV1Api.readStorageClass(storageClassName);
|
||||
const volumeBindingMode = sc.body.volumeBindingMode;
|
||||
|
||||
if (volumeBindingMode === 'WaitForFirstConsumer') {
|
||||
OrchestratorLogger.log(
|
||||
`StorageClass "${storageClassName}" uses WaitForFirstConsumer binding mode. PVC will bind when pod is created. Skipping wait.`,
|
||||
);
|
||||
shouldSkipWait = true;
|
||||
}
|
||||
} catch (scError) {
|
||||
// If we can't check the storage class, proceed with normal wait
|
||||
OrchestratorLogger.log(
|
||||
`Could not check storage class binding mode: ${scError}. Proceeding with normal wait.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (pvcReadError) {
|
||||
// If we can't read PVC, proceed with normal wait
|
||||
OrchestratorLogger.log(
|
||||
`Could not read PVC to check storage class: ${pvcReadError}. Proceeding with normal wait.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (shouldSkipWait) {
|
||||
OrchestratorLogger.log(`Skipping PVC wait - will bind when pod is created`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const initialPhase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||
OrchestratorLogger.log(`Initial PVC phase: ${initialPhase}`);
|
||||
|
||||
// Wait until PVC is NOT Pending (i.e., Bound or Available)
|
||||
await waitUntil(
|
||||
async () => {
|
||||
checkCount++;
|
||||
const phase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||
|
||||
// Log progress every 4 checks (every ~60 seconds)
|
||||
if (checkCount % 4 === 0) {
|
||||
OrchestratorLogger.log(`PVC ${name} still ${phase} (check ${checkCount})`);
|
||||
|
||||
// Fetch and log PVC events for diagnostics
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const pvcEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.kind === 'PersistentVolumeClaim' && x.involvedObject?.name === name)
|
||||
.map((x) => ({
|
||||
message: x.message || '',
|
||||
reason: x.reason || '',
|
||||
type: x.type || '',
|
||||
count: x.count || 0,
|
||||
}))
|
||||
.slice(-5); // Get last 5 events
|
||||
|
||||
if (pvcEvents.length > 0) {
|
||||
OrchestratorLogger.log(`PVC Events: ${JSON.stringify(pvcEvents, undefined, 2)}`);
|
||||
|
||||
// Check if event indicates WaitForFirstConsumer
|
||||
const waitForConsumerEvent = pvcEvents.find(
|
||||
(event) =>
|
||||
event.reason === 'WaitForFirstConsumer' || event.message?.includes('waiting for first consumer'),
|
||||
);
|
||||
if (waitForConsumerEvent) {
|
||||
OrchestratorLogger.log(
|
||||
`PVC is waiting for first consumer. This is normal for WaitForFirstConsumer storage classes. Proceeding without waiting.`,
|
||||
);
|
||||
|
||||
return true; // Exit wait loop - PVC will bind when pod is created
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
}
|
||||
|
||||
return phase !== 'Pending';
|
||||
},
|
||||
{
|
||||
timeout: 750000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
|
||||
const finalPhase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||
OrchestratorLogger.log(`PVC phase after wait: ${finalPhase}`);
|
||||
|
||||
if (finalPhase === 'Pending') {
|
||||
throw new Error(`PVC ${name} is still Pending after timeout`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
core.error('Failed to watch PVC');
|
||||
core.error(error.toString());
|
||||
try {
|
||||
const pvcBody = (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body;
|
||||
|
||||
// Fetch PVC events for detailed diagnostics
|
||||
let pvcEvents: any[] = [];
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
pvcEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.kind === 'PersistentVolumeClaim' && x.involvedObject?.name === name)
|
||||
.map((x) => ({
|
||||
message: x.message || '',
|
||||
reason: x.reason || '',
|
||||
type: x.type || '',
|
||||
count: x.count || 0,
|
||||
}));
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
|
||||
// Check if storage class exists
|
||||
let storageClassInfo = '';
|
||||
try {
|
||||
const storageClassName = pvcBody.spec?.storageClassName;
|
||||
if (storageClassName) {
|
||||
// Create StorageV1Api from default config
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
kubeConfig.loadFromDefault();
|
||||
const storageV1Api = kubeConfig.makeApiClient(k8s.StorageV1Api);
|
||||
|
||||
try {
|
||||
const sc = await storageV1Api.readStorageClass(storageClassName);
|
||||
storageClassInfo = `StorageClass "${storageClassName}" exists. Provisioner: ${
|
||||
sc.body.provisioner || 'unknown'
|
||||
}`;
|
||||
} catch (scError: any) {
|
||||
storageClassInfo =
|
||||
scError.statusCode === 404
|
||||
? `StorageClass "${storageClassName}" does NOT exist! This is likely why the PVC is stuck in Pending.`
|
||||
: `Failed to check StorageClass "${storageClassName}": ${scError.message || scError}`;
|
||||
}
|
||||
}
|
||||
} catch (scCheckError) {
|
||||
// Ignore storage class check errors - not critical for diagnostics
|
||||
storageClassInfo = `Could not check storage class: ${scCheckError}`;
|
||||
}
|
||||
|
||||
core.error(
|
||||
`PVC Body: ${JSON.stringify(
|
||||
{
|
||||
phase: pvcBody.status?.phase,
|
||||
conditions: pvcBody.status?.conditions,
|
||||
accessModes: pvcBody.spec?.accessModes,
|
||||
storageClassName: pvcBody.spec?.storageClassName,
|
||||
storageRequest: pvcBody.spec?.resources?.requests?.storage,
|
||||
},
|
||||
undefined,
|
||||
4,
|
||||
)}`,
|
||||
);
|
||||
|
||||
if (storageClassInfo) {
|
||||
core.error(storageClassInfo);
|
||||
}
|
||||
|
||||
if (pvcEvents.length > 0) {
|
||||
core.error(`PVC Events: ${JSON.stringify(pvcEvents, undefined, 2)}`);
|
||||
} else {
|
||||
core.error('No PVC events found - this may indicate the storage provisioner is not responding');
|
||||
}
|
||||
} catch {
|
||||
// Ignore PVC read errors
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async createPVC(
|
||||
pvcName: string,
|
||||
buildParameters: BuildParameters,
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
) {
|
||||
const pvc = new k8s.V1PersistentVolumeClaim();
|
||||
pvc.apiVersion = 'v1';
|
||||
pvc.kind = 'PersistentVolumeClaim';
|
||||
pvc.metadata = {
|
||||
name: pvcName,
|
||||
};
|
||||
pvc.spec = {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
|
||||
resources: {
|
||||
requests: {
|
||||
storage: buildParameters.kubeVolumeSize,
|
||||
},
|
||||
},
|
||||
};
|
||||
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static async handleResult(
|
||||
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
pvcName: string,
|
||||
) {
|
||||
const name = result.body.metadata?.name || '';
|
||||
OrchestratorLogger.log(`PVC ${name} created`);
|
||||
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
|
||||
OrchestratorLogger.log(`PVC ${name} is ready and not pending`);
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesStorage;
|
||||
@@ -0,0 +1,763 @@
|
||||
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { waitUntil } from 'async-wait-until';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import KubernetesPods from './kubernetes-pods';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
|
||||
class KubernetesTaskRunner {
|
||||
static readonly maxRetry: number = 3;
|
||||
static lastReceivedMessage: string = ``;
|
||||
|
||||
static async runTask(
|
||||
kubeConfig: KubeConfig,
|
||||
kubeClient: CoreV1Api,
|
||||
jobName: string,
|
||||
podName: string,
|
||||
containerName: string,
|
||||
namespace: string,
|
||||
) {
|
||||
let output = '';
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
let retriesAfterFinish = 0;
|
||||
let kubectlLogsFailedCount = 0;
|
||||
const maxKubectlLogsFailures = 3;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
OrchestratorLogger.log(
|
||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${Orchestrator.buildParameters.kubeVolumeSize}/${Orchestrator.buildParameters.containerCpu}/${Orchestrator.buildParameters.containerMemory}`,
|
||||
);
|
||||
const isRunning = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
|
||||
const callback = (outputChunk: string) => {
|
||||
// Filter out kubectl error messages about being unable to retrieve container logs
|
||||
// These errors pollute the output and don't contain useful information
|
||||
const lowerChunk = outputChunk.toLowerCase();
|
||||
if (lowerChunk.includes('unable to retrieve container logs')) {
|
||||
OrchestratorLogger.log(`Filtered kubectl error: ${outputChunk.trim()}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
output += outputChunk;
|
||||
|
||||
// split output chunk and handle per line
|
||||
for (const chunk of outputChunk.split(`\n`)) {
|
||||
// Skip empty chunks and kubectl error messages (case-insensitive)
|
||||
const lowerCaseChunk = chunk.toLowerCase();
|
||||
if (chunk.trim() && !lowerCaseChunk.includes('unable to retrieve container logs')) {
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
chunk,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Always specify container name explicitly to avoid containerd:// errors
|
||||
// Use -f for running pods, --previous for terminated pods
|
||||
await OrchestratorSystem.Run(
|
||||
`kubectl logs ${podName} -c ${containerName} -n ${namespace}${isRunning ? ' -f' : ' --previous'}`,
|
||||
false,
|
||||
true,
|
||||
callback,
|
||||
);
|
||||
|
||||
// Reset failure count on success
|
||||
kubectlLogsFailedCount = 0;
|
||||
} catch (error: any) {
|
||||
kubectlLogsFailedCount++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
OrchestratorLogger.log(`K8s logging error ${error} ${continueStreaming}`);
|
||||
|
||||
// Filter out kubectl error messages from the error output
|
||||
const errorMessage = error?.message || error?.toString() || '';
|
||||
const isKubectlLogsError =
|
||||
errorMessage.includes('unable to retrieve container logs for containerd://') ||
|
||||
errorMessage.toLowerCase().includes('unable to retrieve container logs');
|
||||
|
||||
if (isKubectlLogsError) {
|
||||
OrchestratorLogger.log(
|
||||
`Kubectl unable to retrieve logs, attempt ${kubectlLogsFailedCount}/${maxKubectlLogsFailures}`,
|
||||
);
|
||||
|
||||
// If kubectl logs has failed multiple times, try reading the log file directly from the pod
|
||||
// This works even if the pod is terminated, as long as it hasn't been deleted
|
||||
if (kubectlLogsFailedCount >= maxKubectlLogsFailures && !isRunning && !continueStreaming) {
|
||||
OrchestratorLogger.log(`Attempting to read log file directly from pod as fallback...`);
|
||||
try {
|
||||
// Try to read the log file from the pod
|
||||
// Use kubectl exec for running pods, or try to access via PVC if pod is terminated
|
||||
let logFileContent = '';
|
||||
|
||||
if (isRunning) {
|
||||
// Pod is still running, try exec
|
||||
logFileContent = await OrchestratorSystem.Run(
|
||||
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
} else {
|
||||
// Pod is terminated, try to create a temporary pod to read from the PVC
|
||||
// First, check if we can still access the pod's filesystem
|
||||
OrchestratorLogger.log(`Pod is terminated, attempting to read log file via temporary pod...`);
|
||||
|
||||
// For terminated pods, we might not be able to exec, so we'll skip this fallback
|
||||
// and rely on the log file being written to the PVC (if mounted)
|
||||
OrchestratorLogger.logWarning(`Cannot read log file from terminated pod via exec`);
|
||||
}
|
||||
|
||||
if (logFileContent && logFileContent.trim()) {
|
||||
OrchestratorLogger.log(`Successfully read log file from pod (${logFileContent.length} chars)`);
|
||||
|
||||
// Process the log file content line by line
|
||||
for (const line of logFileContent.split(`\n`)) {
|
||||
const lowerLine = line.toLowerCase();
|
||||
if (line.trim() && !lowerLine.includes('unable to retrieve container logs')) {
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
line,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we got the end of transmission marker
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
OrchestratorLogger.log('end of log stream (from log file)');
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
OrchestratorLogger.logWarning(`Log file read returned empty content, continuing with available logs`);
|
||||
|
||||
// If we can't read the log file, break out of the loop to return whatever logs we have
|
||||
// This prevents infinite retries when kubectl logs consistently fails
|
||||
break;
|
||||
}
|
||||
} catch (execError: any) {
|
||||
OrchestratorLogger.logWarning(`Failed to read log file from pod: ${execError}`);
|
||||
|
||||
// If we've exhausted all options, break to return whatever logs we have
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If pod is not running and we tried --previous but it failed, try without --previous
|
||||
if (!isRunning && !continueStreaming && error?.message?.includes('previous terminated container')) {
|
||||
OrchestratorLogger.log(`Previous container not found, trying current container logs...`);
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`kubectl logs ${podName} -c ${containerName} -n ${namespace}`,
|
||||
false,
|
||||
true,
|
||||
callback,
|
||||
);
|
||||
|
||||
// If we successfully got logs, check for end of transmission
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
OrchestratorLogger.log('end of log stream');
|
||||
break;
|
||||
}
|
||||
|
||||
// If we got logs but no end marker, continue trying (might be more logs)
|
||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||
retriesAfterFinish++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we've exhausted retries, break
|
||||
break;
|
||||
} catch (fallbackError: any) {
|
||||
OrchestratorLogger.log(`Fallback log fetch also failed: ${fallbackError}`);
|
||||
|
||||
// If both fail, continue retrying if we haven't exhausted retries
|
||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||
retriesAfterFinish++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only break if we've exhausted all retries
|
||||
OrchestratorLogger.logWarning(
|
||||
`Could not fetch any container logs after ${KubernetesTaskRunner.maxRetry} retries`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (continueStreaming) {
|
||||
continue;
|
||||
}
|
||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||
retriesAfterFinish++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we've exhausted retries and it's not a previous container issue, throw
|
||||
if (!error?.message?.includes('previous terminated container')) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// For previous container errors, we've already tried fallback, so just break
|
||||
OrchestratorLogger.logWarning(
|
||||
`Could not fetch previous container logs after retries, but continuing with available logs`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
OrchestratorLogger.log('end of log stream');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// After kubectl logs loop ends, read log file as fallback to capture any messages
|
||||
// written after kubectl stopped reading (e.g., "Collected Logs" from post-build)
|
||||
// This ensures all log messages are included in BuildResults for test assertions
|
||||
// If output is empty, we need to be more aggressive about getting logs
|
||||
const needsFallback = output.trim().length === 0;
|
||||
const missingCollectedLogs = !output.includes('Collected Logs');
|
||||
|
||||
if (needsFallback) {
|
||||
OrchestratorLogger.log('Output is empty, attempting aggressive log collection fallback...');
|
||||
|
||||
// Give the pod a moment to finish writing logs before we try to read them
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
}
|
||||
|
||||
// Always try fallback if output is empty, if pod is terminated, or if "Collected Logs" is missing
|
||||
// The "Collected Logs" check ensures we try to get post-build messages even if we have some output
|
||||
try {
|
||||
const isPodStillRunning = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
const shouldTryFallback = !isPodStillRunning || needsFallback || missingCollectedLogs;
|
||||
|
||||
if (shouldTryFallback) {
|
||||
const reason = needsFallback
|
||||
? 'output is empty'
|
||||
: missingCollectedLogs
|
||||
? 'Collected Logs missing from output'
|
||||
: 'pod is terminated';
|
||||
OrchestratorLogger.log(
|
||||
`Pod is ${isPodStillRunning ? 'running' : 'terminated'} and ${reason}, reading log file as fallback...`,
|
||||
);
|
||||
try {
|
||||
// Try to read the log file from the pod
|
||||
// For killed pods (OOM), kubectl exec might not work, so we try multiple approaches
|
||||
// First try --previous flag for terminated containers, then try without it
|
||||
let logFileContent = '';
|
||||
|
||||
// Try multiple approaches to get the log file
|
||||
// Order matters: try terminated container first, then current, then PVC, then kubectl logs as last resort
|
||||
// For K8s, the PVC is mounted at /data, so try reading from there too
|
||||
const attempts = [
|
||||
// For terminated pods, try --previous first
|
||||
`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /home/job-log.txt 2>/dev/null || echo ""`,
|
||||
|
||||
// Try current container
|
||||
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
|
||||
|
||||
// Try reading from PVC (/data) in case log was copied there
|
||||
`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /data/job-log.txt 2>/dev/null || echo ""`,
|
||||
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /data/job-log.txt 2>/dev/null || echo ""`,
|
||||
|
||||
// Try kubectl logs as fallback (might capture stdout even if exec fails)
|
||||
`kubectl logs ${podName} -c ${containerName} -n ${namespace} --previous 2>/dev/null || echo ""`,
|
||||
`kubectl logs ${podName} -c ${containerName} -n ${namespace} 2>/dev/null || echo ""`,
|
||||
];
|
||||
|
||||
for (const attempt of attempts) {
|
||||
// If we already have content with "Collected Logs", no need to try more
|
||||
if (logFileContent && logFileContent.trim() && logFileContent.includes('Collected Logs')) {
|
||||
OrchestratorLogger.log('Found "Collected Logs" in fallback content, stopping attempts.');
|
||||
break;
|
||||
}
|
||||
try {
|
||||
OrchestratorLogger.log(`Trying fallback method: ${attempt.slice(0, 80)}...`);
|
||||
const result = await OrchestratorSystem.Run(attempt, true, true);
|
||||
if (result && result.trim()) {
|
||||
// Prefer content that has "Collected Logs" over content that doesn't
|
||||
if (!logFileContent || !logFileContent.includes('Collected Logs')) {
|
||||
logFileContent = result;
|
||||
OrchestratorLogger.log(
|
||||
`Successfully read logs using fallback method (${logFileContent.length} chars): ${attempt.slice(
|
||||
0,
|
||||
50,
|
||||
)}...`,
|
||||
);
|
||||
|
||||
// If this content has "Collected Logs", we're done
|
||||
if (logFileContent.includes('Collected Logs')) {
|
||||
OrchestratorLogger.log('Fallback method successfully captured "Collected Logs".');
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
OrchestratorLogger.log(`Skipping this result - already have content with "Collected Logs".`);
|
||||
}
|
||||
} else {
|
||||
OrchestratorLogger.log(`Fallback method returned empty result: ${attempt.slice(0, 50)}...`);
|
||||
}
|
||||
} catch (attemptError: any) {
|
||||
OrchestratorLogger.log(
|
||||
`Fallback method failed: ${attempt.slice(0, 50)}... Error: ${attemptError?.message || attemptError}`,
|
||||
);
|
||||
|
||||
// Continue to next attempt
|
||||
}
|
||||
}
|
||||
|
||||
if (!logFileContent || !logFileContent.trim()) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'Could not read log file from pod after all fallback attempts (may be OOM-killed or pod not accessible).',
|
||||
);
|
||||
}
|
||||
|
||||
if (logFileContent && logFileContent.trim()) {
|
||||
OrchestratorLogger.log(
|
||||
`Read log file from pod as fallback (${logFileContent.length} chars) to capture missing messages`,
|
||||
);
|
||||
|
||||
// Get the lines we already have in output to avoid duplicates
|
||||
const existingLines = new Set(output.split('\n').map((line) => line.trim()));
|
||||
|
||||
// Process the log file content line by line and add missing lines
|
||||
for (const line of logFileContent.split(`\n`)) {
|
||||
const trimmedLine = line.trim();
|
||||
const lowerLine = trimmedLine.toLowerCase();
|
||||
|
||||
// Skip empty lines, kubectl errors, and lines we already have
|
||||
if (
|
||||
trimmedLine &&
|
||||
!lowerLine.includes('unable to retrieve container logs') &&
|
||||
!existingLines.has(trimmedLine)
|
||||
) {
|
||||
// Process through FollowLogStreamService - it will append to output
|
||||
// Don't add to output manually since handleIteration does it
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
trimmedLine,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (logFileError: any) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Could not read log file from pod as fallback: ${logFileError?.message || logFileError}`,
|
||||
);
|
||||
|
||||
// Continue with existing output - this is a best-effort fallback
|
||||
}
|
||||
}
|
||||
|
||||
// If output is still empty or missing "Collected Logs" after fallback attempts, add a warning message
|
||||
// This ensures BuildResults is not completely empty, which would cause test failures
|
||||
if ((needsFallback && output.trim().length === 0) || (!output.includes('Collected Logs') && shouldTryFallback)) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'Could not retrieve "Collected Logs" from pod after all attempts. Pod may have been killed before logs were written.',
|
||||
);
|
||||
|
||||
// Add a minimal message so BuildResults is not completely empty
|
||||
// This helps with debugging and prevents test failures due to empty results
|
||||
if (output.trim().length === 0) {
|
||||
output = 'Pod logs unavailable - pod may have been terminated before logs could be collected.\n';
|
||||
} else if (!output.includes('Collected Logs')) {
|
||||
// We have some output but missing "Collected Logs" - append the fallback message
|
||||
output +=
|
||||
'\nPod logs incomplete - "Collected Logs" marker not found. Pod may have been terminated before post-build completed.\n';
|
||||
}
|
||||
}
|
||||
} catch (fallbackError: any) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Error checking pod status for log file fallback: ${fallbackError?.message || fallbackError}`,
|
||||
);
|
||||
|
||||
// If output is empty and we hit an error, still add a message so BuildResults isn't empty
|
||||
if (needsFallback && output.trim().length === 0) {
|
||||
output = `Error retrieving logs: ${fallbackError?.message || fallbackError}\n`;
|
||||
}
|
||||
|
||||
// Continue with existing output - this is a best-effort fallback
|
||||
}
|
||||
|
||||
// Filter out kubectl error messages from the final output
|
||||
// These errors can be added via stderr even when kubectl fails
|
||||
// We filter them out so they don't pollute the BuildResults
|
||||
const lines = output.split('\n');
|
||||
const filteredLines = lines.filter((line) => !line.toLowerCase().includes('unable to retrieve container logs'));
|
||||
const filteredOutput = filteredLines.join('\n');
|
||||
|
||||
// Log if we filtered out significant content
|
||||
const originalLineCount = lines.length;
|
||||
const filteredLineCount = filteredLines.length;
|
||||
if (originalLineCount > filteredLineCount) {
|
||||
OrchestratorLogger.log(
|
||||
`Filtered out ${originalLineCount - filteredLineCount} kubectl error message(s) from output`,
|
||||
);
|
||||
}
|
||||
|
||||
return filteredOutput;
|
||||
}
|
||||
|
||||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||
let waitComplete: boolean = false;
|
||||
let message = ``;
|
||||
let lastPhase = '';
|
||||
let consecutivePendingCount = 0;
|
||||
OrchestratorLogger.log(`Watching ${podName} ${namespace}`);
|
||||
|
||||
try {
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const phase = status?.body.status?.phase || 'Unknown';
|
||||
const conditions = status?.body.status?.conditions || [];
|
||||
const containerStatuses = status?.body.status?.containerStatuses || [];
|
||||
|
||||
// Log phase changes
|
||||
if (phase !== lastPhase) {
|
||||
OrchestratorLogger.log(`Pod ${podName} phase changed: ${lastPhase} -> ${phase}`);
|
||||
lastPhase = phase;
|
||||
consecutivePendingCount = 0;
|
||||
}
|
||||
|
||||
// Check for failure conditions that mean the pod will never start (permanent failures)
|
||||
// Note: We don't treat "Failed" phase as a permanent failure because the pod might have
|
||||
// completed its work before being killed (OOM), and we should still try to get logs
|
||||
const permanentFailureReasons = [
|
||||
'Unschedulable',
|
||||
'ImagePullBackOff',
|
||||
'ErrImagePull',
|
||||
'CreateContainerError',
|
||||
'CreateContainerConfigError',
|
||||
];
|
||||
|
||||
const hasPermanentFailureCondition = conditions.some((condition: any) =>
|
||||
permanentFailureReasons.some((reason) => condition.reason?.includes(reason)),
|
||||
);
|
||||
|
||||
const hasPermanentFailureContainerStatus = containerStatuses.some((containerStatus: any) =>
|
||||
permanentFailureReasons.some((reason) => containerStatus.state?.waiting?.reason?.includes(reason)),
|
||||
);
|
||||
|
||||
// Only treat permanent failures as errors - pods that completed (Failed/Succeeded) should continue
|
||||
if (hasPermanentFailureCondition || hasPermanentFailureContainerStatus) {
|
||||
// Get detailed failure information
|
||||
const failureCondition = conditions.find((condition: any) =>
|
||||
permanentFailureReasons.some((reason) => condition.reason?.includes(reason)),
|
||||
);
|
||||
const failureContainer = containerStatuses.find((containerStatus: any) =>
|
||||
permanentFailureReasons.some((reason) => containerStatus.state?.waiting?.reason?.includes(reason)),
|
||||
);
|
||||
|
||||
message = `Pod ${podName} failed to start (permanent failure):\nPhase: ${phase}\n`;
|
||||
if (failureCondition) {
|
||||
message += `Condition Reason: ${failureCondition.reason}\nCondition Message: ${failureCondition.message}\n`;
|
||||
}
|
||||
if (failureContainer) {
|
||||
message += `Container Reason: ${failureContainer.state?.waiting?.reason}\nContainer Message: ${failureContainer.state?.waiting?.message}\n`;
|
||||
}
|
||||
|
||||
// Log pod events for additional context
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const podEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.name === podName)
|
||||
.map((x) => ({
|
||||
message: x.message || ``,
|
||||
reason: x.reason || ``,
|
||||
type: x.type || ``,
|
||||
}));
|
||||
if (podEvents.length > 0) {
|
||||
message += `\nRecent Events:\n${JSON.stringify(podEvents.slice(-5), undefined, 2)}`;
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
|
||||
OrchestratorLogger.logWarning(message);
|
||||
|
||||
// For permanent failures, mark as incomplete and store the error message
|
||||
// We'll throw an error after the wait loop exits
|
||||
waitComplete = false;
|
||||
|
||||
return true; // Return true to exit wait loop
|
||||
}
|
||||
|
||||
// Pod is complete if it's not Pending or Unknown - it might be Running, Succeeded, or Failed
|
||||
// For Failed/Succeeded pods, we still want to try to get logs, so we mark as complete
|
||||
waitComplete = phase !== 'Pending' && phase !== 'Unknown';
|
||||
|
||||
// If pod completed (Succeeded/Failed), log it but don't throw - we'll try to get logs
|
||||
if (waitComplete && phase !== 'Running') {
|
||||
OrchestratorLogger.log(`Pod ${podName} completed with phase: ${phase}. Will attempt to retrieve logs.`);
|
||||
}
|
||||
|
||||
if (phase === 'Pending') {
|
||||
consecutivePendingCount++;
|
||||
|
||||
// Check for scheduling failures in events (faster than waiting for conditions)
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const podEvents = events.body.items.filter((x) => x.involvedObject?.name === podName);
|
||||
const failedSchedulingEvents = podEvents.filter(
|
||||
(x) => x.reason === 'FailedScheduling' || x.reason === 'SchedulingGated',
|
||||
);
|
||||
|
||||
if (failedSchedulingEvents.length > 0) {
|
||||
const schedulingMessage = failedSchedulingEvents
|
||||
.map((x) => `${x.reason}: ${x.message || ''}`)
|
||||
.join('; ');
|
||||
message = `Pod ${podName} cannot be scheduled:\n${schedulingMessage}`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
waitComplete = false;
|
||||
|
||||
return true; // Exit wait loop to throw error
|
||||
}
|
||||
|
||||
// Check if pod is actively pulling an image - if so, allow more time
|
||||
const isPullingImage = podEvents.some(
|
||||
(x) => x.reason === 'Pulling' || x.reason === 'Pulled' || x.message?.includes('Pulling image'),
|
||||
);
|
||||
const hasImagePullError = podEvents.some(
|
||||
(x) => x.reason === 'Failed' && (x.message?.includes('pull') || x.message?.includes('image')),
|
||||
);
|
||||
|
||||
if (hasImagePullError) {
|
||||
message = `Pod ${podName} failed to pull image. Check image availability and credentials.`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
waitComplete = false;
|
||||
|
||||
return true; // Exit wait loop to throw error
|
||||
}
|
||||
|
||||
// If actively pulling image, reset pending count to allow more time
|
||||
// Large images (like Unity 3.9GB) can take 3-5 minutes to pull
|
||||
if (isPullingImage && consecutivePendingCount > 4) {
|
||||
OrchestratorLogger.log(
|
||||
`Pod ${podName} is pulling image (check ${consecutivePendingCount}). This may take several minutes for large images.`,
|
||||
);
|
||||
|
||||
// Don't increment consecutivePendingCount if we're actively pulling
|
||||
consecutivePendingCount = Math.max(4, consecutivePendingCount - 1);
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
|
||||
// For tests, allow more time if image is being pulled (large images need 5+ minutes)
|
||||
// Otherwise fail faster if stuck in Pending (2 minutes = 8 checks at 15s interval)
|
||||
const isTest = process.env['orchestratorTests'] === 'true';
|
||||
const isPullingImage =
|
||||
containerStatuses.some(
|
||||
(cs: any) => cs.state?.waiting?.reason === 'ImagePull' || cs.state?.waiting?.reason === 'ErrImagePull',
|
||||
) || conditions.some((c: any) => c.reason?.includes('Pulling'));
|
||||
|
||||
// Allow up to 20 minutes for image pulls in tests (80 checks), 2 minutes otherwise
|
||||
const maxPendingChecks = isTest && isPullingImage ? 80 : isTest ? 8 : 80;
|
||||
|
||||
if (consecutivePendingCount >= maxPendingChecks) {
|
||||
message = `Pod ${podName} stuck in Pending state for too long (${consecutivePendingCount} checks). This indicates a scheduling problem.`;
|
||||
|
||||
// Get events for context
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const podEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.name === podName)
|
||||
.slice(-10)
|
||||
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
||||
if (podEvents.length > 0) {
|
||||
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
||||
}
|
||||
|
||||
// Get pod details to check for scheduling issues
|
||||
try {
|
||||
const podStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const podSpec = podStatus.body.spec;
|
||||
const podStatusDetails = podStatus.body.status;
|
||||
|
||||
// Check container resource requests
|
||||
if (podSpec?.containers?.[0]?.resources?.requests) {
|
||||
const requests = podSpec.containers[0].resources.requests;
|
||||
message += `\n\nContainer Resource Requests:\n CPU: ${requests.cpu || 'not set'}\n Memory: ${
|
||||
requests.memory || 'not set'
|
||||
}\n Ephemeral Storage: ${requests['ephemeral-storage'] || 'not set'}`;
|
||||
}
|
||||
|
||||
// Check node selector and tolerations
|
||||
if (podSpec?.nodeSelector && Object.keys(podSpec.nodeSelector).length > 0) {
|
||||
message += `\n\nNode Selector: ${JSON.stringify(podSpec.nodeSelector)}`;
|
||||
}
|
||||
if (podSpec?.tolerations && podSpec.tolerations.length > 0) {
|
||||
message += `\n\nTolerations: ${JSON.stringify(podSpec.tolerations)}`;
|
||||
}
|
||||
|
||||
// Check pod conditions for scheduling issues
|
||||
if (podStatusDetails?.conditions) {
|
||||
const allConditions = podStatusDetails.conditions.map(
|
||||
(c: any) =>
|
||||
`${c.type}: ${c.status}${c.reason ? ` (${c.reason})` : ''}${
|
||||
c.message ? ` - ${c.message}` : ''
|
||||
}`,
|
||||
);
|
||||
message += `\n\nPod Conditions:\n${allConditions.join('\n')}`;
|
||||
|
||||
const unschedulable = podStatusDetails.conditions.find(
|
||||
(c: any) => c.type === 'PodScheduled' && c.status === 'False',
|
||||
);
|
||||
if (unschedulable) {
|
||||
message += `\n\nScheduling Issue: ${unschedulable.reason || 'Unknown'} - ${
|
||||
unschedulable.message || 'No message'
|
||||
}`;
|
||||
}
|
||||
|
||||
// Check if pod is assigned to a node
|
||||
message += podStatusDetails?.hostIP
|
||||
? `\n\nPod assigned to node: ${podStatusDetails.hostIP}`
|
||||
: `\n\nPod not yet assigned to a node (scheduling pending)`;
|
||||
}
|
||||
|
||||
// Check node resources if pod is assigned
|
||||
if (podStatusDetails?.hostIP) {
|
||||
try {
|
||||
const nodes = await kubeClient.listNode();
|
||||
const hostIP = podStatusDetails.hostIP;
|
||||
const assignedNode = nodes.body.items.find((n: any) =>
|
||||
n.status?.addresses?.some((a: any) => a.address === hostIP),
|
||||
);
|
||||
if (assignedNode?.status && assignedNode.metadata?.name) {
|
||||
const allocatable = assignedNode.status.allocatable || {};
|
||||
message += `\n\nNode Resources (${assignedNode.metadata.name}):\n Allocatable CPU: ${
|
||||
allocatable.cpu || 'unknown'
|
||||
}\n Allocatable Memory: ${allocatable.memory || 'unknown'}\n Allocatable Ephemeral Storage: ${
|
||||
allocatable['ephemeral-storage'] || 'unknown'
|
||||
}`;
|
||||
|
||||
// Check for taints that might prevent scheduling
|
||||
if (assignedNode.spec?.taints && assignedNode.spec.taints.length > 0) {
|
||||
const taints = assignedNode.spec.taints
|
||||
.map((t: any) => `${t.key}=${t.value}:${t.effect}`)
|
||||
.join(', ');
|
||||
message += `\n Node Taints: ${taints}`;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore node check errors
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore pod status fetch errors
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
OrchestratorLogger.logWarning(message);
|
||||
waitComplete = false;
|
||||
|
||||
return true; // Exit wait loop to throw error
|
||||
}
|
||||
|
||||
// Log diagnostic info every 4 checks (1 minute) if still pending
|
||||
if (consecutivePendingCount % 4 === 0) {
|
||||
const pendingMessage = `Pod ${podName} still Pending (check ${consecutivePendingCount}/${maxPendingChecks}). Phase: ${phase}`;
|
||||
const conditionMessages = conditions
|
||||
.map((c: any) => `${c.type}: ${c.reason || 'N/A'} - ${c.message || 'N/A'}`)
|
||||
.join('; ');
|
||||
OrchestratorLogger.log(`${pendingMessage}. Conditions: ${conditionMessages || 'None'}`);
|
||||
|
||||
// Log events periodically to help diagnose
|
||||
if (consecutivePendingCount % 8 === 0) {
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const podEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.name === podName)
|
||||
.slice(-3)
|
||||
.map((x) => `${x.type}: ${x.reason} - ${x.message}`)
|
||||
.join('; ');
|
||||
if (podEvents) {
|
||||
OrchestratorLogger.log(`Recent pod events: ${podEvents}`);
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
message = `Phase:${phase} \n Reason:${conditions[0]?.reason || ''} \n Message:${
|
||||
conditions[0]?.message || ''
|
||||
}`;
|
||||
|
||||
if (waitComplete || phase !== 'Pending') return true;
|
||||
|
||||
return false;
|
||||
},
|
||||
{
|
||||
timeout: process.env['orchestratorTests'] === 'true' ? 300000 : 2000000, // 5 minutes for tests, ~33 minutes for production
|
||||
intervalBetweenAttempts: 15000, // 15 seconds
|
||||
},
|
||||
);
|
||||
} catch (waitError: any) {
|
||||
// If waitUntil times out or throws, get final pod status
|
||||
try {
|
||||
const finalStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const phase = finalStatus?.body.status?.phase || 'Unknown';
|
||||
const conditions = finalStatus?.body.status?.conditions || [];
|
||||
message = `Pod ${podName} timed out waiting to start.\nFinal Phase: ${phase}\n`;
|
||||
message += conditions.map((c: any) => `${c.type}: ${c.reason} - ${c.message}`).join('\n');
|
||||
|
||||
// Get events for context
|
||||
try {
|
||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||
const podEvents = events.body.items
|
||||
.filter((x) => x.involvedObject?.name === podName)
|
||||
.slice(-5)
|
||||
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
||||
if (podEvents.length > 0) {
|
||||
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
||||
}
|
||||
} catch {
|
||||
// Ignore event fetch errors
|
||||
}
|
||||
|
||||
OrchestratorLogger.logWarning(message);
|
||||
} catch {
|
||||
message = `Pod ${podName} timed out and could not retrieve final status: ${waitError?.message || waitError}`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
}
|
||||
|
||||
throw new Error(`Pod ${podName} failed to start within timeout. ${message}`);
|
||||
}
|
||||
|
||||
// Only throw if we detected a permanent failure condition
|
||||
// If the pod completed (Failed/Succeeded), we should still try to get logs
|
||||
if (!waitComplete) {
|
||||
// Check the final phase to see if it's a permanent failure or just completed
|
||||
try {
|
||||
const finalStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const finalPhase = finalStatus?.body.status?.phase || 'Unknown';
|
||||
if (finalPhase === 'Failed' || finalPhase === 'Succeeded') {
|
||||
OrchestratorLogger.logWarning(
|
||||
`Pod ${podName} completed with phase ${finalPhase} before reaching Running state. Will attempt to retrieve logs.`,
|
||||
);
|
||||
|
||||
return true; // Allow workflow to continue and try to get logs
|
||||
}
|
||||
} catch {
|
||||
// If we can't check status, fall through to throw error
|
||||
}
|
||||
OrchestratorLogger.logWarning(`Pod ${podName} did not reach running state: ${message}`);
|
||||
throw new Error(`Pod ${podName} did not start successfully: ${message}`);
|
||||
}
|
||||
|
||||
return waitComplete;
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesTaskRunner;
|
||||
@@ -0,0 +1,87 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { quote } from 'shell-quote';
|
||||
|
||||
class LocalOrchestrator implements ProviderInterface {
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(image);
|
||||
OrchestratorLogger.log(buildGuid);
|
||||
OrchestratorLogger.log(commands);
|
||||
|
||||
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
||||
if (process.platform === 'win32') {
|
||||
const inline = commands
|
||||
.replace(/\r/g, '')
|
||||
.split('\n')
|
||||
.filter((x) => x.trim().length > 0)
|
||||
.join(' ; ');
|
||||
|
||||
// Use shell-quote to properly escape the command string, preventing command injection
|
||||
const bashWrapped = `bash -lc ${quote([inline])}`;
|
||||
|
||||
return await OrchestratorSystem.Run(bashWrapped);
|
||||
}
|
||||
|
||||
return await OrchestratorSystem.Run(commands);
|
||||
}
|
||||
}
|
||||
export default LocalOrchestrator;
|
||||
@@ -0,0 +1,278 @@
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import * as fs from 'fs';
|
||||
import path from 'path';
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
export interface GitCloneResult {
|
||||
success: boolean;
|
||||
localPath: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface GitUpdateResult {
|
||||
success: boolean;
|
||||
updated: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages git operations for provider repositories
|
||||
*/
|
||||
export class ProviderGitManager {
|
||||
private static readonly CACHE_DIR = path.join(process.cwd(), '.provider-cache');
|
||||
private static readonly GIT_TIMEOUT = 30000; // 30 seconds
|
||||
|
||||
/**
|
||||
* Ensures the cache directory exists
|
||||
*/
|
||||
private static ensureCacheDir(): void {
|
||||
if (!fs.existsSync(this.CACHE_DIR)) {
|
||||
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
|
||||
OrchestratorLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the local path for a cached repository
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns Local path to the repository
|
||||
*/
|
||||
private static getLocalPath(urlInfo: GitHubUrlInfo): string {
|
||||
const cacheKey = generateCacheKey(urlInfo);
|
||||
|
||||
return path.join(this.CACHE_DIR, cacheKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a repository is already cloned locally
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns True if repository exists locally
|
||||
*/
|
||||
private static isRepositoryCloned(urlInfo: GitHubUrlInfo): boolean {
|
||||
const localPath = this.getLocalPath(urlInfo);
|
||||
|
||||
return fs.existsSync(localPath) && fs.existsSync(path.join(localPath, '.git'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Clones a GitHub repository to the local cache
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns Clone result with success status and local path
|
||||
*/
|
||||
static async cloneRepository(urlInfo: GitHubUrlInfo): Promise<GitCloneResult> {
|
||||
this.ensureCacheDir();
|
||||
const localPath = this.getLocalPath(urlInfo);
|
||||
|
||||
// Remove existing directory if it exists
|
||||
if (fs.existsSync(localPath)) {
|
||||
OrchestratorLogger.log(`Removing existing directory: ${localPath}`);
|
||||
fs.rmSync(localPath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
try {
|
||||
OrchestratorLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
|
||||
|
||||
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
|
||||
OrchestratorLogger.log(`Executing: ${cloneCommand}`);
|
||||
|
||||
const { stderr } = await execAsync(cloneCommand, {
|
||||
timeout: this.GIT_TIMEOUT,
|
||||
cwd: this.CACHE_DIR,
|
||||
});
|
||||
|
||||
if (stderr && !stderr.includes('warning')) {
|
||||
OrchestratorLogger.log(`Git clone stderr: ${stderr}`);
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Successfully cloned repository to: ${localPath}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
localPath,
|
||||
};
|
||||
} catch (error: any) {
|
||||
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
|
||||
OrchestratorLogger.log(`Error: ${errorMessage}`);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
localPath,
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a locally cloned repository
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns Update result with success status and whether it was updated
|
||||
*/
|
||||
static async updateRepository(urlInfo: GitHubUrlInfo): Promise<GitUpdateResult> {
|
||||
const localPath = this.getLocalPath(urlInfo);
|
||||
|
||||
if (!this.isRepositoryCloned(urlInfo)) {
|
||||
return {
|
||||
success: false,
|
||||
updated: false,
|
||||
error: 'Repository not found locally',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
OrchestratorLogger.log(`Updating repository: ${localPath}`);
|
||||
|
||||
// Fetch latest changes
|
||||
await execAsync('git fetch origin', {
|
||||
timeout: this.GIT_TIMEOUT,
|
||||
cwd: localPath,
|
||||
});
|
||||
|
||||
// Check if there are updates
|
||||
const { stdout: statusOutput } = await execAsync(`git status -uno`, {
|
||||
timeout: this.GIT_TIMEOUT,
|
||||
cwd: localPath,
|
||||
});
|
||||
|
||||
const hasUpdates =
|
||||
statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
|
||||
|
||||
if (hasUpdates) {
|
||||
OrchestratorLogger.log(`Updates available, pulling latest changes...`);
|
||||
|
||||
// Reset to origin/branch to get latest changes
|
||||
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
|
||||
timeout: this.GIT_TIMEOUT,
|
||||
cwd: localPath,
|
||||
});
|
||||
|
||||
OrchestratorLogger.log(`Repository updated successfully`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
updated: true,
|
||||
};
|
||||
} else {
|
||||
OrchestratorLogger.log(`Repository is already up to date`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
updated: false,
|
||||
};
|
||||
}
|
||||
} catch (error: any) {
|
||||
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
|
||||
OrchestratorLogger.log(`Error: ${errorMessage}`);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
updated: false,
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures a repository is available locally (clone if needed, update if exists)
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns Local path to the repository
|
||||
*/
|
||||
static async ensureRepositoryAvailable(urlInfo: GitHubUrlInfo): Promise<string> {
|
||||
this.ensureCacheDir();
|
||||
|
||||
if (this.isRepositoryCloned(urlInfo)) {
|
||||
OrchestratorLogger.log(`Repository already exists locally, checking for updates...`);
|
||||
const updateResult = await this.updateRepository(urlInfo);
|
||||
|
||||
if (!updateResult.success) {
|
||||
OrchestratorLogger.log(`Failed to update repository, attempting fresh clone...`);
|
||||
const cloneResult = await this.cloneRepository(urlInfo);
|
||||
if (!cloneResult.success) {
|
||||
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
|
||||
}
|
||||
|
||||
return cloneResult.localPath;
|
||||
}
|
||||
|
||||
return this.getLocalPath(urlInfo);
|
||||
} else {
|
||||
OrchestratorLogger.log(`Repository not found locally, cloning...`);
|
||||
const cloneResult = await this.cloneRepository(urlInfo);
|
||||
|
||||
if (!cloneResult.success) {
|
||||
throw new Error(`Failed to clone repository: ${cloneResult.error}`);
|
||||
}
|
||||
|
||||
return cloneResult.localPath;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the path to the provider module within a repository
|
||||
* @param urlInfo GitHub URL information
|
||||
* @param localPath Local path to the repository
|
||||
* @returns Path to the provider module
|
||||
*/
|
||||
static getProviderModulePath(urlInfo: GitHubUrlInfo, localPath: string): string {
|
||||
if (urlInfo.path) {
|
||||
return path.join(localPath, urlInfo.path);
|
||||
}
|
||||
|
||||
// Look for common provider entry points
|
||||
const commonEntryPoints = [
|
||||
'index.js',
|
||||
'index.ts',
|
||||
'src/index.js',
|
||||
'src/index.ts',
|
||||
'lib/index.js',
|
||||
'lib/index.ts',
|
||||
'dist/index.js',
|
||||
'dist/index.js.map',
|
||||
];
|
||||
|
||||
for (const entryPoint of commonEntryPoints) {
|
||||
const fullPath = path.join(localPath, entryPoint);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
OrchestratorLogger.log(`Found provider entry point: ${entryPoint}`);
|
||||
|
||||
return fullPath;
|
||||
}
|
||||
}
|
||||
|
||||
// Default to repository root
|
||||
OrchestratorLogger.log(`No specific entry point found, using repository root`);
|
||||
|
||||
return localPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up old cached repositories (optional maintenance)
|
||||
* @param maxAgeDays Maximum age in days for cached repositories
|
||||
*/
|
||||
static async cleanupOldRepositories(maxAgeDays: number = 30): Promise<void> {
|
||||
this.ensureCacheDir();
|
||||
|
||||
try {
|
||||
const entries = fs.readdirSync(this.CACHE_DIR, { withFileTypes: true });
|
||||
const now = Date.now();
|
||||
const maxAge = maxAgeDays * 24 * 60 * 60 * 1000; // Convert to milliseconds
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const entryPath = path.join(this.CACHE_DIR, entry.name);
|
||||
const stats = fs.statSync(entryPath);
|
||||
|
||||
if (now - stats.mtime.getTime() > maxAge) {
|
||||
OrchestratorLogger.log(`Cleaning up old repository: ${entry.name}`);
|
||||
fs.rmSync(entryPath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`Error during cleanup: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import OrchestratorEnvironmentVariable from '../options/orchestrator-environment-variable';
|
||||
import OrchestratorSecret from '../options/orchestrator-secret';
|
||||
import { ProviderResource } from './provider-resource';
|
||||
import { ProviderWorkflow } from './provider-workflow';
|
||||
|
||||
export interface ProviderInterface {
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): any;
|
||||
setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): any;
|
||||
runTaskInWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
image: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
commands: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string>;
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string>;
|
||||
listResources(): Promise<ProviderResource[]>;
|
||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||
watchWorkflow(): Promise<string>;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { ProviderInterface } from './provider-interface';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser';
|
||||
import { ProviderGitManager } from './provider-git-manager';
|
||||
|
||||
// import path from 'path'; // Not currently used
|
||||
|
||||
/**
|
||||
* Dynamically load a provider package by name, URL, or path.
|
||||
* @param providerSource Provider source (name, URL, or path)
|
||||
* @param buildParameters Build parameters passed to the provider constructor
|
||||
* @throws Error when the provider cannot be loaded or does not implement ProviderInterface
|
||||
*/
|
||||
export default async function loadProvider(
|
||||
providerSource: string,
|
||||
buildParameters: BuildParameters,
|
||||
): Promise<ProviderInterface> {
|
||||
OrchestratorLogger.log(`Loading provider: ${providerSource}`);
|
||||
|
||||
// Parse the provider source to determine its type
|
||||
const sourceInfo = parseProviderSource(providerSource);
|
||||
logProviderSource(providerSource, sourceInfo);
|
||||
|
||||
let modulePath: string;
|
||||
let importedModule: any;
|
||||
|
||||
try {
|
||||
// Handle different source types
|
||||
switch (sourceInfo.type) {
|
||||
case 'github': {
|
||||
OrchestratorLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
|
||||
|
||||
// Ensure the repository is available locally
|
||||
const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
|
||||
|
||||
// Get the path to the provider module within the repository
|
||||
modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
|
||||
|
||||
OrchestratorLogger.log(`Loading provider from: ${modulePath}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'local': {
|
||||
modulePath = sourceInfo.path;
|
||||
OrchestratorLogger.log(`Loading provider from local path: ${modulePath}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'npm': {
|
||||
modulePath = sourceInfo.packageName;
|
||||
OrchestratorLogger.log(`Loading provider from NPM package: ${modulePath}`);
|
||||
break;
|
||||
}
|
||||
|
||||
default: {
|
||||
// Fallback to built-in providers or direct import
|
||||
const providerModuleMap: Record<string, string> = {
|
||||
aws: './aws',
|
||||
k8s: './k8s',
|
||||
test: './test',
|
||||
'local-docker': './docker',
|
||||
'local-system': './local',
|
||||
local: './local',
|
||||
};
|
||||
|
||||
modulePath = providerModuleMap[providerSource] || providerSource;
|
||||
OrchestratorLogger.log(`Loading provider from module path: ${modulePath}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Import the module
|
||||
importedModule = await import(modulePath);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load provider package '${providerSource}': ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
// Extract the provider class/function
|
||||
const Provider = importedModule.default || importedModule;
|
||||
|
||||
// Validate that we have a constructor
|
||||
if (typeof Provider !== 'function') {
|
||||
throw new TypeError(`Provider package '${providerSource}' does not export a constructor function`);
|
||||
}
|
||||
|
||||
// Instantiate the provider
|
||||
let instance: any;
|
||||
try {
|
||||
instance = new Provider(buildParameters);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to instantiate provider '${providerSource}': ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
// Validate that the instance implements the required interface
|
||||
const requiredMethods = [
|
||||
'cleanupWorkflow',
|
||||
'setupWorkflow',
|
||||
'runTaskInWorkflow',
|
||||
'garbageCollect',
|
||||
'listResources',
|
||||
'listWorkflow',
|
||||
'watchWorkflow',
|
||||
];
|
||||
|
||||
for (const method of requiredMethods) {
|
||||
if (typeof instance[method] !== 'function') {
|
||||
throw new TypeError(
|
||||
`Provider package '${providerSource}' does not implement ProviderInterface. Missing method '${method}'.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Successfully loaded provider: ${providerSource}`);
|
||||
|
||||
return instance as ProviderInterface;
|
||||
}
|
||||
|
||||
/**
|
||||
* ProviderLoader class for backward compatibility and additional utilities
|
||||
*/
|
||||
export class ProviderLoader {
|
||||
/**
|
||||
* Dynamically loads a provider by name, URL, or path (wrapper around loadProvider function)
|
||||
* @param providerSource - The provider source (name, URL, or path) to load
|
||||
* @param buildParameters - Build parameters to pass to the provider constructor
|
||||
* @returns Promise<ProviderInterface> - The loaded provider instance
|
||||
* @throws Error if provider package is missing or doesn't implement ProviderInterface
|
||||
*/
|
||||
static async loadProvider(providerSource: string, buildParameters: BuildParameters): Promise<ProviderInterface> {
|
||||
return loadProvider(providerSource, buildParameters);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a list of available provider names
|
||||
* @returns string[] - Array of available provider names
|
||||
*/
|
||||
static getAvailableProviders(): string[] {
|
||||
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up old cached repositories
|
||||
* @param maxAgeDays Maximum age in days for cached repositories (default: 30)
|
||||
*/
|
||||
static async cleanupCache(maxAgeDays: number = 30): Promise<void> {
|
||||
await ProviderGitManager.cleanupOldRepositories(maxAgeDays);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets information about a provider source without loading it
|
||||
* @param providerSource The provider source to analyze
|
||||
* @returns ProviderSourceInfo object with parsed details
|
||||
*/
|
||||
static analyzeProviderSource(providerSource: string): ProviderSourceInfo {
|
||||
return parseProviderSource(providerSource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export class ProviderResource {
|
||||
public Name!: string;
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
|
||||
export interface GitHubUrlInfo {
|
||||
type: 'github';
|
||||
owner: string;
|
||||
repo: string;
|
||||
branch?: string;
|
||||
path?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface LocalPathInfo {
|
||||
type: 'local';
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface NpmPackageInfo {
|
||||
type: 'npm';
|
||||
packageName: string;
|
||||
}
|
||||
|
||||
export type ProviderSourceInfo = GitHubUrlInfo | LocalPathInfo | NpmPackageInfo;
|
||||
|
||||
/**
|
||||
* Parses a provider source string and determines its type and details
|
||||
* @param source The provider source string (URL, path, or package name)
|
||||
* @returns ProviderSourceInfo object with parsed details
|
||||
*/
|
||||
export function parseProviderSource(source: string): ProviderSourceInfo {
|
||||
// Check if it's a GitHub URL
|
||||
const githubMatch = source.match(
|
||||
/^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/,
|
||||
);
|
||||
if (githubMatch) {
|
||||
const [, owner, repo, branch, path] = githubMatch;
|
||||
|
||||
return {
|
||||
type: 'github',
|
||||
owner,
|
||||
repo,
|
||||
branch: branch || 'main',
|
||||
path: path || '',
|
||||
url: `https://github.com/${owner}/${repo}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if it's a GitHub SSH URL
|
||||
const githubSshMatch = source.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?\/?(?:tree\/([^/]+))?(?:\/(.+))?$/);
|
||||
if (githubSshMatch) {
|
||||
const [, owner, repo, branch, path] = githubSshMatch;
|
||||
|
||||
return {
|
||||
type: 'github',
|
||||
owner,
|
||||
repo,
|
||||
branch: branch || 'main',
|
||||
path: path || '',
|
||||
url: `https://github.com/${owner}/${repo}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if it's a shorthand GitHub reference (owner/repo)
|
||||
const shorthandMatch = source.match(/^([^/@]+)\/([^/@]+)(?:@([^/]+))?(?:\/(.+))?$/);
|
||||
if (shorthandMatch && !source.startsWith('.') && !source.startsWith('/') && !source.includes('\\')) {
|
||||
const [, owner, repo, branch, path] = shorthandMatch;
|
||||
|
||||
return {
|
||||
type: 'github',
|
||||
owner,
|
||||
repo,
|
||||
branch: branch || 'main',
|
||||
path: path || '',
|
||||
url: `https://github.com/${owner}/${repo}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if it's a local path
|
||||
if (source.startsWith('./') || source.startsWith('../') || source.startsWith('/') || source.includes('\\')) {
|
||||
return {
|
||||
type: 'local',
|
||||
path: source,
|
||||
};
|
||||
}
|
||||
|
||||
// Default to npm package
|
||||
return {
|
||||
type: 'npm',
|
||||
packageName: source,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a cache key for a GitHub repository
|
||||
* @param urlInfo GitHub URL information
|
||||
* @returns Cache key string
|
||||
*/
|
||||
export function generateCacheKey(urlInfo: GitHubUrlInfo): string {
|
||||
return `github_${urlInfo.owner}_${urlInfo.repo}_${urlInfo.branch}`.replace(/[^\w-]/g, '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates if a string looks like a valid GitHub URL or reference
|
||||
* @param source The source string to validate
|
||||
* @returns True if it looks like a GitHub reference
|
||||
*/
|
||||
export function isGitHubSource(source: string): boolean {
|
||||
const parsed = parseProviderSource(source);
|
||||
|
||||
return parsed.type === 'github';
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs the parsed provider source information
|
||||
* @param source The original source string
|
||||
* @param parsed The parsed source information
|
||||
*/
|
||||
export function logProviderSource(source: string, parsed: ProviderSourceInfo): void {
|
||||
OrchestratorLogger.log(`Provider source: ${source}`);
|
||||
switch (parsed.type) {
|
||||
case 'github':
|
||||
OrchestratorLogger.log(` Type: GitHub repository`);
|
||||
OrchestratorLogger.log(` Owner: ${parsed.owner}`);
|
||||
OrchestratorLogger.log(` Repository: ${parsed.repo}`);
|
||||
OrchestratorLogger.log(` Branch: ${parsed.branch}`);
|
||||
if (parsed.path) {
|
||||
OrchestratorLogger.log(` Path: ${parsed.path}`);
|
||||
}
|
||||
break;
|
||||
case 'local':
|
||||
OrchestratorLogger.log(` Type: Local path`);
|
||||
OrchestratorLogger.log(` Path: ${parsed.path}`);
|
||||
break;
|
||||
case 'npm':
|
||||
OrchestratorLogger.log(` Type: NPM package`);
|
||||
OrchestratorLogger.log(` Package: ${parsed.packageName}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export class ProviderWorkflow {
|
||||
public Name!: string;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
class TestOrchestrator implements ProviderInterface {
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTaskInWorkflow(
|
||||
commands: string,
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(image);
|
||||
OrchestratorLogger.log(buildGuid);
|
||||
OrchestratorLogger.log(commands);
|
||||
|
||||
return await new Promise((result) => {
|
||||
result(commands);
|
||||
});
|
||||
}
|
||||
}
|
||||
export default TestOrchestrator;
|
||||
Reference in New Issue
Block a user