mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-11 16:34:03 -07:00
Rename Cloud Runner to Orchestrator (#775)
* Rename "Cloud Runner" to "Orchestrator" across entire codebase Breaking change: All CloudRunner classes, options, environment variables, and action.yml inputs have been renamed to Orchestrator equivalents. - Renamed src/model/cloud-runner/ directory to src/model/orchestrator/ - Renamed all cloud-runner-* files to orchestrator-* - Renamed all CloudRunner* classes to Orchestrator* (15+ classes) - Renamed all cloudRunner* properties to orchestrator* equivalents - Renamed CLOUD_RUNNER_* env vars to ORCHESTRATOR_* - Updated action.yml [CloudRunner] markers to [Orchestrator] - Updated workflow files and package.json test scripts - Updated all runtime strings (cache paths, log messages, branch refs) - Rebuilt dist/index.js No backward compatibility layer is provided. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Remove tracked log/temp files and add to .gitignore Remove $LOG_FILE and temp/job-log.txt debug artifacts that should not be in the repository. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frostebite
@@ -0,0 +1,118 @@
|
||||
import { BuildParameters, Input } from '../../..';
|
||||
import YAML from 'yaml';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import * as fs from 'node:fs';
|
||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||
import { CommandHook } from './command-hook';
|
||||
|
||||
// import OrchestratorLogger from './orchestrator-logger';
|
||||
|
||||
export class CommandHookService {
|
||||
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
|
||||
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
|
||||
OrchestratorLogger.log(`Applying hooks ${hooks.length}`);
|
||||
|
||||
return `echo "---"
|
||||
echo "start orchestrator init"
|
||||
${OrchestratorOptions.orchestratorDebug ? `printenv` : `#`}
|
||||
echo "start of orchestrator job"
|
||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${commands}
|
||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
echo "end of orchestrator job"
|
||||
echo "---${buildParameters.logId}"`;
|
||||
}
|
||||
|
||||
public static getHooks(customCommandHooks: string): CommandHook[] {
|
||||
const experimentHooks = customCommandHooks;
|
||||
let output = new Array<CommandHook>();
|
||||
if (experimentHooks && experimentHooks !== '') {
|
||||
try {
|
||||
output = YAML.parse(experimentHooks);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
|
||||
...CommandHookService.GetCustomHooksFromFiles(`before`),
|
||||
...CommandHookService.GetCustomHooksFromFiles(`after`),
|
||||
];
|
||||
}
|
||||
|
||||
static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
|
||||
const results: CommandHook[] = [];
|
||||
|
||||
// RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
|
||||
try {
|
||||
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
if (!OrchestratorOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||
continue;
|
||||
}
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
|
||||
if (fileContentsObject.hook.includes(hookLifecycle)) {
|
||||
results.push(fileContentsObject);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
||||
}
|
||||
|
||||
// RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static ConvertYamlSecrets(object: CommandHook) {
|
||||
if (object.secrets === undefined) {
|
||||
object.secrets = [];
|
||||
|
||||
return;
|
||||
}
|
||||
object.secrets = object.secrets.map((x: any) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
public static ParseHooks(hooks: string): CommandHook[] {
|
||||
if (hooks === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
// if (Orchestrator.buildParameters?.orchestratorIntegrationTests) {
|
||||
|
||||
// OrchestratorLogger.log(`Parsing build hooks: ${steps}`);
|
||||
|
||||
// }
|
||||
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
|
||||
const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
|
||||
for (const hook of object) {
|
||||
CommandHookService.ConvertYamlSecrets(hook);
|
||||
if (hook.secrets === undefined) {
|
||||
hook.secrets = [];
|
||||
}
|
||||
}
|
||||
if (object === undefined) {
|
||||
throw new Error(`Failed to parse ${hooks}`);
|
||||
}
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
public static getSecrets(hooks: any) {
|
||||
const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
|
||||
|
||||
// eslint-disable-next-line unicorn/no-array-reduce
|
||||
return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
|
||||
export class CommandHook {
|
||||
public commands: string[] = new Array<string>();
|
||||
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
|
||||
public name!: string;
|
||||
public hook!: string[];
|
||||
public step!: string[];
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
import YAML from 'yaml';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CustomWorkflow } from '../../workflows/custom-workflow';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import Input from '../../../input';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { ContainerHook as ContainerHook } from './container-hook';
|
||||
import { OrchestratorStepParameters } from '../../options/orchestrator-step-parameters';
|
||||
|
||||
export class ContainerHookService {
|
||||
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
|
||||
const results: ContainerHook[] = [];
|
||||
try {
|
||||
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomStepsPath);
|
||||
for (const file of files) {
|
||||
if (!OrchestratorOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
|
||||
continue;
|
||||
}
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
|
||||
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
results.push(fileContentsObject);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
||||
}
|
||||
|
||||
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
|
||||
|
||||
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
|
||||
`- name: aws-s3-upload-build
|
||||
image: amazon/aws-cli
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} s3://${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-upload-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: awsSecretAccessKey
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: awsDefaultRegion
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||
- name: aws-s3-pull-build
|
||||
image: amazon/aws-cli
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ || true
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build || true
|
||||
aws s3 cp s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-pull-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: AWS_S3_ENDPOINT
|
||||
- name: steam-deploy-client
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: RELEASE_BRANCH
|
||||
- name: steam-deploy-project
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_2
|
||||
- name: RELEASE_BRANCH
|
||||
- name: aws-s3-upload-cache
|
||||
image: amazon/aws-cli
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-upload-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||
- name: aws-s3-pull-cache
|
||||
image: amazon/aws-cli
|
||||
hook: before
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ 2>/dev/null || true
|
||||
aws $ENDPOINT_ARGS s3 ls ${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/ 2>/dev/null || true
|
||||
BUCKET1="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/Library/"
|
||||
OBJECT1=""
|
||||
LS_OUTPUT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 2>/dev/null || echo '')"
|
||||
if [ -n "$LS_OUTPUT1" ] && [ "$LS_OUTPUT1" != "" ]; then
|
||||
OBJECT1="$(echo "$LS_OUTPUT1" | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
if [ -n "$OBJECT1" ] && [ "$OBJECT1" != "" ]; then
|
||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
BUCKET2="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/lfs/"
|
||||
OBJECT2=""
|
||||
LS_OUTPUT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 2>/dev/null || echo '')"
|
||||
if [ -n "$LS_OUTPUT2" ] && [ "$LS_OUTPUT2" != "" ]; then
|
||||
OBJECT2="$(echo "$LS_OUTPUT2" | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
if [ -n "$OBJECT2" ] && [ "$OBJECT2" != "" ]; then
|
||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-pull-cache"
|
||||
fi
|
||||
- name: rclone-upload-build
|
||||
image: rclone/rclone
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} ${Orchestrator.buildParameters.rcloneRemote}/orchestrator-cache/$CACHE_KEY/build/ || true
|
||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-upload-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-pull-build
|
||||
image: rclone/rclone
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-pull-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-upload-cache
|
||||
image: rclone/rclone
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy /data/cache/$CACHE_KEY/lfs ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||
rclone copy /data/cache/$CACHE_KEY/Library ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-upload-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-pull-cache
|
||||
image: rclone/rclone
|
||||
hook: before
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-pull-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: debug-cache
|
||||
image: ubuntu
|
||||
hook: after
|
||||
commands: |
|
||||
apt-get update > /dev/null || true
|
||||
${OrchestratorOptions.orchestratorDebug ? `apt-get install -y tree > /dev/null || true` : `#`}
|
||||
${OrchestratorOptions.orchestratorDebug ? `tree -L 3 /data/cache || true` : `#`}
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: awsSecretAccessKey
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: awsDefaultRegion
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`,
|
||||
).filter((x) => OrchestratorOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
|
||||
|
||||
// In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks
|
||||
const provider = Orchestrator.buildParameters?.providerStrategy;
|
||||
const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker';
|
||||
const hasAwsCreds =
|
||||
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
(process.env.awsAccessKeyId && process.env.awsSecretAccessKey);
|
||||
|
||||
// Always include AWS hooks on the AWS provider (task role provides creds),
|
||||
// otherwise require explicit creds for other containerized providers.
|
||||
const shouldIncludeAwsHooks =
|
||||
isContainerized && !Orchestrator.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds));
|
||||
const filteredBuiltIns = shouldIncludeAwsHooks
|
||||
? builtInContainerHooks
|
||||
: builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli');
|
||||
|
||||
if (filteredBuiltIns.length > 0) {
|
||||
results.push(...filteredBuiltIns);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static ConvertYamlSecrets(object: ContainerHook) {
|
||||
if (object.secrets === undefined) {
|
||||
object.secrets = [];
|
||||
|
||||
return;
|
||||
}
|
||||
object.secrets = object.secrets.map((x: { [key: string]: any }) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
public static ParseContainerHooks(steps: string): ContainerHook[] {
|
||||
if (steps === '') {
|
||||
return [];
|
||||
}
|
||||
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
|
||||
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
|
||||
for (const step of object) {
|
||||
ContainerHookService.ConvertYamlSecrets(step);
|
||||
if (step.secrets === undefined) {
|
||||
step.secrets = [];
|
||||
} else {
|
||||
for (const secret of step.secrets) {
|
||||
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
||||
if (Orchestrator.buildParameters?.orchestratorDebug) {
|
||||
// OrchestratorLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
||||
}
|
||||
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (step.image === undefined) {
|
||||
step.image = `ubuntu`;
|
||||
}
|
||||
|
||||
// Ensure allowFailure defaults to false if not explicitly set
|
||||
if (step.allowFailure === undefined) {
|
||||
step.allowFailure = false;
|
||||
}
|
||||
}
|
||||
if (object === undefined) {
|
||||
throw new Error(`Failed to parse ${steps}`);
|
||||
}
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
static async RunPostBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
||||
let output = ``;
|
||||
const steps: ContainerHook[] = [
|
||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.postBuildContainerHooks),
|
||||
...ContainerHookService.GetContainerHooksFromFiles(`after`),
|
||||
];
|
||||
|
||||
if (steps.length > 0) {
|
||||
output += await CustomWorkflow.runContainerJob(
|
||||
steps,
|
||||
orchestratorStepState.environment,
|
||||
orchestratorStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
static async RunPreBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
||||
let output = ``;
|
||||
const steps: ContainerHook[] = [
|
||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.preBuildContainerHooks),
|
||||
...ContainerHookService.GetContainerHooksFromFiles(`before`),
|
||||
];
|
||||
|
||||
if (steps.length > 0) {
|
||||
output += await CustomWorkflow.runContainerJob(
|
||||
steps,
|
||||
orchestratorStepState.environment,
|
||||
orchestratorStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
|
||||
export class ContainerHook {
|
||||
public commands!: string;
|
||||
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
|
||||
public name!: string;
|
||||
public image: string = `ubuntu`;
|
||||
public hook!: string;
|
||||
public allowFailure: boolean = false; // If true, hook failures won't stop the build
|
||||
}
|
||||
Reference in New Issue
Block a user