mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-10 16:03:54 -07:00
Rename Cloud Runner to Orchestrator (#775)
* Rename "Cloud Runner" to "Orchestrator" across entire codebase Breaking change: All CloudRunner classes, options, environment variables, and action.yml inputs have been renamed to Orchestrator equivalents. - Renamed src/model/cloud-runner/ directory to src/model/orchestrator/ - Renamed all cloud-runner-* files to orchestrator-* - Renamed all CloudRunner* classes to Orchestrator* (15+ classes) - Renamed all cloudRunner* properties to orchestrator* equivalents - Renamed CLOUD_RUNNER_* env vars to ORCHESTRATOR_* - Updated action.yml [CloudRunner] markers to [Orchestrator] - Updated workflow files and package.json test scripts - Updated all runtime strings (cache paths, log messages, branch refs) - Rebuilt dist/index.js No backward compatibility layer is provided. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Remove tracked log/temp files and add to .gitignore Remove $LOG_FILE and temp/job-log.txt debug artifacts that should not be in the repository. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frostebite
@@ -0,0 +1,57 @@
|
||||
import GitHub from '../../../github';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { OrchestratorStatics } from '../../options/orchestrator-statics';
|
||||
import OrchestratorLogger from './orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
export class FollowLogStreamService {
|
||||
static Reset() {
|
||||
FollowLogStreamService.DidReceiveEndOfTransmission = false;
|
||||
}
|
||||
static errors = ``;
|
||||
public static DidReceiveEndOfTransmission = false;
|
||||
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
|
||||
if (message.includes(`---${Orchestrator.buildParameters.logId}`)) {
|
||||
OrchestratorLogger.log('End of log transmission received');
|
||||
FollowLogStreamService.DidReceiveEndOfTransmission = true;
|
||||
shouldReadLogs = false;
|
||||
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
|
||||
GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
|
||||
core.warning('LIBRARY NOT FOUND!');
|
||||
core.setOutput('library-found', 'false');
|
||||
} else if (message.includes('Build succeeded')) {
|
||||
GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
|
||||
core.setOutput('build-result', 'success');
|
||||
} else if (message.includes('Build fail')) {
|
||||
GitHub.updateGitHubCheck(
|
||||
`Build failed\n${FollowLogStreamService.errors}`,
|
||||
`Build failed`,
|
||||
`failure`,
|
||||
`completed`,
|
||||
);
|
||||
core.setOutput('build-result', 'failed');
|
||||
core.setFailed('unity build failed');
|
||||
core.error('BUILD FAILED!');
|
||||
} else if (message.toLowerCase().includes('error ')) {
|
||||
core.error(message);
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
} else if (message.toLowerCase().includes('error: ')) {
|
||||
core.error(message);
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
} else if (message.toLowerCase().includes('command failed: ')) {
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
} else if (message.toLowerCase().includes('invalid ')) {
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
} else if (message.toLowerCase().includes('incompatible ')) {
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
} else if (message.toLowerCase().includes('cannot be found')) {
|
||||
FollowLogStreamService.errors += `\n${message}`;
|
||||
}
|
||||
|
||||
// Always append log lines to output so tests can assert on BuildResults
|
||||
output += `${message}\n`;
|
||||
OrchestratorLogger.log(`[${OrchestratorStatics.logPrefix}] ${message}`);
|
||||
|
||||
return { shouldReadLogs, shouldCleanup, output };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import * as core from '@actions/core';
|
||||
|
||||
class OrchestratorLogger {
|
||||
private static timestamp: number;
|
||||
private static globalTimestamp: number;
|
||||
|
||||
public static setup() {
|
||||
this.timestamp = this.createTimestamp();
|
||||
this.globalTimestamp = this.timestamp;
|
||||
}
|
||||
|
||||
public static log(message: string) {
|
||||
core.info(message);
|
||||
}
|
||||
|
||||
public static logWarning(message: string) {
|
||||
core.warning(message);
|
||||
}
|
||||
|
||||
public static logLine(message: string) {
|
||||
core.info(`${message}\n`);
|
||||
}
|
||||
|
||||
public static error(message: string) {
|
||||
core.error(message);
|
||||
}
|
||||
|
||||
public static logWithTime(message: string) {
|
||||
const newTimestamp = this.createTimestamp();
|
||||
core.info(
|
||||
`${message} (Since previous: ${this.calculateTimeDiff(
|
||||
newTimestamp,
|
||||
this.timestamp,
|
||||
)}, Total time: ${this.calculateTimeDiff(newTimestamp, this.globalTimestamp)})`,
|
||||
);
|
||||
this.timestamp = newTimestamp;
|
||||
}
|
||||
|
||||
private static calculateTimeDiff(x: number, y: number) {
|
||||
return Math.floor((x - y) / 1000);
|
||||
}
|
||||
|
||||
private static createTimestamp() {
|
||||
return Date.now();
|
||||
}
|
||||
}
|
||||
export default OrchestratorLogger;
|
||||
@@ -0,0 +1,24 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
|
||||
class OrchestratorResult {
|
||||
public BuildParameters: BuildParameters;
|
||||
public BuildResults: string;
|
||||
public BuildSucceeded: boolean;
|
||||
public BuildFinished: boolean;
|
||||
public LibraryCacheUsed: boolean;
|
||||
|
||||
public constructor(
|
||||
buildParameters: BuildParameters,
|
||||
buildResults: string,
|
||||
buildSucceeded: boolean,
|
||||
buildFinished: boolean,
|
||||
libraryCacheUsed: boolean,
|
||||
) {
|
||||
this.BuildParameters = buildParameters;
|
||||
this.BuildResults = buildResults;
|
||||
this.BuildSucceeded = buildSucceeded;
|
||||
this.BuildFinished = buildFinished;
|
||||
this.LibraryCacheUsed = libraryCacheUsed;
|
||||
}
|
||||
}
|
||||
export default OrchestratorResult;
|
||||
@@ -0,0 +1,69 @@
|
||||
import { exec } from 'child_process';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
|
||||
export class OrchestratorSystem {
|
||||
public static async RunAndReadLines(command: string): Promise<string[]> {
|
||||
const result = await OrchestratorSystem.Run(command, false, true);
|
||||
|
||||
return result
|
||||
.split(`\n`)
|
||||
.map((x) => x.replace(`\r`, ``))
|
||||
.filter((x) => x !== ``)
|
||||
.map((x) => {
|
||||
const lineValues = x.split(` `);
|
||||
|
||||
return lineValues[lineValues.length - 1];
|
||||
});
|
||||
}
|
||||
|
||||
public static async Run(
|
||||
command: string,
|
||||
suppressError = false,
|
||||
suppressLogs = false,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
outputCallback?: (output: string) => void,
|
||||
) {
|
||||
for (const element of command.split(`\n`)) {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
}
|
||||
|
||||
return await new Promise<string>((promise, throwError) => {
|
||||
let output = '';
|
||||
const child = exec(command, { maxBuffer: 1024 * 10000 }, (error, stdout, stderr) => {
|
||||
if (!suppressError && error) {
|
||||
RemoteClientLogger.log(error.toString());
|
||||
throwError(error);
|
||||
}
|
||||
if (stderr) {
|
||||
const diagnosticOutput = `${stderr.toString()}`;
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
|
||||
}
|
||||
output += diagnosticOutput;
|
||||
}
|
||||
const outputChunk = `${stdout}`;
|
||||
if (outputCallback) {
|
||||
outputCallback(outputChunk);
|
||||
}
|
||||
output += outputChunk;
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(`[${code}]`);
|
||||
}
|
||||
if (code !== 0 && !suppressError) {
|
||||
throwError(output);
|
||||
}
|
||||
const outputLines = output.split(`\n`);
|
||||
for (const element of outputLines) {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
}
|
||||
promise(output);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import OrchestratorLogger from './orchestrator-logger';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { OrchestratorSystem } from './orchestrator-system';
|
||||
|
||||
class ResourceTracking {
|
||||
static isEnabled(): boolean {
|
||||
return (
|
||||
OrchestratorOptions.resourceTracking ||
|
||||
OrchestratorOptions.orchestratorDebug ||
|
||||
process.env['orchestratorTests'] === 'true'
|
||||
);
|
||||
}
|
||||
|
||||
static logAllocationSummary(context: string) {
|
||||
if (!ResourceTracking.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const buildParameters = Orchestrator.buildParameters;
|
||||
const allocations = {
|
||||
providerStrategy: buildParameters.providerStrategy,
|
||||
containerCpu: buildParameters.containerCpu,
|
||||
containerMemory: buildParameters.containerMemory,
|
||||
dockerCpuLimit: buildParameters.dockerCpuLimit,
|
||||
dockerMemoryLimit: buildParameters.dockerMemoryLimit,
|
||||
kubeVolumeSize: buildParameters.kubeVolumeSize,
|
||||
kubeStorageClass: buildParameters.kubeStorageClass,
|
||||
kubeVolume: buildParameters.kubeVolume,
|
||||
containerNamespace: buildParameters.containerNamespace,
|
||||
storageProvider: buildParameters.storageProvider,
|
||||
rcloneRemote: buildParameters.rcloneRemote,
|
||||
dockerWorkspacePath: buildParameters.dockerWorkspacePath,
|
||||
cacheKey: buildParameters.cacheKey,
|
||||
maxRetainedWorkspaces: buildParameters.maxRetainedWorkspaces,
|
||||
useCompressionStrategy: buildParameters.useCompressionStrategy,
|
||||
useLargePackages: buildParameters.useLargePackages,
|
||||
ephemeralStorageRequest: process.env['orchestratorTests'] === 'true' ? 'not set' : '2Gi',
|
||||
};
|
||||
|
||||
OrchestratorLogger.log(`[ResourceTracking] Allocation summary (${context}):`);
|
||||
OrchestratorLogger.log(JSON.stringify(allocations, undefined, 2));
|
||||
}
|
||||
|
||||
static async logDiskUsageSnapshot(context: string) {
|
||||
if (!ResourceTracking.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[ResourceTracking] Disk usage snapshot (${context})`);
|
||||
await ResourceTracking.runAndLog('df -h', 'df -h');
|
||||
await ResourceTracking.runAndLog('du -sh .', 'du -sh .');
|
||||
await ResourceTracking.runAndLog('du -sh ./orchestrator-cache', 'du -sh ./orchestrator-cache');
|
||||
await ResourceTracking.runAndLog('du -sh ./temp', 'du -sh ./temp');
|
||||
await ResourceTracking.runAndLog('du -sh ./logs', 'du -sh ./logs');
|
||||
}
|
||||
|
||||
static async logK3dNodeDiskUsage(context: string) {
|
||||
if (!ResourceTracking.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nodes = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
|
||||
OrchestratorLogger.log(`[ResourceTracking] K3d node disk usage (${context})`);
|
||||
for (const node of nodes) {
|
||||
await ResourceTracking.runAndLog(
|
||||
`k3d node ${node}`,
|
||||
`docker exec ${node} sh -c "df -h /var/lib/rancher/k3s 2>/dev/null || df -h / 2>/dev/null || true" || true`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static async runAndLog(label: string, command: string) {
|
||||
try {
|
||||
const output = await OrchestratorSystem.Run(command, true, true);
|
||||
const trimmed = output.trim();
|
||||
OrchestratorLogger.log(`[ResourceTracking] ${label}:\n${trimmed || 'no output'}`);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`[ResourceTracking] ${label} failed: ${error?.message || error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default ResourceTracking;
|
||||
@@ -0,0 +1,402 @@
|
||||
import OrchestratorLogger from './orchestrator-logger';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import Input from '../../../input';
|
||||
import {
|
||||
CreateBucketCommand,
|
||||
DeleteObjectCommand,
|
||||
HeadBucketCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { AwsClientFactory } from '../../providers/aws/aws-client-factory';
|
||||
import { promisify } from 'node:util';
|
||||
import { exec as execCallback } from 'node:child_process';
|
||||
const exec = promisify(execCallback);
|
||||
export class SharedWorkspaceLocking {
|
||||
private static _s3: S3;
|
||||
private static get s3(): S3 {
|
||||
if (!SharedWorkspaceLocking._s3) {
|
||||
// Use factory so LocalStack endpoint/path-style settings are honored
|
||||
SharedWorkspaceLocking._s3 = AwsClientFactory.getS3();
|
||||
}
|
||||
|
||||
return SharedWorkspaceLocking._s3;
|
||||
}
|
||||
private static get useRclone() {
|
||||
return Orchestrator.buildParameters.storageProvider === 'rclone';
|
||||
}
|
||||
private static async rclone(command: string): Promise<string> {
|
||||
const { stdout } = await exec(`rclone ${command}`);
|
||||
|
||||
return stdout.toString();
|
||||
}
|
||||
private static get bucket() {
|
||||
return SharedWorkspaceLocking.useRclone
|
||||
? Orchestrator.buildParameters.rcloneRemote
|
||||
: Orchestrator.buildParameters.awsStackName;
|
||||
}
|
||||
public static get workspaceBucketRoot() {
|
||||
return SharedWorkspaceLocking.useRclone
|
||||
? `${SharedWorkspaceLocking.bucket}/`
|
||||
: `s3://${SharedWorkspaceLocking.bucket}/`;
|
||||
}
|
||||
public static get workspaceRoot() {
|
||||
return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
|
||||
}
|
||||
private static get workspacePrefix() {
|
||||
return `locks/`;
|
||||
}
|
||||
private static async ensureBucketExists(): Promise<void> {
|
||||
const bucket = SharedWorkspaceLocking.bucket;
|
||||
if (SharedWorkspaceLocking.useRclone) {
|
||||
try {
|
||||
await SharedWorkspaceLocking.rclone(`lsf ${bucket}`);
|
||||
} catch {
|
||||
await SharedWorkspaceLocking.rclone(`mkdir ${bucket}`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await SharedWorkspaceLocking.s3.send(new HeadBucketCommand({ Bucket: bucket }));
|
||||
} catch {
|
||||
const region = Input.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1';
|
||||
const createParameters: any = { Bucket: bucket };
|
||||
if (region && region !== 'us-east-1') {
|
||||
createParameters.CreateBucketConfiguration = { LocationConstraint: region };
|
||||
}
|
||||
await SharedWorkspaceLocking.s3.send(new CreateBucketCommand(createParameters));
|
||||
}
|
||||
}
|
||||
private static async listObjects(prefix: string, bucket = SharedWorkspaceLocking.bucket): Promise<string[]> {
|
||||
await SharedWorkspaceLocking.ensureBucketExists();
|
||||
if (prefix !== '' && !prefix.endsWith('/')) {
|
||||
prefix += '/';
|
||||
}
|
||||
if (SharedWorkspaceLocking.useRclone) {
|
||||
const path = `${bucket}/${prefix}`;
|
||||
try {
|
||||
const output = await SharedWorkspaceLocking.rclone(`lsjson ${path}`);
|
||||
const json = JSON.parse(output) as { Name: string; IsDir: boolean }[];
|
||||
|
||||
return json.map((entry) => (entry.IsDir ? `${entry.Name}/` : entry.Name));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
const result = await SharedWorkspaceLocking.s3.send(
|
||||
new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, Delimiter: '/' }),
|
||||
);
|
||||
const entries: string[] = [];
|
||||
for (const p of result.CommonPrefixes || []) {
|
||||
if (p.Prefix) entries.push(p.Prefix.slice(prefix.length));
|
||||
}
|
||||
for (const c of result.Contents || []) {
|
||||
if (c.Key && c.Key !== prefix) entries.push(c.Key.slice(prefix.length));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
||||
if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return (
|
||||
await SharedWorkspaceLocking.listObjects(
|
||||
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
|
||||
)
|
||||
)
|
||||
.map((x) => x.replace(`/`, ``))
|
||||
.filter((x) => x.endsWith(`_workspace`))
|
||||
.map((x) => x.split(`_`)[1]);
|
||||
}
|
||||
public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
|
||||
try {
|
||||
const rootLines = await SharedWorkspaceLocking.listObjects('');
|
||||
const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
|
||||
|
||||
if (lockFolderExists) {
|
||||
const lines = await SharedWorkspaceLocking.listObjects(SharedWorkspaceLocking.workspacePrefix);
|
||||
|
||||
return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static NewWorkspaceName() {
|
||||
return `${Orchestrator.retainedWorkspacePrefix}-${Orchestrator.buildParameters.buildGuid}`;
|
||||
}
|
||||
public static async GetAllLocksForWorkspace(
|
||||
workspace: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<string[]> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return (
|
||||
await SharedWorkspaceLocking.listObjects(
|
||||
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
|
||||
)
|
||||
)
|
||||
.map((x) => x.replace(`/`, ``))
|
||||
.filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
|
||||
}
|
||||
public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
|
||||
if (buildParametersContext.maxRetainedWorkspaces === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
|
||||
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
||||
OrchestratorLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
|
||||
for (const element of workspaces) {
|
||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
|
||||
OrchestratorLogger.log(
|
||||
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
|
||||
);
|
||||
|
||||
if (lockResult) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
||||
workspace = SharedWorkspaceLocking.NewWorkspaceName();
|
||||
Orchestrator.lockedWorkspace = workspace;
|
||||
}
|
||||
|
||||
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
|
||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
|
||||
OrchestratorLogger.log(
|
||||
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
|
||||
);
|
||||
|
||||
return createResult && lockResult;
|
||||
}
|
||||
|
||||
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
|
||||
return (
|
||||
(await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
|
||||
.length > 0
|
||||
);
|
||||
}
|
||||
public static async HasWorkspaceLock(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
|
||||
.map((x) => {
|
||||
return {
|
||||
name: x,
|
||||
timestamp: Number(x.split(`_`)[0]),
|
||||
};
|
||||
})
|
||||
.sort((x) => x.timestamp);
|
||||
const lockMatches = locks.filter((x) => x.name.includes(runId));
|
||||
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
|
||||
OrchestratorLogger.log(
|
||||
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
|
||||
lockMatches.length
|
||||
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
|
||||
);
|
||||
|
||||
return includesRunLock;
|
||||
}
|
||||
|
||||
public static async GetFreeWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
||||
const result: string[] = [];
|
||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||
for (const element of workspaces) {
|
||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
|
||||
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
|
||||
OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
||||
if (!isLocked && isBelowMax) {
|
||||
result.push(element);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static async IsWorkspaceBelowMax(
|
||||
workspace: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||
if (workspace === ``) {
|
||||
return (
|
||||
workspaces.length < buildParametersContext.maxRetainedWorkspaces ||
|
||||
buildParametersContext.maxRetainedWorkspaces === 0
|
||||
);
|
||||
}
|
||||
const ordered: any[] = [];
|
||||
for (const ws of workspaces) {
|
||||
ordered.push({
|
||||
name: ws,
|
||||
timestamp: await SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
|
||||
});
|
||||
}
|
||||
ordered.sort((x) => x.timestamp);
|
||||
const matches = ordered.filter((x) => x.name.includes(workspace));
|
||||
const isWorkspaceBelowMax =
|
||||
matches.length > 0 &&
|
||||
(ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces ||
|
||||
buildParametersContext.maxRetainedWorkspaces === 0);
|
||||
|
||||
return isWorkspaceBelowMax;
|
||||
}
|
||||
|
||||
public static async GetWorkspaceTimestamp(
|
||||
workspace: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<Number> {
|
||||
if (workspace.split(`_`).length > 0) {
|
||||
return Number(workspace.split(`_`)[1]);
|
||||
}
|
||||
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
||||
}
|
||||
|
||||
return (
|
||||
await SharedWorkspaceLocking.listObjects(
|
||||
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
|
||||
)
|
||||
)
|
||||
.map((x) => x.replace(`/`, ``))
|
||||
.filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
|
||||
.map((x) => Number(x))[0];
|
||||
}
|
||||
|
||||
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
throw new Error(`workspace doesn't exist ${workspace}`);
|
||||
}
|
||||
const files = await SharedWorkspaceLocking.listObjects(
|
||||
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
|
||||
);
|
||||
|
||||
const lockFilesExist =
|
||||
files.filter((x) => {
|
||||
return x.includes(workspace) && x.endsWith(`_lock`);
|
||||
}).length > 0;
|
||||
|
||||
return lockFilesExist;
|
||||
}
|
||||
|
||||
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
|
||||
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
||||
throw new Error(`${workspace} already exists`);
|
||||
}
|
||||
const timestamp = Date.now();
|
||||
const key = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${timestamp}_${workspace}_workspace`;
|
||||
await SharedWorkspaceLocking.ensureBucketExists();
|
||||
await (SharedWorkspaceLocking.useRclone
|
||||
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
|
||||
: SharedWorkspaceLocking.s3.send(
|
||||
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
||||
));
|
||||
|
||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||
|
||||
OrchestratorLogger.log(`All workspaces ${workspaces}`);
|
||||
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
|
||||
OrchestratorLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
|
||||
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public static async LockWorkspace(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
const existingWorkspace = workspace.endsWith(`_workspace`);
|
||||
const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
|
||||
const key = `${SharedWorkspaceLocking.workspacePrefix}${
|
||||
buildParametersContext.cacheKey
|
||||
}/${Date.now()}_${runId}_${ending}_lock`;
|
||||
await SharedWorkspaceLocking.ensureBucketExists();
|
||||
await (SharedWorkspaceLocking.useRclone
|
||||
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
|
||||
: SharedWorkspaceLocking.s3.send(
|
||||
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
||||
));
|
||||
|
||||
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||
|
||||
if (hasLock) {
|
||||
Orchestrator.lockedWorkspace = workspace;
|
||||
} else {
|
||||
await (SharedWorkspaceLocking.useRclone
|
||||
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`)
|
||||
: SharedWorkspaceLocking.s3.send(new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key })));
|
||||
}
|
||||
|
||||
return hasLock;
|
||||
}
|
||||
|
||||
public static async ReleaseWorkspace(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
await SharedWorkspaceLocking.ensureBucketExists();
|
||||
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
|
||||
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
|
||||
OrchestratorLogger.log(`All Locks ${files} ${workspace} ${runId}`);
|
||||
OrchestratorLogger.log(`Deleting lock ${workspace}/${file}`);
|
||||
OrchestratorLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
|
||||
if (file) {
|
||||
await (SharedWorkspaceLocking.useRclone
|
||||
? SharedWorkspaceLocking.rclone(
|
||||
`delete ${SharedWorkspaceLocking.bucket}/${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
||||
)
|
||||
: SharedWorkspaceLocking.s3.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: SharedWorkspaceLocking.bucket,
|
||||
Key: `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
|
||||
}
|
||||
|
||||
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
|
||||
const prefix = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`;
|
||||
const files = await SharedWorkspaceLocking.listObjects(prefix);
|
||||
for (const file of files.filter((x) => x.includes(`_${workspace}_`))) {
|
||||
await (SharedWorkspaceLocking.useRclone
|
||||
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${prefix}${file}`)
|
||||
: SharedWorkspaceLocking.s3.send(
|
||||
new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: `${prefix}${file}` }),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
public static async ReadLines(command: string): Promise<string[]> {
|
||||
const path = command.replace('aws s3 ls', '').replace('rclone lsf', '').trim();
|
||||
const withoutScheme = path.replace('s3://', '');
|
||||
const [bucket, ...rest] = withoutScheme.split('/');
|
||||
const prefix = rest.join('/');
|
||||
|
||||
return SharedWorkspaceLocking.listObjects(prefix, bucket);
|
||||
}
|
||||
}
|
||||
|
||||
export default SharedWorkspaceLocking;
|
||||
@@ -0,0 +1,200 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import Input from '../../../input';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorOptionsReader from '../../options/orchestrator-options-reader';
|
||||
import OrchestratorQueryOverride from '../../options/orchestrator-query-override';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { CommandHookService } from '../hooks/command-hook-service';
|
||||
|
||||
export class TaskParameterSerializer {
|
||||
static readonly blockedParameterNames: Set<string> = new Set([
|
||||
'0',
|
||||
'length',
|
||||
'prototype',
|
||||
'',
|
||||
'unityVersion',
|
||||
'CACHE_UNITY_INSTALLATION_ON_MAC',
|
||||
'RUNNER_TEMP_PATH',
|
||||
'NAME',
|
||||
'CUSTOM_JOB',
|
||||
]);
|
||||
public static createOrchestratorEnvironmentVariables(
|
||||
buildParameters: BuildParameters,
|
||||
): OrchestratorEnvironmentVariable[] {
|
||||
const result: OrchestratorEnvironmentVariable[] = this.uniqBy(
|
||||
[
|
||||
...[
|
||||
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
|
||||
{ name: 'UNITY_VERSION', value: buildParameters.editorVersion },
|
||||
{ name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
|
||||
],
|
||||
...TaskParameterSerializer.serializeFromObject(buildParameters),
|
||||
...TaskParameterSerializer.serializeInput(),
|
||||
...TaskParameterSerializer.serializeOrchestratorOptions(),
|
||||
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
|
||||
|
||||
// Include AWS environment variables for LocalStack compatibility
|
||||
...TaskParameterSerializer.serializeAwsEnvironmentVariables(),
|
||||
]
|
||||
.filter(
|
||||
(x) =>
|
||||
!TaskParameterSerializer.blockedParameterNames.has(x.name) &&
|
||||
x.value !== '' &&
|
||||
x.value !== undefined &&
|
||||
x.value !== `undefined`,
|
||||
)
|
||||
.map((x) => {
|
||||
x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
|
||||
x.value = `${x.value}`;
|
||||
|
||||
return x;
|
||||
}),
|
||||
(item: OrchestratorEnvironmentVariable) => item.name,
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
static uniqBy(a: OrchestratorEnvironmentVariable[], key: (parameters: OrchestratorEnvironmentVariable) => string) {
|
||||
const seen: { [key: string]: boolean } = {};
|
||||
|
||||
return a.filter(function (item) {
|
||||
const k = key(item);
|
||||
|
||||
return seen.hasOwnProperty(k) ? false : (seen[k] = true);
|
||||
});
|
||||
}
|
||||
|
||||
public static readBuildParameterFromEnvironment(): BuildParameters {
|
||||
const buildParameters = new BuildParameters();
|
||||
const keys = [
|
||||
...new Set(
|
||||
Object.getOwnPropertyNames(process.env)
|
||||
.filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
|
||||
.map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
|
||||
),
|
||||
];
|
||||
|
||||
for (const element of keys) {
|
||||
if (element !== `customJob`) {
|
||||
buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
|
||||
}
|
||||
}
|
||||
|
||||
return buildParameters;
|
||||
}
|
||||
|
||||
private static serializeInput() {
|
||||
return TaskParameterSerializer.serializeFromType(Input);
|
||||
}
|
||||
|
||||
private static serializeOrchestratorOptions() {
|
||||
return TaskParameterSerializer.serializeFromType(OrchestratorOptions);
|
||||
}
|
||||
|
||||
private static serializeAwsEnvironmentVariables() {
|
||||
const awsEnvironmentVariables = [
|
||||
'AWS_ACCESS_KEY_ID',
|
||||
'AWS_SECRET_ACCESS_KEY',
|
||||
'AWS_DEFAULT_REGION',
|
||||
'AWS_REGION',
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
];
|
||||
|
||||
return awsEnvironmentVariables
|
||||
.filter((key) => process.env[key] !== undefined)
|
||||
.map((key) => ({
|
||||
name: key,
|
||||
value: process.env[key] || '',
|
||||
}));
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string): string {
|
||||
return OrchestratorOptions.ToEnvVarFormat(input);
|
||||
}
|
||||
|
||||
public static UndoEnvVarFormat(element: string): string {
|
||||
return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
|
||||
}
|
||||
|
||||
private static camelize(string: string) {
|
||||
return TaskParameterSerializer.uncapitalizeFirstLetter(
|
||||
string
|
||||
.replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
|
||||
return index === 0 ? word.toLowerCase() : word.toUpperCase();
|
||||
})
|
||||
.replace(/\s+/g, ''),
|
||||
);
|
||||
}
|
||||
|
||||
private static uncapitalizeFirstLetter(string: string) {
|
||||
return string.charAt(0).toLowerCase() + string.slice(1);
|
||||
}
|
||||
|
||||
private static serializeFromObject(buildParameters: any) {
|
||||
const array: any[] = [];
|
||||
const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
|
||||
for (const element of keys) {
|
||||
array.push({
|
||||
name: TaskParameterSerializer.ToEnvVarFormat(element),
|
||||
value: buildParameters[element],
|
||||
});
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static serializeFromType(type: any) {
|
||||
const array: any[] = [];
|
||||
const input = OrchestratorOptionsReader.GetProperties();
|
||||
for (const element of input) {
|
||||
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
|
||||
array.push({
|
||||
name: element,
|
||||
value: `${type[element]}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
public static readDefaultSecrets(): OrchestratorSecret[] {
|
||||
let array = new Array();
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
|
||||
|
||||
// array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
private static getValue(key: string) {
|
||||
return OrchestratorQueryOverride.queryOverrides !== undefined &&
|
||||
OrchestratorQueryOverride.queryOverrides[key] !== undefined
|
||||
? OrchestratorQueryOverride.queryOverrides[key]
|
||||
: process.env[key];
|
||||
}
|
||||
|
||||
private static tryAddInput(array: OrchestratorSecret[], key: string): OrchestratorSecret[] {
|
||||
const value = TaskParameterSerializer.getValue(key);
|
||||
if (value !== undefined && value !== '' && value !== 'null') {
|
||||
array.push({
|
||||
ParameterKey: key,
|
||||
EnvironmentVariable: key,
|
||||
ParameterValue: value,
|
||||
});
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { BuildParameters, Input } from '../../..';
|
||||
import YAML from 'yaml';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import * as fs from 'node:fs';
|
||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||
import { CommandHook } from './command-hook';
|
||||
|
||||
// import OrchestratorLogger from './orchestrator-logger';
|
||||
|
||||
export class CommandHookService {
|
||||
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
|
||||
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
|
||||
OrchestratorLogger.log(`Applying hooks ${hooks.length}`);
|
||||
|
||||
return `echo "---"
|
||||
echo "start orchestrator init"
|
||||
${OrchestratorOptions.orchestratorDebug ? `printenv` : `#`}
|
||||
echo "start of orchestrator job"
|
||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${commands}
|
||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
echo "end of orchestrator job"
|
||||
echo "---${buildParameters.logId}"`;
|
||||
}
|
||||
|
||||
public static getHooks(customCommandHooks: string): CommandHook[] {
|
||||
const experimentHooks = customCommandHooks;
|
||||
let output = new Array<CommandHook>();
|
||||
if (experimentHooks && experimentHooks !== '') {
|
||||
try {
|
||||
output = YAML.parse(experimentHooks);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
|
||||
...CommandHookService.GetCustomHooksFromFiles(`before`),
|
||||
...CommandHookService.GetCustomHooksFromFiles(`after`),
|
||||
];
|
||||
}
|
||||
|
||||
static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
|
||||
const results: CommandHook[] = [];
|
||||
|
||||
// RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
|
||||
try {
|
||||
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
if (!OrchestratorOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||
continue;
|
||||
}
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
|
||||
if (fileContentsObject.hook.includes(hookLifecycle)) {
|
||||
results.push(fileContentsObject);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
||||
}
|
||||
|
||||
// RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static ConvertYamlSecrets(object: CommandHook) {
|
||||
if (object.secrets === undefined) {
|
||||
object.secrets = [];
|
||||
|
||||
return;
|
||||
}
|
||||
object.secrets = object.secrets.map((x: any) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
public static ParseHooks(hooks: string): CommandHook[] {
|
||||
if (hooks === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
// if (Orchestrator.buildParameters?.orchestratorIntegrationTests) {
|
||||
|
||||
// OrchestratorLogger.log(`Parsing build hooks: ${steps}`);
|
||||
|
||||
// }
|
||||
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
|
||||
const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
|
||||
for (const hook of object) {
|
||||
CommandHookService.ConvertYamlSecrets(hook);
|
||||
if (hook.secrets === undefined) {
|
||||
hook.secrets = [];
|
||||
}
|
||||
}
|
||||
if (object === undefined) {
|
||||
throw new Error(`Failed to parse ${hooks}`);
|
||||
}
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
public static getSecrets(hooks: any) {
|
||||
const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
|
||||
|
||||
// eslint-disable-next-line unicorn/no-array-reduce
|
||||
return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
|
||||
export class CommandHook {
|
||||
public commands: string[] = new Array<string>();
|
||||
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
|
||||
public name!: string;
|
||||
public hook!: string[];
|
||||
public step!: string[];
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
import YAML from 'yaml';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CustomWorkflow } from '../../workflows/custom-workflow';
|
||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import Input from '../../../input';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { ContainerHook as ContainerHook } from './container-hook';
|
||||
import { OrchestratorStepParameters } from '../../options/orchestrator-step-parameters';
|
||||
|
||||
export class ContainerHookService {
|
||||
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
|
||||
const results: ContainerHook[] = [];
|
||||
try {
|
||||
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomStepsPath);
|
||||
for (const file of files) {
|
||||
if (!OrchestratorOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
|
||||
continue;
|
||||
}
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
|
||||
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
results.push(fileContentsObject);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
||||
}
|
||||
|
||||
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
|
||||
|
||||
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
|
||||
`- name: aws-s3-upload-build
|
||||
image: amazon/aws-cli
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} s3://${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-upload-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: awsSecretAccessKey
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: awsDefaultRegion
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||
- name: aws-s3-pull-build
|
||||
image: amazon/aws-cli
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ || true
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build || true
|
||||
aws s3 cp s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-pull-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: AWS_S3_ENDPOINT
|
||||
- name: steam-deploy-client
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: RELEASE_BRANCH
|
||||
- name: steam-deploy-project
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_2
|
||||
- name: RELEASE_BRANCH
|
||||
- name: aws-s3-upload-cache
|
||||
image: amazon/aws-cli
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-upload-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||
- name: aws-s3-pull-cache
|
||||
image: amazon/aws-cli
|
||||
hook: before
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||
if command -v aws > /dev/null 2>&1; then
|
||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
||||
fi
|
||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
||||
fi
|
||||
ENDPOINT_ARGS=""
|
||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ 2>/dev/null || true
|
||||
aws $ENDPOINT_ARGS s3 ls ${
|
||||
Orchestrator.buildParameters.awsStackName
|
||||
}/orchestrator-cache/$CACHE_KEY/ 2>/dev/null || true
|
||||
BUCKET1="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/Library/"
|
||||
OBJECT1=""
|
||||
LS_OUTPUT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 2>/dev/null || echo '')"
|
||||
if [ -n "$LS_OUTPUT1" ] && [ "$LS_OUTPUT1" != "" ]; then
|
||||
OBJECT1="$(echo "$LS_OUTPUT1" | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
if [ -n "$OBJECT1" ] && [ "$OBJECT1" != "" ]; then
|
||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
BUCKET2="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/lfs/"
|
||||
OBJECT2=""
|
||||
LS_OUTPUT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 2>/dev/null || echo '')"
|
||||
if [ -n "$LS_OUTPUT2" ] && [ "$LS_OUTPUT2" != "" ]; then
|
||||
OBJECT2="$(echo "$LS_OUTPUT2" | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
if [ -n "$OBJECT2" ] && [ "$OBJECT2" != "" ]; then
|
||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "AWS CLI not available, skipping aws-s3-pull-cache"
|
||||
fi
|
||||
- name: rclone-upload-build
|
||||
image: rclone/rclone
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} ${Orchestrator.buildParameters.rcloneRemote}/orchestrator-cache/$CACHE_KEY/build/ || true
|
||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-upload-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-pull-build
|
||||
image: rclone/rclone
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
} || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-pull-build"
|
||||
fi
|
||||
secrets:
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-upload-cache
|
||||
image: rclone/rclone
|
||||
hook: after
|
||||
commands: |
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy /data/cache/$CACHE_KEY/lfs ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||
rclone copy /data/cache/$CACHE_KEY/Library ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-upload-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: rclone-pull-cache
|
||||
image: rclone/rclone
|
||||
hook: before
|
||||
commands: |
|
||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||
if command -v rclone > /dev/null 2>&1; then
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true
|
||||
rclone copy ${
|
||||
Orchestrator.buildParameters.rcloneRemote
|
||||
}/orchestrator-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true
|
||||
else
|
||||
echo "rclone not available, skipping rclone-pull-cache"
|
||||
fi
|
||||
secrets:
|
||||
- name: RCLONE_REMOTE
|
||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
||||
- name: debug-cache
|
||||
image: ubuntu
|
||||
hook: after
|
||||
commands: |
|
||||
apt-get update > /dev/null || true
|
||||
${OrchestratorOptions.orchestratorDebug ? `apt-get install -y tree > /dev/null || true` : `#`}
|
||||
${OrchestratorOptions.orchestratorDebug ? `tree -L 3 /data/cache || true` : `#`}
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
- name: awsSecretAccessKey
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: awsDefaultRegion
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: AWS_S3_ENDPOINT
|
||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`,
|
||||
).filter((x) => OrchestratorOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
|
||||
|
||||
// In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks
|
||||
const provider = Orchestrator.buildParameters?.providerStrategy;
|
||||
const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker';
|
||||
const hasAwsCreds =
|
||||
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
(process.env.awsAccessKeyId && process.env.awsSecretAccessKey);
|
||||
|
||||
// Always include AWS hooks on the AWS provider (task role provides creds),
|
||||
// otherwise require explicit creds for other containerized providers.
|
||||
const shouldIncludeAwsHooks =
|
||||
isContainerized && !Orchestrator.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds));
|
||||
const filteredBuiltIns = shouldIncludeAwsHooks
|
||||
? builtInContainerHooks
|
||||
: builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli');
|
||||
|
||||
if (filteredBuiltIns.length > 0) {
|
||||
results.push(...filteredBuiltIns);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static ConvertYamlSecrets(object: ContainerHook) {
|
||||
if (object.secrets === undefined) {
|
||||
object.secrets = [];
|
||||
|
||||
return;
|
||||
}
|
||||
object.secrets = object.secrets.map((x: { [key: string]: any }) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
public static ParseContainerHooks(steps: string): ContainerHook[] {
|
||||
if (steps === '') {
|
||||
return [];
|
||||
}
|
||||
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
|
||||
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
|
||||
for (const step of object) {
|
||||
ContainerHookService.ConvertYamlSecrets(step);
|
||||
if (step.secrets === undefined) {
|
||||
step.secrets = [];
|
||||
} else {
|
||||
for (const secret of step.secrets) {
|
||||
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
||||
if (Orchestrator.buildParameters?.orchestratorDebug) {
|
||||
// OrchestratorLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
||||
}
|
||||
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (step.image === undefined) {
|
||||
step.image = `ubuntu`;
|
||||
}
|
||||
|
||||
// Ensure allowFailure defaults to false if not explicitly set
|
||||
if (step.allowFailure === undefined) {
|
||||
step.allowFailure = false;
|
||||
}
|
||||
}
|
||||
if (object === undefined) {
|
||||
throw new Error(`Failed to parse ${steps}`);
|
||||
}
|
||||
|
||||
return object;
|
||||
}
|
||||
|
||||
static async RunPostBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
||||
let output = ``;
|
||||
const steps: ContainerHook[] = [
|
||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.postBuildContainerHooks),
|
||||
...ContainerHookService.GetContainerHooksFromFiles(`after`),
|
||||
];
|
||||
|
||||
if (steps.length > 0) {
|
||||
output += await CustomWorkflow.runContainerJob(
|
||||
steps,
|
||||
orchestratorStepState.environment,
|
||||
orchestratorStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
static async RunPreBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
||||
let output = ``;
|
||||
const steps: ContainerHook[] = [
|
||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.preBuildContainerHooks),
|
||||
...ContainerHookService.GetContainerHooksFromFiles(`before`),
|
||||
];
|
||||
|
||||
if (steps.length > 0) {
|
||||
output += await CustomWorkflow.runContainerJob(
|
||||
steps,
|
||||
orchestratorStepState.environment,
|
||||
orchestratorStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
|
||||
export class ContainerHook {
|
||||
public commands!: string;
|
||||
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
|
||||
public name!: string;
|
||||
public image: string = `ubuntu`;
|
||||
public hook!: string;
|
||||
public allowFailure: boolean = false; // If true, hook failures won't stop the build
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import path from 'node:path';
|
||||
import { OrchestratorFolders } from '../../options/orchestrator-folders';
|
||||
import { OrchestratorSystem } from '../core/orchestrator-system';
|
||||
import fs from 'node:fs';
|
||||
import { Cli } from '../../../cli/cli';
|
||||
import { CliFunction } from '../../../cli/cli-functions-repository';
|
||||
|
||||
export class LfsHashing {
|
||||
public static async createLFSHashFiles() {
|
||||
await OrchestratorSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
|
||||
await OrchestratorSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
|
||||
const lfsHashes = {
|
||||
lfsGuid: fs
|
||||
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
|
||||
.replace(/\n/g, ``),
|
||||
lfsGuidSum: fs
|
||||
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
|
||||
.replace(' .lfs-assets-guid', '')
|
||||
.replace(/\n/g, ``),
|
||||
};
|
||||
|
||||
return lfsHashes;
|
||||
}
|
||||
public static async hashAllFiles(folder: string) {
|
||||
const startPath = process.cwd();
|
||||
process.chdir(folder);
|
||||
const result = await (await OrchestratorSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
|
||||
.replace(/\n/g, '')
|
||||
.split(` `)[0];
|
||||
process.chdir(startPath);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@CliFunction(`hash`, `hash all folder contents`)
|
||||
static async hash() {
|
||||
if (!Cli.options) {
|
||||
return;
|
||||
}
|
||||
const folder = Cli.options['cachePushFrom'];
|
||||
LfsHashing.hashAllFiles(folder);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user