mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-10 16:03:54 -07:00
Rename Cloud Runner to Orchestrator (#775)
* Rename "Cloud Runner" to "Orchestrator" across entire codebase Breaking change: All CloudRunner classes, options, environment variables, and action.yml inputs have been renamed to Orchestrator equivalents. - Renamed src/model/cloud-runner/ directory to src/model/orchestrator/ - Renamed all cloud-runner-* files to orchestrator-* - Renamed all CloudRunner* classes to Orchestrator* (15+ classes) - Renamed all cloudRunner* properties to orchestrator* equivalents - Renamed CLOUD_RUNNER_* env vars to ORCHESTRATOR_* - Updated action.yml [CloudRunner] markers to [Orchestrator] - Updated workflow files and package.json test scripts - Updated all runtime strings (cache paths, log messages, branch refs) - Rebuilt dist/index.js No backward compatibility layer is provided. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Remove tracked log/temp files and add to .gitignore Remove $LOG_FILE and temp/job-log.txt debug artifacts that should not be in the repository. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frostebite
@@ -0,0 +1,483 @@
|
||||
import { assert } from 'node:console';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import Orchestrator from '../orchestrator';
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import { OrchestratorFolders } from '../options/orchestrator-folders';
|
||||
import { OrchestratorSystem } from '../services/core/orchestrator-system';
|
||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
// eslint-disable-next-line github/no-then
|
||||
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
|
||||
|
||||
export class Caching {
|
||||
@CliFunction(`cache-push`, `push to cache`)
|
||||
static async cachePush() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
Orchestrator.buildParameters = buildParameter;
|
||||
await Caching.PushToCache(
|
||||
Cli.options!['cachePushTo'],
|
||||
Cli.options!['cachePushFrom'],
|
||||
Cli.options!['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
@CliFunction(`cache-pull`, `pull from cache`)
|
||||
static async cachePull() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
Orchestrator.buildParameters = buildParameter;
|
||||
await Caching.PullFromCache(
|
||||
Cli.options!['cachePushFrom'],
|
||||
Cli.options!['cachePushTo'],
|
||||
Cli.options!['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||
OrchestratorLogger.log(`Pushing to cache ${sourceFolder}`);
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
let compressionSuffix = '';
|
||||
if (Orchestrator.buildParameters.useCompressionStrategy === true) {
|
||||
compressionSuffix = `.lz4`;
|
||||
}
|
||||
OrchestratorLogger.log(`Compression: ${Orchestrator.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await OrchestratorSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
process.chdir(path.resolve(sourceFolder, '..'));
|
||||
|
||||
if (Orchestrator.buildParameters.orchestratorDebug === true) {
|
||||
OrchestratorLogger.log(
|
||||
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||
sourceFolder,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
const contents = await fs.promises.readdir(path.basename(sourceFolder));
|
||||
OrchestratorLogger.log(
|
||||
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
|
||||
if (contents.length === 0) {
|
||||
OrchestratorLogger.log(
|
||||
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
process.chdir(`${startPath}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Check disk space before creating tar archive and clean up if needed
|
||||
let diskUsagePercent = 0;
|
||||
try {
|
||||
const diskCheckOutput = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
|
||||
OrchestratorLogger.log(`Disk space before tar: ${diskCheckOutput}`);
|
||||
|
||||
// Parse disk usage percentage (e.g., "72G 72G 196M 100%")
|
||||
const usageMatch = diskCheckOutput.match(/(\d+)%/);
|
||||
if (usageMatch) {
|
||||
diskUsagePercent = Number.parseInt(usageMatch[1], 10);
|
||||
}
|
||||
} catch {
|
||||
// Ignore disk check errors
|
||||
}
|
||||
|
||||
// If disk usage is high (>90%), proactively clean up old cache files
|
||||
if (diskUsagePercent > 90) {
|
||||
OrchestratorLogger.log(`Disk usage is ${diskUsagePercent}% - cleaning up old cache files before tar operation`);
|
||||
try {
|
||||
const cacheParent = path.dirname(cacheFolder);
|
||||
if (await fileExists(cacheParent)) {
|
||||
// Try to fix permissions first to avoid permission denied errors
|
||||
await OrchestratorSystem.Run(
|
||||
`chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Remove cache files older than 6 hours (more aggressive than 1 day)
|
||||
// Use multiple methods to handle permission issues
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Try with sudo if available
|
||||
await OrchestratorSystem.Run(
|
||||
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// As last resort, try to remove files one by one
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheParent} -name "*.tar*" -type f -mmin +360 -exec rm -f {} + 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Also try to remove old cache directories
|
||||
await OrchestratorSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`);
|
||||
|
||||
// If disk is still very high (>95%), be even more aggressive
|
||||
if (diskUsagePercent > 95) {
|
||||
OrchestratorLogger.log(
|
||||
`Disk usage is very high (${diskUsagePercent}%), performing aggressive cleanup...`,
|
||||
);
|
||||
|
||||
// Remove files older than 1 hour
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
|
||||
);
|
||||
await OrchestratorSystem.Run(
|
||||
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
|
||||
);
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`Cleanup completed. Checking disk space again...`);
|
||||
const diskCheckAfter = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
|
||||
OrchestratorLogger.log(`Disk space after cleanup: ${diskCheckAfter}`);
|
||||
|
||||
// Check disk usage again after cleanup
|
||||
let diskUsageAfterCleanup = 0;
|
||||
try {
|
||||
const usageMatchAfter = diskCheckAfter.match(/(\d+)%/);
|
||||
if (usageMatchAfter) {
|
||||
diskUsageAfterCleanup = Number.parseInt(usageMatchAfter[1], 10);
|
||||
}
|
||||
} catch {
|
||||
// Ignore parsing errors
|
||||
}
|
||||
|
||||
// If disk is still at 100% after cleanup, skip tar operation to prevent hang.
|
||||
// Do NOT fail the build here – it's better to skip caching than to fail the job
|
||||
// due to shared CI disk pressure.
|
||||
if (diskUsageAfterCleanup >= 100) {
|
||||
const message = `Cannot create cache archive: disk is still at ${diskUsageAfterCleanup}% after cleanup. Tar operation would hang. Skipping cache push; please free up disk space manually if this persists.`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
RemoteClientLogger.log(message);
|
||||
|
||||
// Restore working directory before early return
|
||||
process.chdir(`${startPath}`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (cleanupError) {
|
||||
// If cleanupError is our disk space error, rethrow it
|
||||
if (cleanupError instanceof Error && cleanupError.message.includes('Cannot create cache archive')) {
|
||||
throw cleanupError;
|
||||
}
|
||||
OrchestratorLogger.log(`Proactive cleanup failed: ${cleanupError}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up any existing incomplete tar files
|
||||
try {
|
||||
await OrchestratorSystem.Run(`rm -f ${cacheArtifactName}.tar${compressionSuffix} 2>/dev/null || true`);
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
|
||||
try {
|
||||
// Add timeout to tar command to prevent hanging when disk is full
|
||||
// Use timeout command with 10 minute limit (600 seconds) if available
|
||||
// Check if timeout command exists, otherwise use regular tar
|
||||
const tarCommand = `tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`;
|
||||
let tarCommandToRun = tarCommand;
|
||||
try {
|
||||
// Check if timeout command is available
|
||||
await OrchestratorSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
|
||||
|
||||
// Use timeout if available (600 seconds = 10 minutes)
|
||||
tarCommandToRun = `timeout 600 ${tarCommand}`;
|
||||
} catch {
|
||||
// timeout command not available, use regular tar
|
||||
// Note: This could still hang if disk is full, but the disk space check above should prevent this
|
||||
tarCommandToRun = tarCommand;
|
||||
}
|
||||
|
||||
await OrchestratorSystem.Run(tarCommandToRun);
|
||||
} catch (error: any) {
|
||||
// Check if error is due to disk space or timeout
|
||||
const errorMessage = error?.message || error?.toString() || '';
|
||||
if (
|
||||
errorMessage.includes('No space left') ||
|
||||
errorMessage.includes('Wrote only') ||
|
||||
errorMessage.includes('timeout') ||
|
||||
errorMessage.includes('Terminated')
|
||||
) {
|
||||
OrchestratorLogger.log(`Disk space error detected. Attempting aggressive cleanup...`);
|
||||
|
||||
// Try to clean up old cache files more aggressively
|
||||
try {
|
||||
const cacheParent = path.dirname(cacheFolder);
|
||||
if (await fileExists(cacheParent)) {
|
||||
// Try to fix permissions first to avoid permission denied errors
|
||||
await OrchestratorSystem.Run(
|
||||
`chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Remove cache files older than 1 hour (very aggressive)
|
||||
// Use multiple methods to handle permission issues
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
|
||||
);
|
||||
await OrchestratorSystem.Run(
|
||||
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// As last resort, try to remove files one by one
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -exec rm -f {} + 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Remove empty cache directories
|
||||
await OrchestratorSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`);
|
||||
|
||||
// Also try to clean up the entire cache folder if it's getting too large
|
||||
const cacheRoot = path.resolve(cacheParent, '..');
|
||||
if (await fileExists(cacheRoot)) {
|
||||
// Try to fix permissions for cache root too
|
||||
await OrchestratorSystem.Run(
|
||||
`chmod -R u+w ${cacheRoot} 2>/dev/null || chown -R $(whoami) ${cacheRoot} 2>/dev/null || true`,
|
||||
);
|
||||
|
||||
// Remove cache entries older than 30 minutes
|
||||
await OrchestratorSystem.Run(
|
||||
`find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`,
|
||||
);
|
||||
await OrchestratorSystem.Run(
|
||||
`sudo find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`,
|
||||
);
|
||||
}
|
||||
OrchestratorLogger.log(`Aggressive cleanup completed. Retrying tar operation...`);
|
||||
|
||||
// Retry the tar operation once after cleanup
|
||||
let retrySucceeded = false;
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
|
||||
);
|
||||
|
||||
// If retry succeeds, mark it - we'll continue normally without throwing
|
||||
retrySucceeded = true;
|
||||
} catch (retryError: any) {
|
||||
throw new Error(
|
||||
`Failed to create cache archive after cleanup. Original error: ${errorMessage}. Retry error: ${
|
||||
retryError?.message || retryError
|
||||
}`,
|
||||
);
|
||||
}
|
||||
|
||||
// If retry succeeded, don't throw the original error - let execution continue after catch block
|
||||
if (!retrySucceeded) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// If we get here, retry succeeded - execution will continue after the catch block
|
||||
} else {
|
||||
throw new Error(
|
||||
`Failed to create cache archive due to insufficient disk space. Error: ${errorMessage}. Cleanup not possible - cache folder missing.`,
|
||||
);
|
||||
}
|
||||
} catch (cleanupError: any) {
|
||||
OrchestratorLogger.log(`Cleanup attempt failed: ${cleanupError}`);
|
||||
throw new Error(
|
||||
`Failed to create cache archive due to insufficient disk space. Error: ${errorMessage}. Cleanup failed: ${
|
||||
cleanupError?.message || cleanupError
|
||||
}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await OrchestratorSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
|
||||
// Ensure the cache folder directory exists before moving the file
|
||||
// (it might have been deleted by cleanup if it was empty)
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await OrchestratorSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
await OrchestratorSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||
assert(
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
||||
'cache archive exists inside cache folder',
|
||||
);
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||
OrchestratorLogger.log(`Pulling from cache ${destinationFolder} ${Orchestrator.buildParameters.skipCache}`);
|
||||
if (`${Orchestrator.buildParameters.skipCache}` === `true`) {
|
||||
OrchestratorLogger.log(`Skipping cache debugSkipCache is true`);
|
||||
|
||||
return;
|
||||
}
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
let compressionSuffix = '';
|
||||
if (Orchestrator.buildParameters.useCompressionStrategy === true) {
|
||||
compressionSuffix = `.lz4`;
|
||||
}
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await fs.promises.mkdir(cacheFolder);
|
||||
}
|
||||
|
||||
if (!(await fileExists(destinationFolder))) {
|
||||
await fs.promises.mkdir(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (
|
||||
await OrchestratorSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
||||
)
|
||||
.replace(/\n/g, ``)
|
||||
.replace(`.tar${compressionSuffix}`, '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection =
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||
? cacheArtifactName
|
||||
: latestInBranch;
|
||||
await OrchestratorLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
// Check disk space before extraction to prevent hangs
|
||||
let diskUsagePercent = 0;
|
||||
try {
|
||||
const diskCheckOutput = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
|
||||
const usageMatch = diskCheckOutput.match(/(\d+)%/);
|
||||
if (usageMatch) {
|
||||
diskUsagePercent = Number.parseInt(usageMatch[1], 10);
|
||||
}
|
||||
} catch {
|
||||
// Ignore disk check errors
|
||||
}
|
||||
|
||||
// If disk is at 100%, skip cache extraction to prevent hangs
|
||||
if (diskUsagePercent >= 100) {
|
||||
const message = `Disk is at ${diskUsagePercent}% - skipping cache extraction to prevent hang. Cache may be incomplete or corrupted.`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate tar file integrity before extraction
|
||||
try {
|
||||
// Use tar -t to test the archive without extracting (fast check)
|
||||
// This will fail if the archive is corrupted
|
||||
await OrchestratorSystem.Run(
|
||||
`tar -tf ${cacheSelection}.tar${compressionSuffix} > /dev/null 2>&1 || (echo "Tar file validation failed" && exit 1)`,
|
||||
);
|
||||
} catch {
|
||||
const message = `Cache archive ${cacheSelection}.tar${compressionSuffix} appears to be corrupted or incomplete. Skipping cache extraction.`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const resultsFolder = `results${Orchestrator.buildParameters.buildGuid}`;
|
||||
await OrchestratorSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
|
||||
// Extract with timeout to prevent infinite hangs
|
||||
try {
|
||||
let tarExtractCommand = `tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`;
|
||||
|
||||
// Add timeout if available (600 seconds = 10 minutes)
|
||||
try {
|
||||
await OrchestratorSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
|
||||
tarExtractCommand = `timeout 600 ${tarExtractCommand}`;
|
||||
} catch {
|
||||
// timeout command not available, use regular tar
|
||||
}
|
||||
|
||||
await OrchestratorSystem.Run(tarExtractCommand);
|
||||
} catch (extractError: any) {
|
||||
const errorMessage = extractError?.message || extractError?.toString() || '';
|
||||
|
||||
// Check for common tar errors that indicate corruption or disk issues
|
||||
if (
|
||||
errorMessage.includes('Unexpected EOF') ||
|
||||
errorMessage.includes('rmtlseek') ||
|
||||
errorMessage.includes('No space left') ||
|
||||
errorMessage.includes('timeout') ||
|
||||
errorMessage.includes('Terminated')
|
||||
) {
|
||||
const message = `Cache extraction failed (likely due to corrupted archive or disk space): ${errorMessage}. Continuing without cache.`;
|
||||
OrchestratorLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Re-throw other errors
|
||||
throw extractError;
|
||||
}
|
||||
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
|
||||
if (await fileExists(destinationFolder)) {
|
||||
await fs.promises.rmdir(destinationFolder, { recursive: true });
|
||||
}
|
||||
await OrchestratorSystem.Run(
|
||||
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
|
||||
);
|
||||
const contents = await fs.promises.readdir(
|
||||
path.join(destinationParentFolder, path.basename(destinationFolder)),
|
||||
);
|
||||
OrchestratorLogger.log(
|
||||
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
RemoteClientLogger.logWarning(
|
||||
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
||||
);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(startPath);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(startPath);
|
||||
}
|
||||
|
||||
public static async handleCachePurging() {
|
||||
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
||||
RemoteClientLogger.log(`purging ${OrchestratorFolders.purgeRemoteCaching}`);
|
||||
fs.promises.rmdir(OrchestratorFolders.cacheFolder, { recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
import fs from 'node:fs';
|
||||
import Orchestrator from '../orchestrator';
|
||||
import { OrchestratorFolders } from '../options/orchestrator-folders';
|
||||
import { Caching } from './caching';
|
||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import { assert } from 'node:console';
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
import { OrchestratorSystem } from '../services/core/orchestrator-system';
|
||||
import YAML from 'yaml';
|
||||
import GitHub from '../../github';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import OrchestratorOptions from '../options/orchestrator-options';
|
||||
import ResourceTracking from '../services/core/resource-tracking';
|
||||
|
||||
export class RemoteClient {
|
||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
static async setupRemoteClient() {
|
||||
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
|
||||
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||
}
|
||||
|
||||
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
|
||||
public static async remoteClientLogStream() {
|
||||
const logFile = Cli.options!['logFile'];
|
||||
process.stdin.resume();
|
||||
process.stdin.setEncoding('utf8');
|
||||
|
||||
// For K8s, ensure stdout is unbuffered so messages are captured immediately
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
process.stdout.setDefaultEncoding('utf8');
|
||||
}
|
||||
|
||||
let lingeringLine = '';
|
||||
|
||||
process.stdin.on('data', (chunk) => {
|
||||
const lines = chunk.toString().split('\n');
|
||||
|
||||
lines[0] = lingeringLine + lines[0];
|
||||
lingeringLine = lines.pop() || '';
|
||||
|
||||
for (const element of lines) {
|
||||
// Always write to log file so output can be collected by providers
|
||||
if (element.trim()) {
|
||||
fs.appendFileSync(logFile, `${element}\n`);
|
||||
}
|
||||
|
||||
// For K8s, also write to stdout so kubectl logs can capture it
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
// Write to stdout so kubectl logs can capture it - ensure newline is included
|
||||
// Stdout flushes automatically on newline, so no explicit flush needed
|
||||
process.stdout.write(`${element}\n`);
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(element);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdin.on('end', () => {
|
||||
if (lingeringLine) {
|
||||
// Always write to log file so output can be collected by providers
|
||||
fs.appendFileSync(logFile, `${lingeringLine}\n`);
|
||||
|
||||
// For K8s, also write to stdout so kubectl logs can capture it
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
// Stdout flushes automatically on newline
|
||||
process.stdout.write(`${lingeringLine}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(lingeringLine);
|
||||
});
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli-post-build`, `runs a orchestrator build`)
|
||||
public static async remoteClientPostBuild(): Promise<string> {
|
||||
try {
|
||||
RemoteClientLogger.log(`Running POST build tasks`);
|
||||
|
||||
// Ensure cache key is present in logs for assertions
|
||||
RemoteClientLogger.log(`CACHE_KEY=${Orchestrator.buildParameters.cacheKey}`);
|
||||
OrchestratorLogger.log(`${Orchestrator.buildParameters.cacheKey}`);
|
||||
|
||||
// Guard: only push Library cache if the folder exists and has contents
|
||||
try {
|
||||
const libraryFolderHost = OrchestratorFolders.libraryFolderAbsolute;
|
||||
if (fs.existsSync(libraryFolderHost)) {
|
||||
let libraryEntries: string[] = [];
|
||||
try {
|
||||
libraryEntries = await fs.promises.readdir(libraryFolderHost);
|
||||
} catch {
|
||||
libraryEntries = [];
|
||||
}
|
||||
if (libraryEntries.length > 0) {
|
||||
await Caching.PushToCache(
|
||||
OrchestratorFolders.ToLinuxFolder(`${OrchestratorFolders.cacheFolderForCacheKeyFull}/Library`),
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryFolderAbsolute),
|
||||
`lib-${Orchestrator.buildParameters.buildGuid}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Library cache push (folder is empty)`);
|
||||
}
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Library cache push (folder missing)`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logWarning(`Library cache push skipped with error: ${error.message}`);
|
||||
}
|
||||
|
||||
// Guard: only push Build cache if the folder exists and has contents
|
||||
try {
|
||||
const buildFolderHost = OrchestratorFolders.projectBuildFolderAbsolute;
|
||||
if (fs.existsSync(buildFolderHost)) {
|
||||
let buildEntries: string[] = [];
|
||||
try {
|
||||
buildEntries = await fs.promises.readdir(buildFolderHost);
|
||||
} catch {
|
||||
buildEntries = [];
|
||||
}
|
||||
if (buildEntries.length > 0) {
|
||||
await Caching.PushToCache(
|
||||
OrchestratorFolders.ToLinuxFolder(`${OrchestratorFolders.cacheFolderForCacheKeyFull}/build`),
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.projectBuildFolderAbsolute),
|
||||
`build-${Orchestrator.buildParameters.buildGuid}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Build cache push (folder is empty)`);
|
||||
}
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Build cache push (folder missing)`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logWarning(`Build cache push skipped with error: ${error.message}`);
|
||||
}
|
||||
|
||||
if (!BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)) {
|
||||
const uniqueJobFolderLinux = OrchestratorFolders.ToLinuxFolder(
|
||||
OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute,
|
||||
);
|
||||
if (
|
||||
fs.existsSync(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute) ||
|
||||
fs.existsSync(uniqueJobFolderLinux)
|
||||
) {
|
||||
await OrchestratorSystem.Run(`rm -r ${uniqueJobFolderLinux} || true`);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping cleanup; unique job folder missing`);
|
||||
}
|
||||
}
|
||||
|
||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||
|
||||
// WIP - need to give the pod permissions to create config map
|
||||
await RemoteClientLogger.handleLogManagementPostJob();
|
||||
} catch (error: any) {
|
||||
// Log error but don't fail - post-build tasks are best-effort
|
||||
RemoteClientLogger.logWarning(`Post-build task error: ${error.message}`);
|
||||
OrchestratorLogger.log(`Post-build task error: ${error.message}`);
|
||||
}
|
||||
|
||||
// Ensure success marker is always present in logs for tests, even if post-build tasks failed
|
||||
// For K8s, kubectl logs reads from stdout/stderr, so we must write to stdout
|
||||
// For all providers, we write to stdout so it gets piped through the log stream
|
||||
// The log stream will capture it and add it to BuildResults
|
||||
const successMessage = `Activation successful`;
|
||||
|
||||
// Write directly to log file first to ensure it's captured even if pipe fails
|
||||
// This is critical for all providers, especially K8s where timing matters
|
||||
try {
|
||||
const logFilePath = Orchestrator.isOrchestratorEnvironment
|
||||
? `/home/job-log.txt`
|
||||
: path.join(process.cwd(), 'temp', 'job-log.txt');
|
||||
if (fs.existsSync(path.dirname(logFilePath))) {
|
||||
fs.appendFileSync(logFilePath, `${successMessage}\n`);
|
||||
}
|
||||
} catch {
|
||||
// If direct file write fails, continue with other methods
|
||||
}
|
||||
|
||||
// Write to stdout so it gets piped through remote-cli-log-stream when invoked via pipe
|
||||
// This ensures the message is captured in BuildResults for all providers
|
||||
// Use synchronous write and ensure newline is included for proper flushing
|
||||
process.stdout.write(`${successMessage}\n`, 'utf8');
|
||||
|
||||
// For K8s, also write to stderr as a backup since kubectl logs reads from both stdout and stderr
|
||||
// This ensures the message is captured even if stdout pipe has issues
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
process.stderr.write(`${successMessage}\n`, 'utf8');
|
||||
}
|
||||
|
||||
// Ensure stdout is flushed before process exits (critical for K8s where process might exit quickly)
|
||||
// For non-TTY streams, we need to explicitly ensure the write completes
|
||||
if (!process.stdout.isTTY) {
|
||||
// Give the pipe a moment to process the write
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
// Also log via OrchestratorLogger and RemoteClientLogger for GitHub Actions and log file
|
||||
// This ensures the message appears in log files for providers that read from log files
|
||||
// RemoteClientLogger.log writes directly to the log file, which is important for providers
|
||||
// that read from the log file rather than stdout
|
||||
RemoteClientLogger.log(successMessage);
|
||||
OrchestratorLogger.log(successMessage);
|
||||
await ResourceTracking.logDiskUsageSnapshot('remote-cli-post-build (end)');
|
||||
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
static async runCustomHookFiles(hookLifecycle: string) {
|
||||
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||
const gameCiCustomHooksPath = path.join(OrchestratorFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||
try {
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = YAML.parse(fileContents.toString());
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
|
||||
await OrchestratorSystem.Run(fileContentsObject.commands);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||
}
|
||||
}
|
||||
public static async bootstrapRepository() {
|
||||
await OrchestratorSystem.Run(
|
||||
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
|
||||
);
|
||||
await OrchestratorSystem.Run(
|
||||
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.cacheFolderForCacheKeyFull)}`,
|
||||
);
|
||||
await RemoteClient.cloneRepoWithoutLFSFiles();
|
||||
await RemoteClient.sizeOfFolder(
|
||||
'repo before lfs cache pull',
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.repoPathAbsolute),
|
||||
);
|
||||
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
||||
if (fs.existsSync(OrchestratorFolders.libraryFolderAbsolute)) {
|
||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||
}
|
||||
await Caching.PullFromCache(
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsCacheFolderFull),
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after lfs cache pull', OrchestratorFolders.repoPathAbsolute);
|
||||
await RemoteClient.pullLatestLFS();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs git pull', OrchestratorFolders.repoPathAbsolute);
|
||||
await Caching.PushToCache(
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsCacheFolderFull),
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await Caching.PullFromCache(
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryCacheFolderFull),
|
||||
OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryFolderAbsolute),
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after library cache pull', OrchestratorFolders.repoPathAbsolute);
|
||||
await Caching.handleCachePurging();
|
||||
}
|
||||
|
||||
private static async sizeOfFolder(message: string, folder: string) {
|
||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
||||
OrchestratorLogger.log(`Size of ${message}`);
|
||||
await OrchestratorSystem.Run(`du -sh ${folder}`);
|
||||
}
|
||||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
process.chdir(`${OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute}`);
|
||||
if (
|
||||
fs.existsSync(OrchestratorFolders.repoPathAbsolute) &&
|
||||
!fs.existsSync(path.join(OrchestratorFolders.repoPathAbsolute, `.git`))
|
||||
) {
|
||||
await OrchestratorSystem.Run(`rm -r ${OrchestratorFolders.repoPathAbsolute}`);
|
||||
OrchestratorLogger.log(`${OrchestratorFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
|
||||
}
|
||||
|
||||
if (
|
||||
BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters) &&
|
||||
fs.existsSync(path.join(OrchestratorFolders.repoPathAbsolute, `.git`))
|
||||
) {
|
||||
process.chdir(OrchestratorFolders.repoPathAbsolute);
|
||||
RemoteClientLogger.log(
|
||||
`${
|
||||
OrchestratorFolders.repoPathAbsolute
|
||||
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
|
||||
Orchestrator.buildParameters,
|
||||
)}`,
|
||||
);
|
||||
await OrchestratorSystem.Run(`git fetch && git reset --hard ${Orchestrator.buildParameters.gitSha}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await OrchestratorSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
await OrchestratorSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
||||
await OrchestratorSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
||||
try {
|
||||
const depthArgument = OrchestratorOptions.cloneDepth !== '0' ? `--depth ${OrchestratorOptions.cloneDepth}` : '';
|
||||
await OrchestratorSystem.Run(
|
||||
`git clone ${depthArgument} ${OrchestratorFolders.targetBuildRepoUrl} ${path.basename(
|
||||
OrchestratorFolders.repoPathAbsolute,
|
||||
)}`.trim(),
|
||||
);
|
||||
} catch (error: any) {
|
||||
throw error;
|
||||
}
|
||||
process.chdir(OrchestratorFolders.repoPathAbsolute);
|
||||
await OrchestratorSystem.Run(`git lfs install`);
|
||||
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||
RemoteClientLogger.log(`${Orchestrator.buildParameters.branch}`);
|
||||
|
||||
// Ensure refs exist (tags and PR refs)
|
||||
await OrchestratorSystem.Run(`git fetch --all --tags || true`);
|
||||
const branchForPrFetch = Orchestrator.buildParameters.branch || '';
|
||||
if (branchForPrFetch.startsWith('pull/')) {
|
||||
// Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731)
|
||||
const prNumber = branchForPrFetch.split('/')[1];
|
||||
if (prNumber) {
|
||||
await OrchestratorSystem.Run(
|
||||
`git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const targetSha = Orchestrator.buildParameters.gitSha;
|
||||
const targetBranch = Orchestrator.buildParameters.branch;
|
||||
if (targetSha) {
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git checkout ${targetSha}`);
|
||||
} catch {
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git fetch origin ${targetSha} || true`);
|
||||
await OrchestratorSystem.Run(`git checkout ${targetSha}`);
|
||||
} catch (error) {
|
||||
RemoteClientLogger.logWarning(`Falling back to branch checkout; SHA not found: ${targetSha}`);
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git checkout ${targetBranch}`);
|
||||
} catch {
|
||||
if ((targetBranch || '').startsWith('pull/')) {
|
||||
await OrchestratorSystem.Run(`git checkout origin/${targetBranch}`);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git checkout ${targetBranch}`);
|
||||
} catch (_error) {
|
||||
if ((targetBranch || '').startsWith('pull/')) {
|
||||
await OrchestratorSystem.Run(`git checkout origin/${targetBranch}`);
|
||||
} else {
|
||||
throw _error;
|
||||
}
|
||||
}
|
||||
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
|
||||
}
|
||||
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${Orchestrator.buildParameters.branch}`);
|
||||
}
|
||||
|
||||
static async replaceLargePackageReferencesWithSharedReferences() {
|
||||
OrchestratorLogger.log(`Use Shared Pkgs ${Orchestrator.buildParameters.useLargePackages}`);
|
||||
GitHub.updateGitHubCheck(`Use Shared Pkgs ${Orchestrator.buildParameters.useLargePackages}`, ``);
|
||||
if (Orchestrator.buildParameters.useLargePackages) {
|
||||
const filePath = path.join(OrchestratorFolders.projectPathAbsolute, `Packages/manifest.json`);
|
||||
let manifest = fs.readFileSync(filePath, 'utf8');
|
||||
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
|
||||
fs.writeFileSync(filePath, manifest);
|
||||
OrchestratorLogger.log(`Package Manifest \n ${manifest}`);
|
||||
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
|
||||
}
|
||||
}
|
||||
|
||||
private static async pullLatestLFS() {
|
||||
process.chdir(OrchestratorFolders.repoPathAbsolute);
|
||||
await OrchestratorSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
||||
await OrchestratorSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
|
||||
if (Orchestrator.buildParameters.skipLfs) {
|
||||
RemoteClientLogger.log(`Skipping LFS pull (skipLfs=true)`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Best effort: try plain pull first (works for public repos or pre-configured auth)
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git lfs pull`, true);
|
||||
await OrchestratorSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Pulled LFS files without explicit token configuration`);
|
||||
|
||||
return;
|
||||
} catch {
|
||||
/* no-op: best-effort git lfs pull without tokens may fail */
|
||||
void 0;
|
||||
}
|
||||
|
||||
// Try with GIT_PRIVATE_TOKEN
|
||||
try {
|
||||
const gitPrivateToken = process.env.GIT_PRIVATE_TOKEN;
|
||||
if (gitPrivateToken) {
|
||||
RemoteClientLogger.log(`Attempting to pull LFS files with GIT_PRIVATE_TOKEN...`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(
|
||||
`git config --global url."https://${gitPrivateToken}@github.com/".insteadOf "https://github.com/"`,
|
||||
);
|
||||
await OrchestratorSystem.Run(`git lfs pull`, true);
|
||||
await OrchestratorSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Successfully pulled LFS files with GIT_PRIVATE_TOKEN`);
|
||||
|
||||
return;
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logCliError(`Failed with GIT_PRIVATE_TOKEN: ${error.message}`);
|
||||
}
|
||||
|
||||
// Try with GITHUB_TOKEN
|
||||
try {
|
||||
const githubToken = process.env.GITHUB_TOKEN;
|
||||
if (githubToken) {
|
||||
RemoteClientLogger.log(`Attempting to pull LFS files with GITHUB_TOKEN fallback...`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||
await OrchestratorSystem.Run(
|
||||
`git config --global url."https://${githubToken}@github.com/".insteadOf "https://github.com/"`,
|
||||
);
|
||||
await OrchestratorSystem.Run(`git lfs pull`, true);
|
||||
await OrchestratorSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Successfully pulled LFS files with GITHUB_TOKEN`);
|
||||
|
||||
return;
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logCliError(`Failed with GITHUB_TOKEN: ${error.message}`);
|
||||
}
|
||||
|
||||
// If we get here, all strategies failed; continue without failing the build
|
||||
RemoteClientLogger.logWarning(`Proceeding without LFS files (no tokens or pull failed)`);
|
||||
}
|
||||
static async handleRetainedWorkspace() {
|
||||
RemoteClientLogger.log(
|
||||
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)}`,
|
||||
);
|
||||
|
||||
// Log cache key explicitly to aid debugging and assertions
|
||||
OrchestratorLogger.log(`Cache Key: ${Orchestrator.buildParameters.cacheKey}`);
|
||||
if (
|
||||
BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters) &&
|
||||
fs.existsSync(OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)) &&
|
||||
fs.existsSync(OrchestratorFolders.ToLinuxFolder(path.join(OrchestratorFolders.repoPathAbsolute, `.git`)))
|
||||
) {
|
||||
OrchestratorLogger.log(`Retained Workspace Already Exists!`);
|
||||
process.chdir(OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.repoPathAbsolute));
|
||||
await OrchestratorSystem.Run(`git fetch --all --tags || true`);
|
||||
const retainedBranchForPrFetch = Orchestrator.buildParameters.branch || '';
|
||||
if (retainedBranchForPrFetch.startsWith('pull/')) {
|
||||
// Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731)
|
||||
const prNumber = retainedBranchForPrFetch.split('/')[1];
|
||||
if (prNumber) {
|
||||
await OrchestratorSystem.Run(
|
||||
`git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await OrchestratorSystem.Run(`git lfs pull`);
|
||||
await OrchestratorSystem.Run(`git lfs checkout || true`);
|
||||
const sha = Orchestrator.buildParameters.gitSha;
|
||||
const branch = Orchestrator.buildParameters.branch;
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git reset --hard "${sha}"`);
|
||||
await OrchestratorSystem.Run(`git checkout ${sha}`);
|
||||
} catch {
|
||||
RemoteClientLogger.logWarning(`Retained workspace: SHA not found, falling back to branch ${branch}`);
|
||||
try {
|
||||
await OrchestratorSystem.Run(`git checkout ${branch}`);
|
||||
} catch (error) {
|
||||
if ((branch || '').startsWith('pull/')) {
|
||||
await OrchestratorSystem.Run(`git checkout origin/${branch}`);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import Orchestrator from '../orchestrator';
|
||||
import OrchestratorOptions from '../options/orchestrator-options';
|
||||
|
||||
export class RemoteClientLogger {
|
||||
private static get LogFilePath() {
|
||||
// Use a cross-platform temporary directory for local development
|
||||
if (process.platform === 'win32') {
|
||||
return path.join(process.cwd(), 'temp', 'job-log.txt');
|
||||
}
|
||||
|
||||
return path.join(`/home`, `job-log.txt`);
|
||||
}
|
||||
|
||||
public static log(message: string) {
|
||||
const finalMessage = `[Client] ${message}`;
|
||||
this.appendToFile(finalMessage);
|
||||
OrchestratorLogger.log(finalMessage);
|
||||
}
|
||||
|
||||
public static logCliError(message: string) {
|
||||
OrchestratorLogger.log(`[Client][Error] ${message}`);
|
||||
}
|
||||
|
||||
public static logCliDiagnostic(message: string) {
|
||||
OrchestratorLogger.log(`[Client][Diagnostic] ${message}`);
|
||||
}
|
||||
|
||||
public static logWarning(message: string) {
|
||||
OrchestratorLogger.logWarning(message);
|
||||
}
|
||||
|
||||
public static appendToFile(message: string) {
|
||||
if (Orchestrator.isOrchestratorEnvironment) {
|
||||
// Ensure the directory exists before writing
|
||||
const logDirectory = path.dirname(RemoteClientLogger.LogFilePath);
|
||||
if (!fs.existsSync(logDirectory)) {
|
||||
fs.mkdirSync(logDirectory, { recursive: true });
|
||||
}
|
||||
|
||||
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async handleLogManagementPostJob() {
|
||||
if (OrchestratorOptions.providerStrategy !== 'k8s') {
|
||||
return;
|
||||
}
|
||||
const collectedLogsMessage = `Collected Logs`;
|
||||
|
||||
// Write to log file first so it's captured even if kubectl has issues
|
||||
// This ensures the message is available in BuildResults when logs are read from the file
|
||||
RemoteClientLogger.appendToFile(collectedLogsMessage);
|
||||
|
||||
// For K8s, write to stdout/stderr so kubectl logs can capture it
|
||||
// This is critical because kubectl logs reads from stdout/stderr, not from GitHub Actions logs
|
||||
// Write multiple times to increase chance of capture if kubectl is having issues
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
// Write to stdout multiple times to increase chance of capture
|
||||
for (let index = 0; index < 3; index++) {
|
||||
process.stdout.write(`${collectedLogsMessage}\n`, 'utf8');
|
||||
process.stderr.write(`${collectedLogsMessage}\n`, 'utf8');
|
||||
}
|
||||
|
||||
// Ensure stdout/stderr are flushed
|
||||
if (!process.stdout.isTTY) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
}
|
||||
|
||||
// Also log via OrchestratorLogger for GitHub Actions
|
||||
OrchestratorLogger.log(collectedLogsMessage);
|
||||
|
||||
// check for log file not existing
|
||||
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
|
||||
const logFileMissingMessage = `Log file does not exist`;
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
process.stdout.write(`${logFileMissingMessage}\n`, 'utf8');
|
||||
}
|
||||
OrchestratorLogger.log(logFileMissingMessage);
|
||||
|
||||
// check if Orchestrator.isOrchestratorEnvironment is true, log
|
||||
if (!Orchestrator.isOrchestratorEnvironment) {
|
||||
const notCloudEnvironmentMessage = `Orchestrator is not running in a cloud environment, not collecting logs`;
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
process.stdout.write(`${notCloudEnvironmentMessage}\n`, 'utf8');
|
||||
}
|
||||
OrchestratorLogger.log(notCloudEnvironmentMessage);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
const logFileExistsMessage = `Log file exist`;
|
||||
if (OrchestratorOptions.providerStrategy === 'k8s') {
|
||||
process.stdout.write(`${logFileExistsMessage}\n`, 'utf8');
|
||||
}
|
||||
OrchestratorLogger.log(logFileExistsMessage);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
|
||||
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||
//
|
||||
// hashedLogs = md5(hashedLogs);
|
||||
//
|
||||
// for (let index = 0; index < 3; index++) {
|
||||
// OrchestratorLogger.log(`LOGHASH: ${hashedLogs}`);
|
||||
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||
// OrchestratorLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
|
||||
// OrchestratorLogger.log(
|
||||
// `Game CI's "Orchestrator System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
|
||||
// );
|
||||
//
|
||||
// // wait for 15 seconds to allow the log to be sent
|
||||
// await new Promise((resolve) => setTimeout(resolve, 15000));
|
||||
// }
|
||||
}
|
||||
public static HandleLog(message: string): boolean {
|
||||
if (RemoteClientLogger.value !== '') {
|
||||
RemoteClientLogger.value += `\n`;
|
||||
}
|
||||
|
||||
RemoteClientLogger.value += message;
|
||||
|
||||
return false;
|
||||
}
|
||||
static value: string = '';
|
||||
}
|
||||
Reference in New Issue
Block a user