mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-04 21:20:15 -07:00
Compare commits
4 Commits
feature/ge
...
feature/in
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d503b0b61a | ||
|
|
07eec62d28 | ||
|
|
4870fb5a5c | ||
|
|
3033ee0067 |
@@ -1,4 +1,5 @@
|
|||||||
{
|
{
|
||||||
|
"root": true,
|
||||||
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
|
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
|
||||||
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
|
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
|
||||||
"parser": "@typescript-eslint/parser",
|
"parser": "@typescript-eslint/parser",
|
||||||
|
|||||||
46
action.yml
46
action.yml
@@ -182,8 +182,8 @@ inputs:
|
|||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
|
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
|
||||||
keys image, secrets (name, value object array), command line string)'
|
the keys image, secrets (name, value object array), command line string)'
|
||||||
awsStackName:
|
awsStackName:
|
||||||
default: 'game-ci'
|
default: 'game-ci'
|
||||||
required: false
|
required: false
|
||||||
@@ -269,28 +269,6 @@ inputs:
|
|||||||
default: 'false'
|
default: 'false'
|
||||||
required: false
|
required: false
|
||||||
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
|
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
|
||||||
artifactOutputTypes:
|
|
||||||
description: 'Comma-separated list of output types to collect (build, logs, test-results, coverage, images, metrics, data-export, server-build, custom)'
|
|
||||||
required: false
|
|
||||||
default: 'build,logs,test-results'
|
|
||||||
artifactUploadTarget:
|
|
||||||
description: 'Where to upload artifacts: github-artifacts, storage, local, none'
|
|
||||||
required: false
|
|
||||||
default: 'github-artifacts'
|
|
||||||
artifactUploadPath:
|
|
||||||
description: 'Destination path for artifact upload (storage URI or local path)'
|
|
||||||
required: false
|
|
||||||
artifactCompression:
|
|
||||||
description: 'Compression for artifacts: none, gzip, lz4'
|
|
||||||
required: false
|
|
||||||
default: 'gzip'
|
|
||||||
artifactRetentionDays:
|
|
||||||
description: 'Retention period for uploaded artifacts in days'
|
|
||||||
required: false
|
|
||||||
default: '30'
|
|
||||||
artifactCustomTypes:
|
|
||||||
description: 'JSON string defining custom output types [{name, defaultPath, description}]'
|
|
||||||
required: false
|
|
||||||
cloneDepth:
|
cloneDepth:
|
||||||
default: '50'
|
default: '50'
|
||||||
required: false
|
required: false
|
||||||
@@ -301,6 +279,24 @@ inputs:
|
|||||||
description:
|
description:
|
||||||
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
|
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
|
||||||
fixes.'
|
fixes.'
|
||||||
|
syncStrategy:
|
||||||
|
description: 'Workspace sync strategy: full, git-delta, direct-input, storage-pull'
|
||||||
|
required: false
|
||||||
|
default: 'full'
|
||||||
|
syncInputRef:
|
||||||
|
description: 'URI for direct-input or storage-pull content (storage://remote/path or file path)'
|
||||||
|
required: false
|
||||||
|
syncStorageRemote:
|
||||||
|
description: 'rclone remote name for storage-backed inputs (defaults to rcloneRemote)'
|
||||||
|
required: false
|
||||||
|
syncRevertAfter:
|
||||||
|
description: 'Revert overlaid changes after job completion'
|
||||||
|
required: false
|
||||||
|
default: 'true'
|
||||||
|
syncStatePath:
|
||||||
|
description: 'Path to sync state file for delta tracking'
|
||||||
|
required: false
|
||||||
|
default: '.game-ci/sync-state.json'
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
volume:
|
volume:
|
||||||
@@ -314,8 +310,6 @@ outputs:
|
|||||||
'Returns the exit code from the build scripts. This code is 0 if the build was successful. If there was an error
|
'Returns the exit code from the build scripts. This code is 0 if the build was successful. If there was an error
|
||||||
during activation, the code is from the activation step. If activation is successful, the code is from the project
|
during activation, the code is from the activation step. If activation is successful, the code is from the project
|
||||||
build step.'
|
build step.'
|
||||||
artifactManifestPath:
|
|
||||||
description: 'Path to the generated artifact manifest JSON file'
|
|
||||||
branding:
|
branding:
|
||||||
icon: 'box'
|
icon: 'box'
|
||||||
color: 'gray-dark'
|
color: 'gray-dark'
|
||||||
|
|||||||
1014
dist/index.js
generated
vendored
1014
dist/index.js
generated
vendored
File diff suppressed because it is too large
Load Diff
2
dist/index.js.map
generated
vendored
2
dist/index.js.map
generated
vendored
File diff suppressed because one or more lines are too long
136
src/index.ts
136
src/index.ts
@@ -1,12 +1,10 @@
|
|||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import path from 'node:path';
|
|
||||||
import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
|
import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
|
||||||
import { Cli } from './model/cli/cli';
|
import { Cli } from './model/cli/cli';
|
||||||
import MacBuilder from './model/mac-builder';
|
import MacBuilder from './model/mac-builder';
|
||||||
import PlatformSetup from './model/platform-setup';
|
import PlatformSetup from './model/platform-setup';
|
||||||
import { OutputService } from './model/orchestrator/services/output/output-service';
|
import { IncrementalSyncService } from './model/orchestrator/services/sync';
|
||||||
import { OutputTypeRegistry } from './model/orchestrator/services/output/output-type-registry';
|
import { SyncStrategy } from './model/orchestrator/services/sync/sync-state';
|
||||||
import { ArtifactUploadHandler } from './model/orchestrator/services/output/artifact-upload-handler';
|
|
||||||
|
|
||||||
async function runMain() {
|
async function runMain() {
|
||||||
try {
|
try {
|
||||||
@@ -27,6 +25,14 @@ async function runMain() {
|
|||||||
|
|
||||||
if (buildParameters.providerStrategy === 'local') {
|
if (buildParameters.providerStrategy === 'local') {
|
||||||
core.info('Building locally');
|
core.info('Building locally');
|
||||||
|
|
||||||
|
// Apply incremental sync strategy before build
|
||||||
|
const syncStrategy = buildParameters.syncStrategy as SyncStrategy;
|
||||||
|
if (syncStrategy !== 'full') {
|
||||||
|
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
|
||||||
|
await applySyncStrategy(buildParameters, workspace);
|
||||||
|
}
|
||||||
|
|
||||||
await PlatformSetup.setup(buildParameters, actionFolder);
|
await PlatformSetup.setup(buildParameters, actionFolder);
|
||||||
exitCode =
|
exitCode =
|
||||||
process.platform === 'darwin'
|
process.platform === 'darwin'
|
||||||
@@ -36,6 +42,16 @@ async function runMain() {
|
|||||||
actionFolder,
|
actionFolder,
|
||||||
...buildParameters,
|
...buildParameters,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Revert overlays after job completion if configured
|
||||||
|
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||||
|
core.info('[Sync] Reverting overlay changes after job completion');
|
||||||
|
try {
|
||||||
|
await IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
|
||||||
|
} catch (revertError) {
|
||||||
|
core.warning(`[Sync] Overlay revert failed: ${(revertError as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
await Orchestrator.run(buildParameters, baseImage.toString());
|
await Orchestrator.run(buildParameters, baseImage.toString());
|
||||||
exitCode = 0;
|
exitCode = 0;
|
||||||
@@ -46,64 +62,6 @@ async function runMain() {
|
|||||||
await Output.setAndroidVersionCode(buildParameters.androidVersionCode);
|
await Output.setAndroidVersionCode(buildParameters.androidVersionCode);
|
||||||
await Output.setEngineExitCode(exitCode);
|
await Output.setEngineExitCode(exitCode);
|
||||||
|
|
||||||
// Artifact collection and upload (runs on both success and failure)
|
|
||||||
try {
|
|
||||||
// Register custom output types if provided
|
|
||||||
if (buildParameters.artifactCustomTypes) {
|
|
||||||
try {
|
|
||||||
const customTypes = JSON.parse(buildParameters.artifactCustomTypes);
|
|
||||||
if (Array.isArray(customTypes)) {
|
|
||||||
for (const ct of customTypes) {
|
|
||||||
OutputTypeRegistry.registerType({
|
|
||||||
name: ct.name,
|
|
||||||
defaultPath: ct.defaultPath || ct.pattern || `./${ct.name}/`,
|
|
||||||
description: ct.description || `Custom output type: ${ct.name}`,
|
|
||||||
builtIn: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (parseError) {
|
|
||||||
core.warning(`Failed to parse artifactCustomTypes: ${(parseError as Error).message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect outputs and generate manifest
|
|
||||||
const manifestPath = path.join(buildParameters.projectPath, 'output-manifest.json');
|
|
||||||
const manifest = await OutputService.collectOutputs(
|
|
||||||
buildParameters.projectPath,
|
|
||||||
buildParameters.buildGuid,
|
|
||||||
buildParameters.artifactOutputTypes,
|
|
||||||
manifestPath,
|
|
||||||
);
|
|
||||||
|
|
||||||
core.setOutput('artifactManifestPath', manifestPath);
|
|
||||||
|
|
||||||
// Upload artifacts
|
|
||||||
const uploadConfig = ArtifactUploadHandler.parseConfig(
|
|
||||||
buildParameters.artifactUploadTarget,
|
|
||||||
buildParameters.artifactUploadPath || undefined,
|
|
||||||
buildParameters.artifactCompression,
|
|
||||||
buildParameters.artifactRetentionDays,
|
|
||||||
);
|
|
||||||
|
|
||||||
const uploadResult = await ArtifactUploadHandler.uploadArtifacts(
|
|
||||||
manifest,
|
|
||||||
uploadConfig,
|
|
||||||
buildParameters.projectPath,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!uploadResult.success) {
|
|
||||||
core.warning(
|
|
||||||
`Artifact upload completed with errors: ${uploadResult.entries
|
|
||||||
.filter((e) => !e.success)
|
|
||||||
.map((e) => `${e.type}: ${e.error}`)
|
|
||||||
.join('; ')}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (artifactError) {
|
|
||||||
core.warning(`Artifact collection/upload failed: ${(artifactError as Error).message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (exitCode !== 0) {
|
if (exitCode !== 0) {
|
||||||
core.setFailed(`Build failed with exit code ${exitCode}`);
|
core.setFailed(`Build failed with exit code ${exitCode}`);
|
||||||
}
|
}
|
||||||
@@ -112,4 +70,58 @@ async function runMain() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply the configured sync strategy to the workspace before build.
|
||||||
|
*/
|
||||||
|
async function applySyncStrategy(buildParameters: BuildParameters, workspace: string): Promise<void> {
|
||||||
|
const strategy = buildParameters.syncStrategy as SyncStrategy;
|
||||||
|
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
|
||||||
|
|
||||||
|
if (resolvedStrategy === 'full') {
|
||||||
|
core.info('[Sync] Resolved to full sync (no incremental state available)');
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (resolvedStrategy) {
|
||||||
|
case 'git-delta': {
|
||||||
|
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||||
|
const changedFiles = await IncrementalSyncService.syncGitDelta(
|
||||||
|
workspace,
|
||||||
|
targetReference,
|
||||||
|
buildParameters.syncStatePath,
|
||||||
|
);
|
||||||
|
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'direct-input': {
|
||||||
|
if (!buildParameters.syncInputRef) {
|
||||||
|
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
|
||||||
|
}
|
||||||
|
const overlays = await IncrementalSyncService.applyDirectInput(
|
||||||
|
workspace,
|
||||||
|
buildParameters.syncInputRef,
|
||||||
|
buildParameters.syncStorageRemote || undefined,
|
||||||
|
buildParameters.syncStatePath,
|
||||||
|
);
|
||||||
|
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'storage-pull': {
|
||||||
|
if (!buildParameters.syncInputRef) {
|
||||||
|
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
|
||||||
|
}
|
||||||
|
const pulledFiles = await IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
|
||||||
|
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||||
|
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||||
|
statePath: buildParameters.syncStatePath,
|
||||||
|
});
|
||||||
|
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
runMain();
|
runMain();
|
||||||
|
|||||||
@@ -106,12 +106,11 @@ class BuildParameters {
|
|||||||
public cacheUnityInstallationOnMac!: boolean;
|
public cacheUnityInstallationOnMac!: boolean;
|
||||||
public unityHubVersionOnMac!: string;
|
public unityHubVersionOnMac!: string;
|
||||||
public dockerWorkspacePath!: string;
|
public dockerWorkspacePath!: string;
|
||||||
public artifactOutputTypes!: string;
|
public syncStrategy!: string;
|
||||||
public artifactUploadTarget!: string;
|
public syncInputRef!: string;
|
||||||
public artifactUploadPath!: string;
|
public syncStorageRemote!: string;
|
||||||
public artifactCompression!: string;
|
public syncRevertAfter!: boolean;
|
||||||
public artifactRetentionDays!: string;
|
public syncStatePath!: string;
|
||||||
public artifactCustomTypes!: string;
|
|
||||||
|
|
||||||
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
||||||
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
|
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
|
||||||
@@ -248,12 +247,11 @@ class BuildParameters {
|
|||||||
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
||||||
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
||||||
dockerWorkspacePath: Input.dockerWorkspacePath,
|
dockerWorkspacePath: Input.dockerWorkspacePath,
|
||||||
artifactOutputTypes: Input.artifactOutputTypes,
|
syncStrategy: Input.syncStrategy,
|
||||||
artifactUploadTarget: Input.artifactUploadTarget,
|
syncInputRef: Input.syncInputRef,
|
||||||
artifactUploadPath: Input.artifactUploadPath,
|
syncStorageRemote: Input.syncStorageRemote,
|
||||||
artifactCompression: Input.artifactCompression,
|
syncRevertAfter: Input.syncRevertAfter,
|
||||||
artifactRetentionDays: Input.artifactRetentionDays,
|
syncStatePath: Input.syncStatePath,
|
||||||
artifactCustomTypes: Input.artifactCustomTypes,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -241,6 +241,28 @@ class Input {
|
|||||||
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
|
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static get syncStrategy(): string {
|
||||||
|
return Input.getInput('syncStrategy') ?? 'full';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get syncInputRef(): string {
|
||||||
|
return Input.getInput('syncInputRef') ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get syncStorageRemote(): string {
|
||||||
|
return Input.getInput('syncStorageRemote') ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get syncRevertAfter(): boolean {
|
||||||
|
const input = Input.getInput('syncRevertAfter') ?? 'true';
|
||||||
|
|
||||||
|
return input === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get syncStatePath(): string {
|
||||||
|
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
|
||||||
|
}
|
||||||
|
|
||||||
static get dockerCpuLimit(): string {
|
static get dockerCpuLimit(): string {
|
||||||
return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
|
return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
|
||||||
}
|
}
|
||||||
@@ -278,30 +300,6 @@ class Input {
|
|||||||
return Input.getInput('containerRegistryImageVersion') ?? '3';
|
return Input.getInput('containerRegistryImageVersion') ?? '3';
|
||||||
}
|
}
|
||||||
|
|
||||||
static get artifactOutputTypes(): string {
|
|
||||||
return Input.getInput('artifactOutputTypes') ?? 'build,logs,test-results';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get artifactUploadTarget(): string {
|
|
||||||
return Input.getInput('artifactUploadTarget') ?? 'github-artifacts';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get artifactUploadPath(): string {
|
|
||||||
return Input.getInput('artifactUploadPath') ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get artifactCompression(): string {
|
|
||||||
return Input.getInput('artifactCompression') ?? 'gzip';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get artifactRetentionDays(): string {
|
|
||||||
return Input.getInput('artifactRetentionDays') ?? '30';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get artifactCustomTypes(): string {
|
|
||||||
return Input.getInput('artifactCustomTypes') ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get skipActivation(): string {
|
static get skipActivation(): string {
|
||||||
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
|
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,15 +15,24 @@ import BuildParameters from '../../build-parameters';
|
|||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import OrchestratorOptions from '../options/orchestrator-options';
|
||||||
import ResourceTracking from '../services/core/resource-tracking';
|
import ResourceTracking from '../services/core/resource-tracking';
|
||||||
|
import { IncrementalSyncService } from '../services/sync';
|
||||||
|
import { SyncStrategy } from '../services/sync/sync-state';
|
||||||
|
|
||||||
export class RemoteClient {
|
export class RemoteClient {
|
||||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||||
static async setupRemoteClient() {
|
static async setupRemoteClient() {
|
||||||
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
|
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
|
||||||
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
|
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
|
||||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
|
||||||
|
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
|
||||||
|
|
||||||
|
if (syncStrategy !== 'full') {
|
||||||
|
OrchestratorLogger.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
|
||||||
|
await RemoteClient.handleIncrementalSync(syncStrategy);
|
||||||
|
} else if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||||
await RemoteClient.bootstrapRepository();
|
await RemoteClient.bootstrapRepository();
|
||||||
}
|
}
|
||||||
|
|
||||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||||
}
|
}
|
||||||
@@ -157,6 +166,20 @@ export class RemoteClient {
|
|||||||
|
|
||||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||||
|
|
||||||
|
// Revert sync overlays if configured
|
||||||
|
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
|
||||||
|
if (Orchestrator.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||||
|
try {
|
||||||
|
OrchestratorLogger.log('[Sync] Reverting overlay changes after job completion');
|
||||||
|
await IncrementalSyncService.revertOverlays(
|
||||||
|
OrchestratorFolders.repoPathAbsolute,
|
||||||
|
Orchestrator.buildParameters.syncStatePath,
|
||||||
|
);
|
||||||
|
} catch (revertError: any) {
|
||||||
|
RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WIP - need to give the pod permissions to create config map
|
// WIP - need to give the pod permissions to create config map
|
||||||
await RemoteClientLogger.handleLogManagementPostJob();
|
await RemoteClientLogger.handleLogManagementPostJob();
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -229,6 +252,78 @@ export class RemoteClient {
|
|||||||
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
|
||||||
|
*
|
||||||
|
* For git-delta: requires an existing workspace with sync state; fetches and applies
|
||||||
|
* only changed files.
|
||||||
|
*
|
||||||
|
* For direct-input and storage-pull: requires an existing workspace; applies overlay
|
||||||
|
* content on top.
|
||||||
|
*
|
||||||
|
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
|
||||||
|
*/
|
||||||
|
private static async handleIncrementalSync(strategy: SyncStrategy): Promise<void> {
|
||||||
|
const buildParameters = Orchestrator.buildParameters;
|
||||||
|
const workspacePath = OrchestratorFolders.repoPathAbsolute;
|
||||||
|
const statePath = buildParameters.syncStatePath;
|
||||||
|
|
||||||
|
// Resolve strategy — may fall back to 'full' if no state exists
|
||||||
|
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
|
||||||
|
|
||||||
|
if (resolvedStrategy === 'full') {
|
||||||
|
OrchestratorLogger.log('[Sync] Falling back to full bootstrap');
|
||||||
|
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||||
|
await RemoteClient.bootstrapRepository();
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (resolvedStrategy) {
|
||||||
|
case 'git-delta': {
|
||||||
|
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||||
|
OrchestratorLogger.log(`[Sync] Git delta sync to ${targetReference}`);
|
||||||
|
const changedFiles = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
|
||||||
|
OrchestratorLogger.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'direct-input': {
|
||||||
|
const inputReference = buildParameters.syncInputRef;
|
||||||
|
if (!inputReference) {
|
||||||
|
throw new Error('[Sync] direct-input strategy requires syncInputRef');
|
||||||
|
}
|
||||||
|
OrchestratorLogger.log(`[Sync] Applying direct input: ${inputReference}`);
|
||||||
|
await IncrementalSyncService.applyDirectInput(
|
||||||
|
workspacePath,
|
||||||
|
inputReference,
|
||||||
|
buildParameters.syncStorageRemote || undefined,
|
||||||
|
statePath,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'storage-pull': {
|
||||||
|
const storageUri = buildParameters.syncInputRef;
|
||||||
|
if (!storageUri) {
|
||||||
|
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
|
||||||
|
}
|
||||||
|
OrchestratorLogger.log(`[Sync] Storage pull from: ${storageUri}`);
|
||||||
|
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
|
||||||
|
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||||
|
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||||
|
statePath,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
OrchestratorLogger.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
|
||||||
|
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||||
|
await RemoteClient.bootstrapRepository();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static async bootstrapRepository() {
|
public static async bootstrapRepository() {
|
||||||
await OrchestratorSystem.Run(
|
await OrchestratorSystem.Run(
|
||||||
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
|
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
|
||||||
|
|||||||
@@ -1,607 +0,0 @@
|
|||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { OutputTypeRegistry, OutputTypeDefinition } from './output-type-registry';
|
|
||||||
import { OutputService } from './output-service';
|
|
||||||
import { OutputManifest } from './output-manifest';
|
|
||||||
import { ArtifactUploadHandler, ArtifactUploadConfig } from './artifact-upload-handler';
|
|
||||||
|
|
||||||
// Mock node:fs
|
|
||||||
jest.mock('node:fs');
|
|
||||||
const mockedFs = fs as jest.Mocked<typeof fs>;
|
|
||||||
|
|
||||||
// Mock @actions/core (used by OrchestratorLogger)
|
|
||||||
jest.mock('@actions/core', () => ({
|
|
||||||
info: jest.fn(),
|
|
||||||
warning: jest.fn(),
|
|
||||||
error: jest.fn(),
|
|
||||||
setOutput: jest.fn(),
|
|
||||||
getInput: jest.fn(),
|
|
||||||
setFailed: jest.fn(),
|
|
||||||
setSecret: jest.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Mock @actions/exec (used by upload handler for rclone)
|
|
||||||
jest.mock('@actions/exec', () => ({
|
|
||||||
exec: jest.fn().mockResolvedValue(0),
|
|
||||||
}));
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
jest.restoreAllMocks();
|
|
||||||
OutputTypeRegistry.resetCustomTypes();
|
|
||||||
});
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// OutputTypeRegistry Tests
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
describe('OutputTypeRegistry', () => {
|
|
||||||
describe('built-in types', () => {
|
|
||||||
it('should have 8 built-in types', () => {
|
|
||||||
const allTypes = OutputTypeRegistry.getAllTypes();
|
|
||||||
const builtInTypes = allTypes.filter((t) => t.builtIn);
|
|
||||||
expect(builtInTypes).toHaveLength(8);
|
|
||||||
});
|
|
||||||
|
|
||||||
it.each(['build', 'test-results', 'server-build', 'data-export', 'images', 'logs', 'metrics', 'coverage'])(
|
|
||||||
'should include built-in type "%s"',
|
|
||||||
(typeName) => {
|
|
||||||
const typeDef = OutputTypeRegistry.getType(typeName);
|
|
||||||
expect(typeDef).toBeDefined();
|
|
||||||
expect(typeDef!.name).toBe(typeName);
|
|
||||||
expect(typeDef!.builtIn).toBe(true);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
it('should return undefined for unknown types', () => {
|
|
||||||
const typeDef = OutputTypeRegistry.getType('nonexistent');
|
|
||||||
expect(typeDef).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should include default paths for all built-in types', () => {
|
|
||||||
const allTypes = OutputTypeRegistry.getAllTypes();
|
|
||||||
for (const typeDef of allTypes) {
|
|
||||||
expect(typeDef.defaultPath).toBeTruthy();
|
|
||||||
expect(typeof typeDef.defaultPath).toBe('string');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should include descriptions for all built-in types', () => {
|
|
||||||
const allTypes = OutputTypeRegistry.getAllTypes();
|
|
||||||
for (const typeDef of allTypes) {
|
|
||||||
expect(typeDef.description).toBeTruthy();
|
|
||||||
expect(typeof typeDef.description).toBe('string');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('custom type registration', () => {
|
|
||||||
it('should register a custom type', () => {
|
|
||||||
const customType: OutputTypeDefinition = {
|
|
||||||
name: 'custom-reports',
|
|
||||||
defaultPath: './Reports/',
|
|
||||||
description: 'Custom generated reports',
|
|
||||||
builtIn: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
OutputTypeRegistry.registerType(customType);
|
|
||||||
const retrieved = OutputTypeRegistry.getType('custom-reports');
|
|
||||||
expect(retrieved).toBeDefined();
|
|
||||||
expect(retrieved!.name).toBe('custom-reports');
|
|
||||||
expect(retrieved!.builtIn).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should not override built-in types', () => {
|
|
||||||
const override: OutputTypeDefinition = {
|
|
||||||
name: 'build',
|
|
||||||
defaultPath: './Override/',
|
|
||||||
description: 'Should not override',
|
|
||||||
builtIn: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
OutputTypeRegistry.registerType(override);
|
|
||||||
const buildType = OutputTypeRegistry.getType('build');
|
|
||||||
expect(buildType!.defaultPath).not.toBe('./Override/');
|
|
||||||
expect(buildType!.builtIn).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should include custom types in getAllTypes', () => {
|
|
||||||
OutputTypeRegistry.registerType({
|
|
||||||
name: 'custom-a',
|
|
||||||
defaultPath: './A/',
|
|
||||||
description: 'Custom A',
|
|
||||||
builtIn: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
const allTypes = OutputTypeRegistry.getAllTypes();
|
|
||||||
expect(allTypes.length).toBe(9); // 8 built-in + 1 custom
|
|
||||||
expect(allTypes.some((t) => t.name === 'custom-a')).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should reset custom types', () => {
|
|
||||||
OutputTypeRegistry.registerType({
|
|
||||||
name: 'temp-type',
|
|
||||||
defaultPath: './Temp/',
|
|
||||||
description: 'Temporary type',
|
|
||||||
builtIn: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(OutputTypeRegistry.getType('temp-type')).toBeDefined();
|
|
||||||
OutputTypeRegistry.resetCustomTypes();
|
|
||||||
expect(OutputTypeRegistry.getType('temp-type')).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should force builtIn to false when registering custom types', () => {
|
|
||||||
OutputTypeRegistry.registerType({
|
|
||||||
name: 'sneaky',
|
|
||||||
defaultPath: './Sneaky/',
|
|
||||||
description: 'Tries to be built-in',
|
|
||||||
builtIn: true, // Intentionally setting to true
|
|
||||||
});
|
|
||||||
|
|
||||||
const retrieved = OutputTypeRegistry.getType('sneaky');
|
|
||||||
expect(retrieved).toBeDefined();
|
|
||||||
expect(retrieved!.builtIn).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('parseOutputTypes', () => {
|
|
||||||
it('should parse a comma-separated string of valid types', () => {
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes('build,logs,coverage');
|
|
||||||
expect(types).toHaveLength(3);
|
|
||||||
expect(types.map((t) => t.name)).toEqual(['build', 'logs', 'coverage']);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should skip unknown types', () => {
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes('build,unknown,logs');
|
|
||||||
expect(types).toHaveLength(2);
|
|
||||||
expect(types.map((t) => t.name)).toEqual(['build', 'logs']);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should handle empty string', () => {
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes('');
|
|
||||||
expect(types).toHaveLength(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should handle whitespace in type names', () => {
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes(' build , logs , coverage ');
|
|
||||||
expect(types).toHaveLength(3);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should include custom types when parsing', () => {
|
|
||||||
OutputTypeRegistry.registerType({
|
|
||||||
name: 'my-reports',
|
|
||||||
defaultPath: './Reports/',
|
|
||||||
description: 'Custom reports',
|
|
||||||
builtIn: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes('build,my-reports');
|
|
||||||
expect(types).toHaveLength(2);
|
|
||||||
expect(types[1].name).toBe('my-reports');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// OutputService Tests
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
describe('OutputService', () => {
|
|
||||||
const projectPath = '/project';
|
|
||||||
const buildGuid = 'test-guid-1234';
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
// Reset all fs mocks
|
|
||||||
mockedFs.existsSync.mockReset();
|
|
||||||
mockedFs.statSync.mockReset();
|
|
||||||
mockedFs.readdirSync.mockReset();
|
|
||||||
mockedFs.writeFileSync.mockReset();
|
|
||||||
mockedFs.mkdirSync.mockReset();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('collectOutputs', () => {
|
|
||||||
it('should return an empty manifest when no output types are declared', async () => {
|
|
||||||
const manifest = await OutputService.collectOutputs(projectPath, buildGuid, '');
|
|
||||||
expect(manifest.buildGuid).toBe(buildGuid);
|
|
||||||
expect(manifest.outputs).toHaveLength(0);
|
|
||||||
expect(manifest.timestamp).toBeTruthy();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should skip outputs where the path does not exist', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(false);
|
|
||||||
|
|
||||||
const manifest = await OutputService.collectOutputs(projectPath, buildGuid, 'build,logs');
|
|
||||||
expect(manifest.outputs).toHaveLength(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should collect directory outputs with file listings', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => true, size: 0 } as any);
|
|
||||||
mockedFs.readdirSync.mockImplementation((_dirPath: any, options?: any) => {
|
|
||||||
if (options?.withFileTypes) {
|
|
||||||
return [
|
|
||||||
{ name: 'file1.txt', isDirectory: () => false },
|
|
||||||
{ name: 'file2.txt', isDirectory: () => false },
|
|
||||||
] as any;
|
|
||||||
}
|
|
||||||
|
|
||||||
return ['file1.txt', 'file2.txt'] as any;
|
|
||||||
});
|
|
||||||
|
|
||||||
const manifest = await OutputService.collectOutputs(projectPath, buildGuid, 'logs');
|
|
||||||
expect(manifest.outputs).toHaveLength(1);
|
|
||||||
expect(manifest.outputs[0].type).toBe('logs');
|
|
||||||
expect(manifest.outputs[0].files).toEqual(['file1.txt', 'file2.txt']);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should collect file output with correct size', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 4096 } as any);
|
|
||||||
|
|
||||||
const manifest = await OutputService.collectOutputs(projectPath, buildGuid, 'coverage');
|
|
||||||
expect(manifest.outputs).toHaveLength(1);
|
|
||||||
expect(manifest.outputs[0].size).toBe(4096);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should write manifest to disk when manifestPath is provided', async () => {
|
|
||||||
// existsSync returns false for output paths (no outputs found) but mkdirSync/writeFileSync should still be called
|
|
||||||
// The service only writes manifest when at least one output type is declared and types are resolved
|
|
||||||
// So we need to provide a valid output type and have its path exist
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 100 } as any);
|
|
||||||
mockedFs.mkdirSync.mockReturnValue(undefined);
|
|
||||||
mockedFs.writeFileSync.mockImplementation(() => {});
|
|
||||||
|
|
||||||
const manifestPath = '/output/manifest.json';
|
|
||||||
await OutputService.collectOutputs(projectPath, buildGuid, 'logs', manifestPath);
|
|
||||||
|
|
||||||
expect(mockedFs.mkdirSync).toHaveBeenCalledWith(path.dirname(manifestPath), { recursive: true });
|
|
||||||
expect(mockedFs.writeFileSync).toHaveBeenCalledWith(manifestPath, expect.any(String), 'utf8');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should generate valid JSON in the manifest file', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 200 } as any);
|
|
||||||
mockedFs.mkdirSync.mockReturnValue(undefined);
|
|
||||||
mockedFs.writeFileSync.mockImplementation(() => {});
|
|
||||||
|
|
||||||
const manifestPath = '/output/manifest.json';
|
|
||||||
await OutputService.collectOutputs(projectPath, buildGuid, 'coverage', manifestPath);
|
|
||||||
|
|
||||||
const writtenContent = (mockedFs.writeFileSync as jest.Mock).mock.calls[0][1];
|
|
||||||
const parsed = JSON.parse(writtenContent);
|
|
||||||
expect(parsed.buildGuid).toBe(buildGuid);
|
|
||||||
expect(Array.isArray(parsed.outputs)).toBe(true);
|
|
||||||
expect(parsed.outputs.length).toBeGreaterThan(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should set a valid ISO 8601 timestamp', async () => {
|
|
||||||
const manifest = await OutputService.collectOutputs(projectPath, buildGuid, '');
|
|
||||||
const parsed = new Date(manifest.timestamp);
|
|
||||||
expect(parsed.toISOString()).toBe(manifest.timestamp);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// ArtifactUploadHandler Tests
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
describe('ArtifactUploadHandler', () => {
|
|
||||||
const projectPath = '/project';
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
mockedFs.existsSync.mockReset();
|
|
||||||
mockedFs.statSync.mockReset();
|
|
||||||
mockedFs.readdirSync.mockReset();
|
|
||||||
mockedFs.mkdirSync.mockReset();
|
|
||||||
mockedFs.copyFileSync.mockReset();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('parseConfig', () => {
|
|
||||||
it('should parse valid config values', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('github-artifacts', '/dest', 'gzip', '14');
|
|
||||||
expect(config.target).toBe('github-artifacts');
|
|
||||||
expect(config.destination).toBe('/dest');
|
|
||||||
expect(config.compression).toBe('gzip');
|
|
||||||
expect(config.retentionDays).toBe(14);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should default invalid target to github-artifacts', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('invalid', undefined, 'none', '30');
|
|
||||||
expect(config.target).toBe('github-artifacts');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should default invalid compression to gzip', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('local', '/dest', 'brotli', '30');
|
|
||||||
expect(config.compression).toBe('gzip');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should default invalid retention to 30 days', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('local', '/dest', 'gzip', 'abc');
|
|
||||||
expect(config.retentionDays).toBe(30);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should default negative retention to 30 days', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('local', '/dest', 'gzip', '-5');
|
|
||||||
expect(config.retentionDays).toBe(30);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should set destination to undefined when empty string', () => {
|
|
||||||
const config = ArtifactUploadHandler.parseConfig('storage', '', 'none', '7');
|
|
||||||
expect(config.destination).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('uploadArtifacts', () => {
|
|
||||||
it('should skip upload when target is none', async () => {
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/' }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'none',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(true);
|
|
||||||
expect(result.entries).toHaveLength(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return success with no entries for empty manifest', async () => {
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'github-artifacts',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(true);
|
|
||||||
expect(result.entries).toHaveLength(0);
|
|
||||||
expect(result.totalBytes).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail entry when output path does not exist', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(false);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/Missing/' }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'local',
|
|
||||||
destination: '/output',
|
|
||||||
compression: 'none',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(false);
|
|
||||||
expect(result.entries).toHaveLength(1);
|
|
||||||
expect(result.entries[0].success).toBe(false);
|
|
||||||
expect(result.entries[0].error).toContain('does not exist');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should copy files for local upload target', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 1024 } as any);
|
|
||||||
mockedFs.mkdirSync.mockReturnValue(undefined);
|
|
||||||
mockedFs.copyFileSync.mockReturnValue(undefined);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'logs', path: './Logs/build.log', size: 1024 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'local',
|
|
||||||
destination: '/output',
|
|
||||||
compression: 'none',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(true);
|
|
||||||
expect(result.entries).toHaveLength(1);
|
|
||||||
expect(result.entries[0].success).toBe(true);
|
|
||||||
expect(result.totalBytes).toBe(1024);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail local upload when no destination is provided', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 512 } as any);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'logs', path: './Logs/build.log', size: 512 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'local',
|
|
||||||
compression: 'none',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(false);
|
|
||||||
expect(result.entries[0].success).toBe(false);
|
|
||||||
expect(result.entries[0].error).toContain('destination path');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should report correct duration', async () => {
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'none',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('collectFiles', () => {
|
|
||||||
it('should return single file for a file path', () => {
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false } as any);
|
|
||||||
|
|
||||||
const files = ArtifactUploadHandler.collectFiles('/path/to/file.txt');
|
|
||||||
expect(files).toEqual(['/path/to/file.txt']);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return all files recursively for a directory', () => {
|
|
||||||
mockedFs.statSync.mockImplementation((p: any) => {
|
|
||||||
const pathStr = typeof p === 'string' ? p : p.toString();
|
|
||||||
if (pathStr.endsWith('.txt') || pathStr.endsWith('.log')) {
|
|
||||||
return { isDirectory: () => false } as any;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { isDirectory: () => true } as any;
|
|
||||||
});
|
|
||||||
|
|
||||||
mockedFs.readdirSync.mockImplementation((dirPath: any, _options?: any) => {
|
|
||||||
const dirStr = typeof dirPath === 'string' ? dirPath : dirPath.toString();
|
|
||||||
if (dirStr === '/root') {
|
|
||||||
return [
|
|
||||||
{ name: 'file1.txt', isDirectory: () => false },
|
|
||||||
{ name: 'sub', isDirectory: () => true },
|
|
||||||
] as any;
|
|
||||||
}
|
|
||||||
if (dirStr.endsWith('sub')) {
|
|
||||||
return [{ name: 'file2.log', isDirectory: () => false }] as any;
|
|
||||||
}
|
|
||||||
|
|
||||||
return [] as any;
|
|
||||||
});
|
|
||||||
|
|
||||||
const files = ArtifactUploadHandler.collectFiles('/root');
|
|
||||||
expect(files).toHaveLength(2);
|
|
||||||
expect(files).toContain(path.join('/root', 'file1.txt'));
|
|
||||||
expect(files).toContain(path.join('/root', 'sub', 'file2.log'));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('storage upload validation', () => {
|
|
||||||
it('should fail storage upload when no destination is provided', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 256 } as any);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/', size: 256 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'storage',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(false);
|
|
||||||
expect(result.entries[0].error).toContain('destination URI');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail storage upload when destination URI has invalid format', async () => {
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 256 } as any);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/', size: 256 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'storage',
|
|
||||||
destination: '/just/a/local/path',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(false);
|
|
||||||
expect(result.entries[0].error).toContain('Invalid storage destination URI');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail storage upload when rclone is not installed', async () => {
|
|
||||||
// Mock child_process.execFileSync to throw (rclone not found)
|
|
||||||
const childProcess = require('node:child_process');
|
|
||||||
const originalExecFileSync = childProcess.execFileSync;
|
|
||||||
childProcess.execFileSync = jest.fn(() => {
|
|
||||||
throw new Error('ENOENT');
|
|
||||||
});
|
|
||||||
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 256 } as any);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/', size: 256 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'storage',
|
|
||||||
destination: 's3:my-bucket/artifacts',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
expect(result.success).toBe(false);
|
|
||||||
expect(result.entries[0].error).toContain('rclone is not installed');
|
|
||||||
|
|
||||||
// Restore
|
|
||||||
childProcess.execFileSync = originalExecFileSync;
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should accept valid rclone storage URI formats', async () => {
|
|
||||||
// Mock child_process.execFileSync to succeed (rclone available)
|
|
||||||
const childProcess = require('node:child_process');
|
|
||||||
const originalExecFileSync = childProcess.execFileSync;
|
|
||||||
childProcess.execFileSync = jest.fn(() => 'rclone v1.65.0');
|
|
||||||
|
|
||||||
mockedFs.existsSync.mockReturnValue(true);
|
|
||||||
mockedFs.statSync.mockReturnValue({ isDirectory: () => false, size: 256 } as any);
|
|
||||||
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid: 'test-guid',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [{ type: 'build', path: './Builds/', size: 256 }],
|
|
||||||
};
|
|
||||||
|
|
||||||
// s3:bucket format should pass URI validation and reach the exec call
|
|
||||||
const config: ArtifactUploadConfig = {
|
|
||||||
target: 'storage',
|
|
||||||
destination: 's3:my-bucket/artifacts',
|
|
||||||
compression: 'gzip',
|
|
||||||
retentionDays: 30,
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await ArtifactUploadHandler.uploadArtifacts(manifest, config, projectPath);
|
|
||||||
// Should succeed because exec is mocked to return 0
|
|
||||||
expect(result.entries[0].success).toBe(true);
|
|
||||||
|
|
||||||
// Restore
|
|
||||||
childProcess.execFileSync = originalExecFileSync;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,474 +0,0 @@
|
|||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { execFileSync } from 'node:child_process';
|
|
||||||
import { exec } from '@actions/exec';
|
|
||||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
|
||||||
import { OutputManifest, OutputEntry } from './output-manifest';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration for artifact upload.
|
|
||||||
*/
|
|
||||||
export interface ArtifactUploadConfig {
|
|
||||||
/** Upload target: 'github-artifacts', 'storage', 'local', 'none' */
|
|
||||||
target: 'github-artifacts' | 'storage' | 'local' | 'none';
|
|
||||||
|
|
||||||
/** Destination path — storage URI for 'storage', local path for 'local' */
|
|
||||||
destination?: string;
|
|
||||||
|
|
||||||
/** Compression method */
|
|
||||||
compression: 'none' | 'gzip' | 'lz4';
|
|
||||||
|
|
||||||
/** Retention period in days (GitHub Artifacts only) */
|
|
||||||
retentionDays: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Result of an artifact upload operation.
|
|
||||||
*/
|
|
||||||
export interface UploadResult {
|
|
||||||
/** Whether the upload succeeded overall */
|
|
||||||
success: boolean;
|
|
||||||
|
|
||||||
/** Per-entry upload results */
|
|
||||||
entries: UploadEntryResult[];
|
|
||||||
|
|
||||||
/** Total bytes uploaded */
|
|
||||||
totalBytes: number;
|
|
||||||
|
|
||||||
/** Duration in milliseconds */
|
|
||||||
durationMs: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface UploadEntryResult {
|
|
||||||
/** The output type name */
|
|
||||||
type: string;
|
|
||||||
|
|
||||||
/** The output path */
|
|
||||||
path: string;
|
|
||||||
|
|
||||||
/** Whether this entry uploaded successfully */
|
|
||||||
success: boolean;
|
|
||||||
|
|
||||||
/** Bytes uploaded for this entry */
|
|
||||||
bytes: number;
|
|
||||||
|
|
||||||
/** Error message if upload failed */
|
|
||||||
error?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* GitHub Artifacts size limit per artifact (10 GB).
|
|
||||||
* Files larger than this must be split.
|
|
||||||
*/
|
|
||||||
const GITHUB_ARTIFACT_SIZE_LIMIT = 10 * 1024 * 1024 * 1024;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Minimum valid storage URI pattern: "remote:path" or "remote:".
|
|
||||||
* rclone requires at least a remote name followed by a colon.
|
|
||||||
*/
|
|
||||||
const STORAGE_URI_PATTERN = /^[a-zA-Z][\w-]*:/;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check whether rclone is installed and available on PATH.
|
|
||||||
* Returns true if `rclone version` executes successfully.
|
|
||||||
*/
|
|
||||||
function isRcloneAvailable(): boolean {
|
|
||||||
try {
|
|
||||||
execFileSync('rclone', ['version'], { stdio: 'pipe', timeout: 5000 });
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validate that a storage destination URI has the correct rclone format.
|
|
||||||
* Valid format: "remoteName:path" (e.g., "s3:bucket/prefix", "gdrive:folder").
|
|
||||||
*/
|
|
||||||
function isValidStorageUri(uri: string): boolean {
|
|
||||||
return STORAGE_URI_PATTERN.test(uri);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handles uploading build artifacts to various targets.
|
|
||||||
*/
|
|
||||||
export class ArtifactUploadHandler {
|
|
||||||
/**
|
|
||||||
* Upload artifacts described by a manifest to the configured target.
|
|
||||||
*/
|
|
||||||
static async uploadArtifacts(
|
|
||||||
manifest: OutputManifest,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
projectPath: string,
|
|
||||||
): Promise<UploadResult> {
|
|
||||||
const startTime = Date.now();
|
|
||||||
const result: UploadResult = {
|
|
||||||
success: true,
|
|
||||||
entries: [],
|
|
||||||
totalBytes: 0,
|
|
||||||
durationMs: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (config.target === 'none') {
|
|
||||||
OrchestratorLogger.log('[ArtifactUpload] Upload target is "none", skipping upload');
|
|
||||||
result.durationMs = Date.now() - startTime;
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (manifest.outputs.length === 0) {
|
|
||||||
OrchestratorLogger.log('[ArtifactUpload] No outputs in manifest, nothing to upload');
|
|
||||||
result.durationMs = Date.now() - startTime;
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
OrchestratorLogger.log(`[ArtifactUpload] Uploading ${manifest.outputs.length} output(s) to ${config.target}`);
|
|
||||||
|
|
||||||
for (const entry of manifest.outputs) {
|
|
||||||
const entryResult = await ArtifactUploadHandler.uploadEntry(entry, config, projectPath);
|
|
||||||
result.entries.push(entryResult);
|
|
||||||
result.totalBytes += entryResult.bytes;
|
|
||||||
|
|
||||||
if (!entryResult.success) {
|
|
||||||
result.success = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result.durationMs = Date.now() - startTime;
|
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[ArtifactUpload] Upload complete: ${result.entries.filter((e) => e.success).length}/${
|
|
||||||
result.entries.length
|
|
||||||
} succeeded, ${result.totalBytes} bytes, ${result.durationMs}ms`,
|
|
||||||
);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Upload a single output entry.
|
|
||||||
*/
|
|
||||||
private static async uploadEntry(
|
|
||||||
entry: OutputEntry,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
projectPath: string,
|
|
||||||
): Promise<UploadEntryResult> {
|
|
||||||
const entryResult: UploadEntryResult = {
|
|
||||||
type: entry.type,
|
|
||||||
path: entry.path,
|
|
||||||
success: false,
|
|
||||||
bytes: entry.size || 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
const resolvedPath = path.resolve(
|
|
||||||
projectPath,
|
|
||||||
entry.path.replace('{platform}', process.env.BUILD_TARGET || 'Unknown'),
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!fs.existsSync(resolvedPath)) {
|
|
||||||
entryResult.error = `Output path does not exist: ${resolvedPath}`;
|
|
||||||
OrchestratorLogger.logWarning(`[ArtifactUpload] ${entryResult.error}`);
|
|
||||||
|
|
||||||
return entryResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
switch (config.target) {
|
|
||||||
case 'github-artifacts':
|
|
||||||
await ArtifactUploadHandler.uploadToGitHubArtifacts(entry, resolvedPath, config);
|
|
||||||
break;
|
|
||||||
case 'storage':
|
|
||||||
await ArtifactUploadHandler.uploadToStorage(entry, resolvedPath, config);
|
|
||||||
break;
|
|
||||||
case 'local':
|
|
||||||
await ArtifactUploadHandler.uploadToLocal(entry, resolvedPath, config);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
entryResult.success = true;
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[ArtifactUpload] Uploaded '${entry.type}' (${entryResult.bytes} bytes) to ${config.target}`,
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
entryResult.error = error.message || String(error);
|
|
||||||
OrchestratorLogger.logWarning(`[ArtifactUpload] Failed to upload '${entry.type}': ${entryResult.error}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return entryResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Upload to GitHub Artifacts via @actions/artifact.
|
|
||||||
* Handles large file splitting if artifacts exceed the size limit.
|
|
||||||
*/
|
|
||||||
private static async uploadToGitHubArtifacts(
|
|
||||||
entry: OutputEntry,
|
|
||||||
resolvedPath: string,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
// Dynamically require @actions/artifact — it may not be available in all environments.
|
|
||||||
// Using a variable to prevent TypeScript from resolving the module at compile time.
|
|
||||||
let artifact: any;
|
|
||||||
try {
|
|
||||||
const artifactModule = '@actions/artifact';
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
||||||
artifact = require(artifactModule);
|
|
||||||
} catch {
|
|
||||||
throw new Error('@actions/artifact package is not available. Install it to use github-artifacts upload target.');
|
|
||||||
}
|
|
||||||
|
|
||||||
const artifactClient = artifact.DefaultArtifactClient
|
|
||||||
? new artifact.DefaultArtifactClient()
|
|
||||||
: artifact.default
|
|
||||||
? new artifact.default()
|
|
||||||
: artifact;
|
|
||||||
|
|
||||||
const files = ArtifactUploadHandler.collectFiles(resolvedPath);
|
|
||||||
|
|
||||||
if (files.length === 0) {
|
|
||||||
OrchestratorLogger.logWarning(`[ArtifactUpload] No files found at ${resolvedPath} for '${entry.type}'`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const totalSize = entry.size || 0;
|
|
||||||
const artifactName = `unity-output-${entry.type}`;
|
|
||||||
|
|
||||||
if (totalSize > GITHUB_ARTIFACT_SIZE_LIMIT) {
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[ArtifactUpload] Output '${entry.type}' exceeds GitHub Artifacts size limit (${totalSize} > ${GITHUB_ARTIFACT_SIZE_LIMIT}), splitting into chunks`,
|
|
||||||
);
|
|
||||||
await ArtifactUploadHandler.uploadChunked(artifactClient, artifactName, files, resolvedPath, config);
|
|
||||||
} else {
|
|
||||||
const rootDirectory = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : path.dirname(resolvedPath);
|
|
||||||
|
|
||||||
if (typeof artifactClient.uploadArtifact === 'function') {
|
|
||||||
await artifactClient.uploadArtifact(artifactName, files, rootDirectory, {
|
|
||||||
retentionDays: config.retentionDays,
|
|
||||||
compressionLevel: config.compression === 'none' ? 0 : 6,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
throw new Error(
|
|
||||||
'@actions/artifact client does not have uploadArtifact method. Ensure the package version is compatible.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Upload large artifacts in chunks to stay within GitHub size limits.
|
|
||||||
*/
|
|
||||||
private static async uploadChunked(
|
|
||||||
artifactClient: any,
|
|
||||||
baseName: string,
|
|
||||||
files: string[],
|
|
||||||
rootDirectory: string,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
const chunkSize = GITHUB_ARTIFACT_SIZE_LIMIT;
|
|
||||||
let currentChunkFiles: string[] = [];
|
|
||||||
let currentChunkSize = 0;
|
|
||||||
let chunkIndex = 0;
|
|
||||||
|
|
||||||
for (const filePath of files) {
|
|
||||||
const fileSize = fs.statSync(filePath).size;
|
|
||||||
|
|
||||||
if (currentChunkSize + fileSize > chunkSize && currentChunkFiles.length > 0) {
|
|
||||||
await ArtifactUploadHandler.uploadSingleChunk(
|
|
||||||
artifactClient,
|
|
||||||
`${baseName}-part${chunkIndex}`,
|
|
||||||
currentChunkFiles,
|
|
||||||
rootDirectory,
|
|
||||||
config,
|
|
||||||
);
|
|
||||||
chunkIndex++;
|
|
||||||
currentChunkFiles = [];
|
|
||||||
currentChunkSize = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
currentChunkFiles.push(filePath);
|
|
||||||
currentChunkSize += fileSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (currentChunkFiles.length > 0) {
|
|
||||||
await ArtifactUploadHandler.uploadSingleChunk(
|
|
||||||
artifactClient,
|
|
||||||
chunkIndex > 0 ? `${baseName}-part${chunkIndex}` : baseName,
|
|
||||||
currentChunkFiles,
|
|
||||||
rootDirectory,
|
|
||||||
config,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async uploadSingleChunk(
|
|
||||||
artifactClient: any,
|
|
||||||
name: string,
|
|
||||||
files: string[],
|
|
||||||
rootDirectory: string,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
OrchestratorLogger.log(`[ArtifactUpload] Uploading chunk '${name}' with ${files.length} file(s)`);
|
|
||||||
|
|
||||||
if (typeof artifactClient.uploadArtifact === 'function') {
|
|
||||||
await artifactClient.uploadArtifact(name, files, rootDirectory, {
|
|
||||||
retentionDays: config.retentionDays,
|
|
||||||
compressionLevel: config.compression === 'none' ? 0 : 6,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Upload to remote storage via rclone.
|
|
||||||
*
|
|
||||||
* Validates rclone availability and destination URI format before attempting
|
|
||||||
* the upload. If rclone is not installed, falls back to local copy when a
|
|
||||||
* local-compatible destination is provided, or skips with a clear error.
|
|
||||||
*/
|
|
||||||
private static async uploadToStorage(
|
|
||||||
entry: OutputEntry,
|
|
||||||
resolvedPath: string,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
if (!config.destination) {
|
|
||||||
throw new Error('Storage upload requires a destination URI in artifactUploadPath');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate storage URI format before attempting upload
|
|
||||||
if (!isValidStorageUri(config.destination)) {
|
|
||||||
throw new Error(
|
|
||||||
`Invalid storage destination URI: "${config.destination}". ` +
|
|
||||||
'Expected rclone remote format "remoteName:path" (e.g., "s3:my-bucket/artifacts", "gdrive:builds").',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check rclone availability before attempting upload
|
|
||||||
if (!isRcloneAvailable()) {
|
|
||||||
OrchestratorLogger.error(
|
|
||||||
'rclone is not installed or not in PATH. ' +
|
|
||||||
'Install rclone (https://rclone.org/install/) to use storage-based artifact upload. ' +
|
|
||||||
'Falling back to local copy.',
|
|
||||||
);
|
|
||||||
|
|
||||||
// Attempt local copy fallback using the destination as a hint
|
|
||||||
// Strip the remote prefix to get a local-ish path for fallback
|
|
||||||
OrchestratorLogger.logWarning(
|
|
||||||
`[ArtifactUpload] Storage upload skipped for '${entry.type}' — rclone not available`,
|
|
||||||
);
|
|
||||||
throw new Error(
|
|
||||||
'rclone is not installed or not in PATH. ' +
|
|
||||||
'Install rclone from https://rclone.org/install/ to use storage-based artifact upload.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const destination = `${config.destination}/${entry.type}`;
|
|
||||||
|
|
||||||
OrchestratorLogger.log(`[ArtifactUpload] Uploading '${entry.type}' to storage: ${destination}`);
|
|
||||||
|
|
||||||
const args = ['copy', resolvedPath, destination, '--progress'];
|
|
||||||
|
|
||||||
if (config.compression !== 'none') {
|
|
||||||
// rclone doesn't have built-in compression flags for copy;
|
|
||||||
// compression is typically handled by the remote configuration.
|
|
||||||
// Log as informational.
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[ArtifactUpload] Note: compression '${config.compression}' is configured at the remote level for rclone`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await exec('rclone', args);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Upload to a local path (copy).
|
|
||||||
*/
|
|
||||||
private static async uploadToLocal(
|
|
||||||
entry: OutputEntry,
|
|
||||||
resolvedPath: string,
|
|
||||||
config: ArtifactUploadConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
if (!config.destination) {
|
|
||||||
throw new Error('Local upload requires a destination path in artifactUploadPath');
|
|
||||||
}
|
|
||||||
|
|
||||||
const destination = path.join(config.destination, entry.type);
|
|
||||||
fs.mkdirSync(destination, { recursive: true });
|
|
||||||
|
|
||||||
OrchestratorLogger.log(`[ArtifactUpload] Copying '${entry.type}' to local path: ${destination}`);
|
|
||||||
|
|
||||||
ArtifactUploadHandler.copyRecursive(resolvedPath, destination);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Recursively copy files from source to destination.
|
|
||||||
*/
|
|
||||||
private static copyRecursive(source: string, destination: string): void {
|
|
||||||
const stat = fs.statSync(source);
|
|
||||||
|
|
||||||
if (stat.isDirectory()) {
|
|
||||||
fs.mkdirSync(destination, { recursive: true });
|
|
||||||
const entries = fs.readdirSync(source);
|
|
||||||
for (const entry of entries) {
|
|
||||||
ArtifactUploadHandler.copyRecursive(path.join(source, entry), path.join(destination, entry));
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
fs.copyFileSync(source, destination);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Collect all files at a given path (recursively if directory).
|
|
||||||
*/
|
|
||||||
static collectFiles(targetPath: string): string[] {
|
|
||||||
const stat = fs.statSync(targetPath);
|
|
||||||
|
|
||||||
if (!stat.isDirectory()) {
|
|
||||||
return [targetPath];
|
|
||||||
}
|
|
||||||
|
|
||||||
const files: string[] = [];
|
|
||||||
const entries = fs.readdirSync(targetPath, { withFileTypes: true });
|
|
||||||
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path.join(targetPath, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
files.push(...ArtifactUploadHandler.collectFiles(fullPath));
|
|
||||||
} else {
|
|
||||||
files.push(fullPath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return files;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse an ArtifactUploadConfig from action inputs.
|
|
||||||
*/
|
|
||||||
static parseConfig(
|
|
||||||
target: string,
|
|
||||||
destination: string | undefined,
|
|
||||||
compression: string,
|
|
||||||
retentionDays: string,
|
|
||||||
): ArtifactUploadConfig {
|
|
||||||
const validTargets = ['github-artifacts', 'storage', 'local', 'none'] as const;
|
|
||||||
const resolvedTarget = validTargets.includes(target as any)
|
|
||||||
? (target as ArtifactUploadConfig['target'])
|
|
||||||
: 'github-artifacts';
|
|
||||||
|
|
||||||
const validCompressions = ['none', 'gzip', 'lz4'] as const;
|
|
||||||
const resolvedCompression = validCompressions.includes(compression as any)
|
|
||||||
? (compression as ArtifactUploadConfig['compression'])
|
|
||||||
: 'gzip';
|
|
||||||
|
|
||||||
const parsedRetention = Number.parseInt(retentionDays, 10);
|
|
||||||
const resolvedRetention = Number.isNaN(parsedRetention) || parsedRetention <= 0 ? 30 : parsedRetention;
|
|
||||||
|
|
||||||
return {
|
|
||||||
target: resolvedTarget,
|
|
||||||
destination: destination || undefined,
|
|
||||||
compression: resolvedCompression,
|
|
||||||
retentionDays: resolvedRetention,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
export { OutputManifest, OutputEntry } from './output-manifest';
|
|
||||||
export { OutputTypeRegistry, OutputTypeDefinition } from './output-type-registry';
|
|
||||||
export { OutputService } from './output-service';
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
/**
|
|
||||||
* Structured build output manifest.
|
|
||||||
* Describes all artifacts produced by a build with type, path, size, hash, and metadata.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export interface OutputEntry {
|
|
||||||
/** Output type identifier (e.g., 'build', 'test-results', 'images') */
|
|
||||||
type: string;
|
|
||||||
|
|
||||||
/** Relative path to the output */
|
|
||||||
path: string;
|
|
||||||
|
|
||||||
/** Output format (e.g., 'nunit3', 'junit', 'json') */
|
|
||||||
format?: string;
|
|
||||||
|
|
||||||
/** File size in bytes */
|
|
||||||
size?: number;
|
|
||||||
|
|
||||||
/** Content hash (e.g., 'sha256:abc...') */
|
|
||||||
hash?: string;
|
|
||||||
|
|
||||||
/** Individual files within the output path */
|
|
||||||
files?: string[];
|
|
||||||
|
|
||||||
/** Type-specific summary (e.g., test counts, build size) */
|
|
||||||
summary?: Record<string, unknown>;
|
|
||||||
|
|
||||||
/** Arbitrary metadata */
|
|
||||||
metadata?: Record<string, unknown>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OutputManifest {
|
|
||||||
/** Unique build identifier */
|
|
||||||
buildGuid: string;
|
|
||||||
|
|
||||||
/** ISO 8601 timestamp */
|
|
||||||
timestamp: string;
|
|
||||||
|
|
||||||
/** All outputs produced by this build */
|
|
||||||
outputs: OutputEntry[];
|
|
||||||
}
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
|
||||||
import { OutputManifest, OutputEntry } from './output-manifest';
|
|
||||||
import { OutputTypeRegistry } from './output-type-registry';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Service for collecting, manifesting, and managing build outputs.
|
|
||||||
*
|
|
||||||
* After a build completes, this service scans declared output paths,
|
|
||||||
* generates a structured manifest, and prepares outputs for post-processing.
|
|
||||||
*/
|
|
||||||
export class OutputService {
|
|
||||||
/**
|
|
||||||
* Collect outputs from the workspace and generate a manifest.
|
|
||||||
*
|
|
||||||
* @param projectPath - Path to the Unity project root
|
|
||||||
* @param buildGuid - Unique build identifier
|
|
||||||
* @param outputTypesInput - Comma-separated output type names
|
|
||||||
* @param manifestPath - Where to write the manifest JSON (optional)
|
|
||||||
* @returns The generated output manifest
|
|
||||||
*/
|
|
||||||
static async collectOutputs(
|
|
||||||
projectPath: string,
|
|
||||||
buildGuid: string,
|
|
||||||
outputTypesInput: string,
|
|
||||||
manifestPath?: string,
|
|
||||||
): Promise<OutputManifest> {
|
|
||||||
const types = OutputTypeRegistry.parseOutputTypes(outputTypesInput);
|
|
||||||
const manifest: OutputManifest = {
|
|
||||||
buildGuid,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
outputs: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
if (types.length === 0) {
|
|
||||||
OrchestratorLogger.log('[Output] No output types declared, skipping collection');
|
|
||||||
|
|
||||||
return manifest;
|
|
||||||
}
|
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[Output] Collecting ${types.length} output type(s): ${types.map((t) => t.name).join(', ')}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const typeDef of types) {
|
|
||||||
const outputPath = path.join(
|
|
||||||
projectPath,
|
|
||||||
typeDef.defaultPath.replace('{platform}', process.env.BUILD_TARGET || 'Unknown'),
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!fs.existsSync(outputPath)) {
|
|
||||||
OrchestratorLogger.log(`[Output] No output found for '${typeDef.name}' at ${outputPath}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const entry: OutputEntry = {
|
|
||||||
type: typeDef.name,
|
|
||||||
path: typeDef.defaultPath,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Collect file listing for directory outputs
|
|
||||||
try {
|
|
||||||
const stat = fs.statSync(outputPath);
|
|
||||||
if (stat.isDirectory()) {
|
|
||||||
entry.files = fs.readdirSync(outputPath);
|
|
||||||
entry.size = OutputService.getDirectorySize(outputPath);
|
|
||||||
} else {
|
|
||||||
entry.size = stat.size;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
OrchestratorLogger.logWarning(`[Output] Failed to stat output '${typeDef.name}' at ${outputPath}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
manifest.outputs.push(entry);
|
|
||||||
OrchestratorLogger.log(
|
|
||||||
`[Output] Collected '${typeDef.name}': ${entry.files?.length || 1} file(s), ${entry.size || 0} bytes`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write manifest to disk
|
|
||||||
if (manifestPath) {
|
|
||||||
try {
|
|
||||||
const manifestDir = path.dirname(manifestPath);
|
|
||||||
fs.mkdirSync(manifestDir, { recursive: true });
|
|
||||||
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
|
|
||||||
OrchestratorLogger.log(`[Output] Manifest written to ${manifestPath}`);
|
|
||||||
} catch (error: any) {
|
|
||||||
OrchestratorLogger.logWarning(`[Output] Failed to write manifest: ${error.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return manifest;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate total size of a directory recursively.
|
|
||||||
*/
|
|
||||||
private static getDirectorySize(dirPath: string): number {
|
|
||||||
let totalSize = 0;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path.join(dirPath, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
totalSize += OutputService.getDirectorySize(fullPath);
|
|
||||||
} else {
|
|
||||||
totalSize += fs.statSync(fullPath).size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Ignore errors in size calculation
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalSize;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Registry of known output types with default paths and processing hints.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export interface OutputTypeDefinition {
|
|
||||||
/** Type identifier */
|
|
||||||
name: string;
|
|
||||||
|
|
||||||
/** Default output path (relative to project root) */
|
|
||||||
defaultPath: string;
|
|
||||||
|
|
||||||
/** Human-readable description */
|
|
||||||
description: string;
|
|
||||||
|
|
||||||
/** Whether this type is built-in or user-registered */
|
|
||||||
builtIn: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class OutputTypeRegistry {
|
|
||||||
private static readonly builtInTypes: Record<string, OutputTypeDefinition> = {
|
|
||||||
build: {
|
|
||||||
name: 'build',
|
|
||||||
defaultPath: './Builds/{platform}/',
|
|
||||||
description: 'Standard game build artifact',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
'test-results': {
|
|
||||||
name: 'test-results',
|
|
||||||
defaultPath: './TestResults/',
|
|
||||||
description: 'NUnit/JUnit XML test results',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
'server-build': {
|
|
||||||
name: 'server-build',
|
|
||||||
defaultPath: './Builds/{platform}-server/',
|
|
||||||
description: 'Dedicated server build artifact',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
'data-export': {
|
|
||||||
name: 'data-export',
|
|
||||||
defaultPath: './Exports/',
|
|
||||||
description: 'Exported data files (CSV, JSON, binary)',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
images: {
|
|
||||||
name: 'images',
|
|
||||||
defaultPath: './Captures/',
|
|
||||||
description: 'Screenshots, render captures, atlas previews',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
logs: {
|
|
||||||
name: 'logs',
|
|
||||||
defaultPath: './Logs/',
|
|
||||||
description: 'Structured build and test logs',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
metrics: {
|
|
||||||
name: 'metrics',
|
|
||||||
defaultPath: './Metrics/',
|
|
||||||
description: 'Build performance metrics and asset statistics',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
coverage: {
|
|
||||||
name: 'coverage',
|
|
||||||
defaultPath: './Coverage/',
|
|
||||||
description: 'Code coverage reports',
|
|
||||||
builtIn: true,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
private static customTypes: Record<string, OutputTypeDefinition> = {};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a type definition by name. Checks custom types first, then built-in.
|
|
||||||
*/
|
|
||||||
static getType(name: string): OutputTypeDefinition | undefined {
|
|
||||||
return OutputTypeRegistry.customTypes[name] || OutputTypeRegistry.builtInTypes[name];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all registered types (built-in + custom).
|
|
||||||
*/
|
|
||||||
static getAllTypes(): OutputTypeDefinition[] {
|
|
||||||
return [...Object.values(OutputTypeRegistry.builtInTypes), ...Object.values(OutputTypeRegistry.customTypes)];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register a custom output type.
|
|
||||||
*/
|
|
||||||
static registerType(definition: OutputTypeDefinition): void {
|
|
||||||
if (OutputTypeRegistry.builtInTypes[definition.name]) {
|
|
||||||
OrchestratorLogger.logWarning(`[OutputTypes] Cannot override built-in type '${definition.name}'`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
OutputTypeRegistry.customTypes[definition.name] = { ...definition, builtIn: false };
|
|
||||||
OrchestratorLogger.log(`[OutputTypes] Registered custom type '${definition.name}'`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse a comma-separated output types string into type definitions.
|
|
||||||
* Unknown types are logged as warnings and skipped.
|
|
||||||
*/
|
|
||||||
static parseOutputTypes(outputTypesInput: string): OutputTypeDefinition[] {
|
|
||||||
if (!outputTypesInput) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const names = outputTypesInput
|
|
||||||
.split(',')
|
|
||||||
.map((s) => s.trim())
|
|
||||||
.filter(Boolean);
|
|
||||||
const types: OutputTypeDefinition[] = [];
|
|
||||||
|
|
||||||
for (const name of names) {
|
|
||||||
const typeDef = OutputTypeRegistry.getType(name);
|
|
||||||
if (typeDef) {
|
|
||||||
types.push(typeDef);
|
|
||||||
} else {
|
|
||||||
OrchestratorLogger.logWarning(`[OutputTypes] Unknown output type '${name}', skipping`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return types;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reset custom types (for testing).
|
|
||||||
*/
|
|
||||||
static resetCustomTypes(): void {
|
|
||||||
OutputTypeRegistry.customTypes = {};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
315
src/model/orchestrator/services/sync/incremental-sync-service.ts
Normal file
315
src/model/orchestrator/services/sync/incremental-sync-service.ts
Normal file
@@ -0,0 +1,315 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { OrchestratorSystem } from '../core/orchestrator-system';
|
||||||
|
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||||
|
import { SyncState, SyncStrategy } from './sync-state';
|
||||||
|
import { SyncStateManager } from './sync-state-manager';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Service for incremental workspace synchronization.
|
||||||
|
*
|
||||||
|
* Supports multiple sync strategies:
|
||||||
|
* - full: Traditional clone + cache restore (default)
|
||||||
|
* - git-delta: Fetch and apply only changed files since last sync
|
||||||
|
* - direct-input: Apply file changes passed as job input (no git push required)
|
||||||
|
* - storage-pull: Fetch changed files from rclone-backed generic storage
|
||||||
|
*/
|
||||||
|
export class IncrementalSyncService {
|
||||||
|
/**
|
||||||
|
* Load sync state from the workspace.
|
||||||
|
*/
|
||||||
|
static loadSyncState(workspacePath: string, statePath?: string): SyncState | undefined {
|
||||||
|
return SyncStateManager.loadState(workspacePath, statePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save sync state to the workspace.
|
||||||
|
*/
|
||||||
|
static saveSyncState(workspacePath: string, state: SyncState, statePath?: string): void {
|
||||||
|
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine the appropriate sync strategy based on workspace state and configuration.
|
||||||
|
*/
|
||||||
|
static resolveStrategy(requestedStrategy: SyncStrategy, workspacePath: string, statePath?: string): SyncStrategy {
|
||||||
|
if (requestedStrategy === 'full') {
|
||||||
|
return 'full';
|
||||||
|
}
|
||||||
|
|
||||||
|
// git-delta requires an existing sync state
|
||||||
|
if (requestedStrategy === 'git-delta') {
|
||||||
|
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||||
|
if (!state) {
|
||||||
|
OrchestratorLogger.log('[Sync] No sync state found, falling back to full sync');
|
||||||
|
|
||||||
|
return 'full';
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'git-delta';
|
||||||
|
}
|
||||||
|
|
||||||
|
return requestedStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a git-delta sync: fetch latest and apply only changed files.
|
||||||
|
*
|
||||||
|
* @param workspacePath - Path to the git workspace
|
||||||
|
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
|
||||||
|
* @param statePath - Optional custom path for sync state file
|
||||||
|
* @returns Number of files changed
|
||||||
|
*/
|
||||||
|
static async syncGitDelta(workspacePath: string, targetReference: string, statePath?: string): Promise<number> {
|
||||||
|
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||||
|
if (!state) {
|
||||||
|
throw new Error('Cannot git-delta sync without existing sync state');
|
||||||
|
}
|
||||||
|
|
||||||
|
OrchestratorLogger.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
|
||||||
|
|
||||||
|
// Fetch latest
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
|
||||||
|
|
||||||
|
// Get list of changed files
|
||||||
|
const diffOutput = await OrchestratorSystem.Run(
|
||||||
|
`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
|
const changedFiles = diffOutput.split('\n').filter(Boolean);
|
||||||
|
OrchestratorLogger.log(`[Sync] ${changedFiles.length} file(s) changed`);
|
||||||
|
|
||||||
|
if (changedFiles.length > 0) {
|
||||||
|
// Checkout target ref
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update sync state
|
||||||
|
const newState: SyncState = {
|
||||||
|
lastSyncCommit: targetReference,
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
workspaceHash: SyncStateManager.calculateWorkspaceHash(workspacePath),
|
||||||
|
pendingOverlays: state.pendingOverlays,
|
||||||
|
};
|
||||||
|
SyncStateManager.saveState(workspacePath, newState, statePath);
|
||||||
|
|
||||||
|
return changedFiles.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply a direct input overlay from a local archive or storage URI.
|
||||||
|
*
|
||||||
|
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
|
||||||
|
* For local paths, the archive is extracted directly.
|
||||||
|
*
|
||||||
|
* @param workspacePath - Path to the workspace
|
||||||
|
* @param inputReference - Local path or storage:// URI to the input archive
|
||||||
|
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
|
||||||
|
* @param statePath - Optional custom path for sync state file
|
||||||
|
* @returns List of overlay paths applied
|
||||||
|
*/
|
||||||
|
static async applyDirectInput(
|
||||||
|
workspacePath: string,
|
||||||
|
inputReference: string,
|
||||||
|
rcloneRemote?: string,
|
||||||
|
statePath?: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
let localArchive = inputReference;
|
||||||
|
|
||||||
|
// If storage URI, fetch via rclone first
|
||||||
|
if (inputReference.startsWith('storage://')) {
|
||||||
|
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
|
||||||
|
const remote = rcloneRemote || parsed.remote;
|
||||||
|
const remotePath = parsed.path;
|
||||||
|
|
||||||
|
localArchive = path.join(workspacePath, '.game-ci-input-overlay.tar');
|
||||||
|
OrchestratorLogger.log(`[Sync] Fetching input from storage: ${inputReference}`);
|
||||||
|
|
||||||
|
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, path.dirname(localArchive));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fs.existsSync(localArchive)) {
|
||||||
|
throw new Error(`Input archive not found: ${localArchive}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
OrchestratorLogger.log(`[Sync] Applying direct input overlay from ${localArchive}`);
|
||||||
|
|
||||||
|
// Extract overlay
|
||||||
|
await OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
|
||||||
|
|
||||||
|
// Track overlay in sync state
|
||||||
|
const state = SyncStateManager.loadState(workspacePath, statePath) || {
|
||||||
|
lastSyncCommit: '',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
state.pendingOverlays.push(localArchive);
|
||||||
|
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||||
|
|
||||||
|
return [localArchive];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a storage-pull sync: pull changed files from an rclone remote.
|
||||||
|
*
|
||||||
|
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
|
||||||
|
* and overlays it onto the workspace. Supports two modes:
|
||||||
|
* - overlay: extract on top of existing workspace (default)
|
||||||
|
* - clean: fresh git checkout, then apply overlay
|
||||||
|
*
|
||||||
|
* @param workspacePath - Path to the workspace
|
||||||
|
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
|
||||||
|
* @param options - Configuration for the storage-pull operation
|
||||||
|
* @returns List of files pulled from storage
|
||||||
|
*/
|
||||||
|
static async syncStoragePull(
|
||||||
|
workspacePath: string,
|
||||||
|
storageUri: string,
|
||||||
|
options: {
|
||||||
|
rcloneRemote?: string;
|
||||||
|
cleanMode?: boolean;
|
||||||
|
syncRevertAfter?: boolean;
|
||||||
|
statePath?: string;
|
||||||
|
} = {},
|
||||||
|
): Promise<string[]> {
|
||||||
|
if (!storageUri.startsWith('storage://')) {
|
||||||
|
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify rclone is available
|
||||||
|
try {
|
||||||
|
await OrchestratorSystem.Run('rclone version', true, true);
|
||||||
|
} catch {
|
||||||
|
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
|
||||||
|
const remote = options.rcloneRemote || parsed.remote;
|
||||||
|
const remotePath = parsed.path;
|
||||||
|
|
||||||
|
OrchestratorLogger.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
|
||||||
|
|
||||||
|
// Clean mode: reset workspace to clean git state before applying overlay
|
||||||
|
if (options.cleanMode) {
|
||||||
|
OrchestratorLogger.log('[Sync] Clean mode: resetting workspace to HEAD');
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pull from remote storage directly into workspace
|
||||||
|
const rcloneSource = `${remote}:${remotePath}`;
|
||||||
|
await OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
|
||||||
|
|
||||||
|
// List what was pulled for tracking
|
||||||
|
let pulledFiles: string[] = [];
|
||||||
|
try {
|
||||||
|
const lsOutput = await OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
|
||||||
|
pulledFiles = lsOutput
|
||||||
|
.split('\n')
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((line) => {
|
||||||
|
// rclone ls outputs: " <size> <path>"
|
||||||
|
const trimmed = line.trim();
|
||||||
|
const spaceIndex = trimmed.indexOf(' ');
|
||||||
|
|
||||||
|
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
} catch {
|
||||||
|
OrchestratorLogger.logWarning('[Sync] Could not list pulled files from remote');
|
||||||
|
}
|
||||||
|
|
||||||
|
OrchestratorLogger.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
|
||||||
|
|
||||||
|
// Update sync state with overlay tracking
|
||||||
|
const state = SyncStateManager.loadState(workspacePath, options.statePath) || {
|
||||||
|
lastSyncCommit: '',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
state.pendingOverlays.push(storageUri);
|
||||||
|
state.lastSyncTimestamp = new Date().toISOString();
|
||||||
|
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
SyncStateManager.saveState(workspacePath, state, options.statePath);
|
||||||
|
|
||||||
|
return pulledFiles;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a storage:// URI into remote and path components.
|
||||||
|
*
|
||||||
|
* Supported formats:
|
||||||
|
* - storage://remote:bucket/path (explicit remote with colon separator)
|
||||||
|
* - storage://remote/path (remote name is first path segment)
|
||||||
|
*
|
||||||
|
* @param uri - The storage:// URI to parse
|
||||||
|
* @returns Object with remote name and path
|
||||||
|
*/
|
||||||
|
static parseStorageUri(uri: string): { remote: string; path: string } {
|
||||||
|
if (!uri.startsWith('storage://')) {
|
||||||
|
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stripped = uri.replace('storage://', '');
|
||||||
|
|
||||||
|
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
|
||||||
|
const colonIndex = stripped.indexOf(':');
|
||||||
|
if (colonIndex > 0) {
|
||||||
|
return {
|
||||||
|
remote: stripped.slice(0, colonIndex),
|
||||||
|
path: stripped.slice(colonIndex + 1),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
|
||||||
|
const slashIndex = stripped.indexOf('/');
|
||||||
|
if (slashIndex > 0) {
|
||||||
|
return {
|
||||||
|
remote: stripped.slice(0, slashIndex),
|
||||||
|
path: stripped.slice(slashIndex + 1),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Just a remote name with no path
|
||||||
|
return {
|
||||||
|
remote: stripped,
|
||||||
|
path: '',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute rclone copy with standard flags.
|
||||||
|
*/
|
||||||
|
private static async executeRcloneCopy(remote: string, remotePath: string, destinationPath: string): Promise<void> {
|
||||||
|
await OrchestratorSystem.Run(
|
||||||
|
`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Revert pending overlays by restoring git state.
|
||||||
|
*/
|
||||||
|
static async revertOverlays(workspacePath: string, statePath?: string): Promise<void> {
|
||||||
|
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||||
|
if (!state || state.pendingOverlays.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
OrchestratorLogger.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
|
||||||
|
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||||
|
|
||||||
|
// Clean untracked files from overlays
|
||||||
|
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||||
|
|
||||||
|
state.pendingOverlays = [];
|
||||||
|
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||||
|
|
||||||
|
OrchestratorLogger.log('[Sync] Overlays reverted');
|
||||||
|
}
|
||||||
|
}
|
||||||
544
src/model/orchestrator/services/sync/incremental-sync.test.ts
Normal file
544
src/model/orchestrator/services/sync/incremental-sync.test.ts
Normal file
@@ -0,0 +1,544 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { IncrementalSyncService } from './incremental-sync-service';
|
||||||
|
import { SyncStateManager } from './sync-state-manager';
|
||||||
|
import { SyncState } from './sync-state';
|
||||||
|
import { OrchestratorSystem } from '../core/orchestrator-system';
|
||||||
|
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||||
|
|
||||||
|
// Mock dependencies
|
||||||
|
jest.mock('node:fs');
|
||||||
|
jest.mock('../core/orchestrator-system');
|
||||||
|
jest.mock('../core/orchestrator-logger');
|
||||||
|
|
||||||
|
const mockFs = fs as jest.Mocked<typeof fs>;
|
||||||
|
const mockSystem = OrchestratorSystem as jest.Mocked<typeof OrchestratorSystem>;
|
||||||
|
const mockLogger = OrchestratorLogger as jest.Mocked<typeof OrchestratorLogger>;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
jest.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('IncrementalSyncService', () => {
|
||||||
|
const workspacePath = '/workspace/project';
|
||||||
|
|
||||||
|
describe('parseStorageUri', () => {
|
||||||
|
it('parses storage://remote:bucket/path format', () => {
|
||||||
|
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket/some/path');
|
||||||
|
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/some/path' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses storage://remote/path format', () => {
|
||||||
|
const result = IncrementalSyncService.parseStorageUri('storage://myremote/mybucket/path');
|
||||||
|
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/path' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses storage://remote:bucket with no sub-path', () => {
|
||||||
|
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket');
|
||||||
|
expect(result).toEqual({ remote: 'myremote', path: 'mybucket' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles remote-only URI without path', () => {
|
||||||
|
const result = IncrementalSyncService.parseStorageUri('storage://myremote');
|
||||||
|
expect(result).toEqual({ remote: 'myremote', path: '' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws on invalid URI without storage:// prefix', () => {
|
||||||
|
expect(() => IncrementalSyncService.parseStorageUri('http://example.com')).toThrow('Invalid storage URI');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws on empty URI', () => {
|
||||||
|
expect(() => IncrementalSyncService.parseStorageUri('')).toThrow('Invalid storage URI');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('resolveStrategy', () => {
|
||||||
|
it('returns full when full is requested', () => {
|
||||||
|
const result = IncrementalSyncService.resolveStrategy('full', workspacePath);
|
||||||
|
expect(result).toBe('full');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns git-delta when sync state exists', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
|
||||||
|
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
|
||||||
|
expect(result).toBe('git-delta');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to full when git-delta requested but no sync state', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
|
||||||
|
expect(result).toBe('full');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns direct-input as-is', () => {
|
||||||
|
const result = IncrementalSyncService.resolveStrategy('direct-input', workspacePath);
|
||||||
|
expect(result).toBe('direct-input');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns storage-pull as-is', () => {
|
||||||
|
const result = IncrementalSyncService.resolveStrategy('storage-pull', workspacePath);
|
||||||
|
expect(result).toBe('storage-pull');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('syncGitDelta', () => {
|
||||||
|
const targetReference = 'def456789';
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123456',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fetches and checks out changed files', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('file1.txt\nfile2.cs\n'); // git diff
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||||
|
|
||||||
|
expect(result).toBe(2);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('git -C "/workspace/project" fetch origin'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('diff --name-only abc123456..def456789'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('checkout def456789'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips checkout when no files changed', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git diff (empty)
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
|
||||||
|
// Should only have fetch + diff calls, no checkout
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when no sync state exists', async () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
await expect(IncrementalSyncService.syncGitDelta(workspacePath, targetReference)).rejects.toThrow(
|
||||||
|
'Cannot git-delta sync without existing sync state',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('saves updated sync state after delta sync', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('file1.txt\n'); // git diff
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
|
||||||
|
|
||||||
|
await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||||
|
|
||||||
|
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||||
|
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||||
|
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||||
|
expect(savedState.lastSyncCommit).toBe(targetReference);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('applyDirectInput', () => {
|
||||||
|
it('extracts a local archive to workspace', async () => {
|
||||||
|
const archivePath = '/tmp/overlay.tar';
|
||||||
|
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||||
|
if (p === archivePath) return true;
|
||||||
|
|
||||||
|
// State file path does not exist
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // tar extract
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
|
||||||
|
|
||||||
|
expect(result).toEqual([archivePath]);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('tar -xf "/tmp/overlay.tar"'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fetches archive from storage URI via rclone then extracts', async () => {
|
||||||
|
const storageUri = 'storage://s3remote:builds/overlay.tar';
|
||||||
|
|
||||||
|
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||||
|
const pathString = p.toString();
|
||||||
|
if (pathString.includes('.game-ci-input-overlay.tar')) return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
mockSystem.Run.mockResolvedValue(''); // rclone copy + tar extract
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.applyDirectInput(workspacePath, storageUri);
|
||||||
|
|
||||||
|
expect(result.length).toBe(1);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('rclone copy "s3remote:builds/overlay.tar"'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when local archive does not exist', async () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
await expect(IncrementalSyncService.applyDirectInput(workspacePath, '/missing/archive.tar')).rejects.toThrow(
|
||||||
|
'Input archive not found',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tracks overlay in sync state', async () => {
|
||||||
|
const archivePath = '/tmp/overlay.tar';
|
||||||
|
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||||
|
if (p === archivePath) return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('');
|
||||||
|
|
||||||
|
await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
|
||||||
|
|
||||||
|
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||||
|
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||||
|
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||||
|
expect(savedState.pendingOverlays).toContain(archivePath);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('syncStoragePull', () => {
|
||||||
|
const storageUri = 'storage://s3:game-builds/latest';
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false); // no existing state
|
||||||
|
});
|
||||||
|
|
||||||
|
it('pulls files from rclone remote into workspace', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version check
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(' 1234 file1.txt\n 5678 dir/file2.cs\n'); // rclone ls
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||||
|
|
||||||
|
expect(result).toEqual(['file1.txt', 'dir/file2.cs']);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('rclone copy "s3:game-builds/latest" "/workspace/project"'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses custom rclone remote when provided', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
|
||||||
|
|
||||||
|
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
|
||||||
|
rcloneRemote: 'custom-remote',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('rclone copy "custom-remote:game-builds/latest"'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resets workspace in clean mode before pull', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git checkout -- .
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // git clean -fd
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
|
||||||
|
|
||||||
|
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, { cleanMode: true });
|
||||||
|
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('git -C "/workspace/project" clean -fd'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws on invalid storage URI', async () => {
|
||||||
|
await expect(IncrementalSyncService.syncStoragePull(workspacePath, 'http://example.com')).rejects.toThrow(
|
||||||
|
'Invalid storage URI',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when rclone binary is not available', async () => {
|
||||||
|
mockSystem.Run.mockRejectedValueOnce(new Error('command not found: rclone'));
|
||||||
|
|
||||||
|
await expect(IncrementalSyncService.syncStoragePull(workspacePath, storageUri)).rejects.toThrow(
|
||||||
|
'rclone binary not found',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('saves sync state with overlay tracking', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(' 100 a.txt\n'); // rclone ls
|
||||||
|
|
||||||
|
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||||
|
|
||||||
|
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||||
|
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||||
|
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||||
|
expect(savedState.pendingOverlays).toContain(storageUri);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles rclone ls failure gracefully', async () => {
|
||||||
|
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||||
|
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||||
|
mockSystem.Run.mockRejectedValueOnce(new Error('ls failed')); // rclone ls fails
|
||||||
|
|
||||||
|
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||||
|
|
||||||
|
expect(result).toEqual([]);
|
||||||
|
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Could not list pulled files'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('revertOverlays', () => {
|
||||||
|
it('reverts git state and cleans untracked files', async () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: ['/tmp/overlay.tar', 'storage://s3:builds/content'],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
mockSystem.Run.mockResolvedValue('');
|
||||||
|
|
||||||
|
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||||
|
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('git -C "/workspace/project" clean -fd'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears pending overlays in saved state', async () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: ['/tmp/overlay.tar'],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
mockSystem.Run.mockResolvedValue('');
|
||||||
|
|
||||||
|
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||||
|
|
||||||
|
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||||
|
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||||
|
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||||
|
expect(savedState.pendingOverlays).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does nothing when no overlays are pending', async () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: new Date().toISOString(),
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
|
||||||
|
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||||
|
|
||||||
|
expect(mockSystem.Run).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does nothing when no sync state exists', async () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||||
|
|
||||||
|
expect(mockSystem.Run).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('SyncStateManager', () => {
|
||||||
|
const workspacePath = '/workspace/project';
|
||||||
|
|
||||||
|
describe('loadState', () => {
|
||||||
|
it('returns parsed state from default path', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
|
||||||
|
const result = SyncStateManager.loadState(workspacePath);
|
||||||
|
|
||||||
|
expect(result).toEqual(state);
|
||||||
|
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, '.game-ci/sync-state.json'), 'utf8');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses custom state path when provided', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||||
|
|
||||||
|
SyncStateManager.loadState(workspacePath, 'custom/state.json');
|
||||||
|
|
||||||
|
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, 'custom/state.json'), 'utf8');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns undefined when state file does not exist', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
const result = SyncStateManager.loadState(workspacePath);
|
||||||
|
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns undefined and logs warning on malformed JSON', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockReturnValue('not-valid-json{{{');
|
||||||
|
|
||||||
|
const result = SyncStateManager.loadState(workspacePath);
|
||||||
|
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to load sync state'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('saveState', () => {
|
||||||
|
it('writes state to default path with pretty JSON', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||||
|
pendingOverlays: ['overlay1'],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
|
||||||
|
SyncStateManager.saveState(workspacePath, state);
|
||||||
|
|
||||||
|
expect(mockFs.writeFileSync).toHaveBeenCalledWith(
|
||||||
|
path.join(workspacePath, '.game-ci/sync-state.json'),
|
||||||
|
JSON.stringify(state, undefined, 2),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('creates parent directories if they do not exist', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
SyncStateManager.saveState(workspacePath, state);
|
||||||
|
|
||||||
|
expect(mockFs.mkdirSync).toHaveBeenCalledWith(expect.stringContaining('.game-ci'), { recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('logs warning on write failure instead of throwing', () => {
|
||||||
|
const state: SyncState = {
|
||||||
|
lastSyncCommit: 'abc123',
|
||||||
|
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||||
|
pendingOverlays: [],
|
||||||
|
};
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.writeFileSync.mockImplementation(() => {
|
||||||
|
throw new Error('Permission denied');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Should not throw
|
||||||
|
SyncStateManager.saveState(workspacePath, state);
|
||||||
|
|
||||||
|
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to save sync state'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('calculateWorkspaceHash', () => {
|
||||||
|
it('produces a deterministic SHA-256 hash from workspace files', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockImplementation((filePath: fs.PathOrFileDescriptor) => {
|
||||||
|
const p = filePath.toString();
|
||||||
|
if (p.includes('ProjectVersion.txt')) return 'm_EditorVersion: 2022.3.10f1';
|
||||||
|
if (p.includes('manifest.json')) return '{"dependencies":{}}';
|
||||||
|
if (p.includes('packages-lock.json')) return '{"dependencies":{}}';
|
||||||
|
if (p.includes('csc.rsp')) return '-nullable+';
|
||||||
|
|
||||||
|
return '';
|
||||||
|
});
|
||||||
|
|
||||||
|
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
|
||||||
|
expect(hash1).toBe(hash2);
|
||||||
|
expect(hash1).toHaveLength(64); // SHA-256 hex digest
|
||||||
|
});
|
||||||
|
|
||||||
|
it('produces different hashes for different workspace content', () => {
|
||||||
|
let callCount = 0;
|
||||||
|
mockFs.existsSync.mockReturnValue(true);
|
||||||
|
mockFs.readFileSync.mockImplementation(() => {
|
||||||
|
callCount++;
|
||||||
|
|
||||||
|
return callCount <= 4 ? 'content-v1' : 'content-v2';
|
||||||
|
});
|
||||||
|
|
||||||
|
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
|
||||||
|
expect(hash1).not.toBe(hash2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes missing file markers in hash for absent files', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
const hash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
|
||||||
|
expect(hash).toHaveLength(64);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('hasDrifted', () => {
|
||||||
|
it('returns false when workspace hash matches', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
const savedHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
|
||||||
|
const result = SyncStateManager.hasDrifted(workspacePath, savedHash);
|
||||||
|
|
||||||
|
expect(result).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true when workspace hash differs', () => {
|
||||||
|
mockFs.existsSync.mockReturnValue(false);
|
||||||
|
|
||||||
|
const result = SyncStateManager.hasDrifted(workspacePath, 'some-old-hash-that-will-not-match');
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
3
src/model/orchestrator/services/sync/index.ts
Normal file
3
src/model/orchestrator/services/sync/index.ts
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
export { SyncState, SyncStrategy } from './sync-state';
|
||||||
|
export { IncrementalSyncService } from './incremental-sync-service';
|
||||||
|
export { SyncStateManager } from './sync-state-manager';
|
||||||
125
src/model/orchestrator/services/sync/sync-state-manager.ts
Normal file
125
src/model/orchestrator/services/sync/sync-state-manager.ts
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import crypto from 'node:crypto';
|
||||||
|
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||||
|
import { SyncState } from './sync-state';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manages persistent sync state for incremental workspace updates.
|
||||||
|
*
|
||||||
|
* The sync state tracks what has been synced to a workspace, enabling
|
||||||
|
* delta-based updates instead of full clones. State is stored as a JSON
|
||||||
|
* file in the workspace (default: .game-ci/sync-state.json).
|
||||||
|
*/
|
||||||
|
export class SyncStateManager {
|
||||||
|
static readonly DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Key workspace files whose content is hashed for drift detection.
|
||||||
|
* Changes to any of these files indicate the workspace may have been
|
||||||
|
* modified outside of the sync system.
|
||||||
|
*/
|
||||||
|
private static readonly WORKSPACE_HASH_FILES = [
|
||||||
|
'ProjectSettings/ProjectVersion.txt',
|
||||||
|
'Packages/manifest.json',
|
||||||
|
'Packages/packages-lock.json',
|
||||||
|
'Assets/csc.rsp',
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load sync state from the workspace.
|
||||||
|
*
|
||||||
|
* @param workspacePath - Root path of the workspace
|
||||||
|
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||||
|
* @returns The loaded sync state, or undefined if no state exists or parsing fails
|
||||||
|
*/
|
||||||
|
static loadState(workspacePath: string, statePath?: string): SyncState | undefined {
|
||||||
|
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||||
|
|
||||||
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = fs.readFileSync(resolvedPath, 'utf8');
|
||||||
|
|
||||||
|
return JSON.parse(content) as SyncState;
|
||||||
|
} catch {
|
||||||
|
OrchestratorLogger.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save sync state to the workspace.
|
||||||
|
*
|
||||||
|
* Creates parent directories if they do not exist.
|
||||||
|
*
|
||||||
|
* @param workspacePath - Root path of the workspace
|
||||||
|
* @param state - The sync state to persist
|
||||||
|
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||||
|
*/
|
||||||
|
static saveState(workspacePath: string, state: SyncState, statePath?: string): void {
|
||||||
|
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const directory = path.dirname(resolvedPath);
|
||||||
|
if (!fs.existsSync(directory)) {
|
||||||
|
fs.mkdirSync(directory, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
|
||||||
|
OrchestratorLogger.log(
|
||||||
|
`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`,
|
||||||
|
);
|
||||||
|
} catch (error: any) {
|
||||||
|
OrchestratorLogger.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate a SHA-256 hash of key workspace files for drift detection.
|
||||||
|
*
|
||||||
|
* Hashes the content of known workspace files (ProjectVersion.txt,
|
||||||
|
* manifest.json, etc.) to produce a fingerprint. If the hash changes
|
||||||
|
* between syncs, the workspace may have been modified externally.
|
||||||
|
*
|
||||||
|
* Files that do not exist are skipped (their absence is part of the hash).
|
||||||
|
*
|
||||||
|
* @param workspacePath - Root path of the workspace
|
||||||
|
* @returns Hex-encoded SHA-256 hash string
|
||||||
|
*/
|
||||||
|
static calculateWorkspaceHash(workspacePath: string): string {
|
||||||
|
const hash = crypto.createHash('sha256');
|
||||||
|
|
||||||
|
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
|
||||||
|
const filePath = path.join(workspacePath, relativePath);
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(filePath)) {
|
||||||
|
const content = fs.readFileSync(filePath, 'utf8');
|
||||||
|
hash.update(`${relativePath}:${content}`);
|
||||||
|
} else {
|
||||||
|
hash.update(`${relativePath}:__missing__`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
hash.update(`${relativePath}:__error__`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return hash.digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the workspace has drifted from a previously saved hash.
|
||||||
|
*
|
||||||
|
* @param workspacePath - Root path of the workspace
|
||||||
|
* @param savedHash - The previously saved workspace hash to compare against
|
||||||
|
* @returns true if the current workspace hash differs from the saved hash
|
||||||
|
*/
|
||||||
|
static hasDrifted(workspacePath: string, savedHash: string): boolean {
|
||||||
|
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||||
|
|
||||||
|
return currentHash !== savedHash;
|
||||||
|
}
|
||||||
|
}
|
||||||
19
src/model/orchestrator/services/sync/sync-state.ts
Normal file
19
src/model/orchestrator/services/sync/sync-state.ts
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
/**
|
||||||
|
* Persistent sync state for incremental workspace updates.
|
||||||
|
* Stored on the runner to track what has already been synced.
|
||||||
|
*/
|
||||||
|
export interface SyncState {
|
||||||
|
/** Last successfully synced git commit SHA */
|
||||||
|
lastSyncCommit: string;
|
||||||
|
|
||||||
|
/** ISO 8601 timestamp of last sync */
|
||||||
|
lastSyncTimestamp: string;
|
||||||
|
|
||||||
|
/** SHA-256 hash of workspace state (optional) */
|
||||||
|
workspaceHash?: string;
|
||||||
|
|
||||||
|
/** List of overlay paths that haven't been reverted */
|
||||||
|
pendingOverlays: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SyncStrategy = 'full' | 'git-delta' | 'direct-input' | 'storage-pull';
|
||||||
Reference in New Issue
Block a user