mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-02 14:56:16 -07:00
Compare commits
4 Commits
feature/cl
...
feature/in
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d503b0b61a | ||
|
|
07eec62d28 | ||
|
|
4870fb5a5c | ||
|
|
3033ee0067 |
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"root": true,
|
||||
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
|
||||
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
|
||||
122
action.yml
122
action.yml
@@ -279,120 +279,24 @@ inputs:
|
||||
description:
|
||||
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
|
||||
fixes.'
|
||||
gcpProject:
|
||||
syncStrategy:
|
||||
description: 'Workspace sync strategy: full, git-delta, direct-input, storage-pull'
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Google Cloud project ID for Cloud Run Jobs provider. Falls back to
|
||||
GOOGLE_CLOUD_PROJECT env var.'
|
||||
gcpRegion:
|
||||
default: 'full'
|
||||
syncInputRef:
|
||||
description: 'URI for direct-input or storage-pull content (storage://remote/path or file path)'
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Google Cloud region for Cloud Run Jobs (e.g. us-central1). Defaults to the region
|
||||
input if empty.'
|
||||
gcpStorageType:
|
||||
syncStorageRemote:
|
||||
description: 'rclone remote name for storage-backed inputs (defaults to rcloneRemote)'
|
||||
required: false
|
||||
default: 'gcs-fuse'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Storage type for Cloud Run Jobs. Options: gcs-fuse (mount GCS bucket as filesystem,
|
||||
unlimited size, best for large sequential I/O), gcs-copy (copy artifacts in/out via gsutil, simpler, no FUSE
|
||||
overhead), nfs (Filestore NFS mount, true POSIX, good random I/O, up to 100 TiB), in-memory (tmpfs, fastest but
|
||||
volatile, up to 32 GiB).'
|
||||
gcpBucket:
|
||||
syncRevertAfter:
|
||||
description: 'Revert overlaid changes after job completion'
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] GCS bucket name for build artifact storage. Used by gcs-fuse and gcs-copy storage
|
||||
types.'
|
||||
gcpFilestoreIp:
|
||||
default: 'true'
|
||||
syncStatePath:
|
||||
description: 'Path to sync state file for delta tracking'
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Filestore instance IP address for NFS storage type. Required when gcpStorageType is
|
||||
nfs.'
|
||||
gcpFilestoreShare:
|
||||
required: false
|
||||
default: '/share1'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Filestore share name for NFS storage type. Defaults to /share1 (the Filestore
|
||||
default).'
|
||||
gcpMachineType:
|
||||
required: false
|
||||
default: 'e2-standard-4'
|
||||
description: '[Orchestrator] [Experimental] Machine type for Cloud Run Jobs (e.g. e2-standard-4, e2-highmem-8).'
|
||||
gcpDiskSizeGb:
|
||||
required: false
|
||||
default: '100'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Disk size in GB for Cloud Run Jobs in-memory volumes. Only applies to in-memory
|
||||
storage type (max 32).'
|
||||
gcpServiceAccount:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[Orchestrator] [Experimental] Google Cloud service account email for Cloud Run Jobs execution.'
|
||||
gcpVpcConnector:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[Orchestrator] [Experimental] VPC connector name for Cloud Run Jobs private networking.'
|
||||
azureResourceGroup:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Azure resource group for Container Instances provider. Falls back to
|
||||
AZURE_RESOURCE_GROUP env var.'
|
||||
azureLocation:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Azure region for Container Instances (e.g. eastus, westeurope). Defaults to the
|
||||
region input if empty.'
|
||||
azureStorageType:
|
||||
required: false
|
||||
default: 'azure-files'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Storage type for Azure Container Instances. Options: azure-files (SMB file share
|
||||
mount, up to 100 TiB, premium throughput), blob-copy (copy artifacts in/out via az storage blob, no mount
|
||||
overhead), azure-files-nfs (NFS 4.1 file share mount, true POSIX, no SMB lock overhead), in-memory (emptyDir
|
||||
tmpfs, fastest but volatile, size limited by container memory).'
|
||||
azureStorageAccount:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Azure Storage Account name. Used by azure-files, azure-files-nfs, and blob-copy
|
||||
storage types.'
|
||||
azureFileShareName:
|
||||
required: false
|
||||
default: 'unity-builds'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] Azure File Share name within the storage account. Used by azure-files and
|
||||
azure-files-nfs storage types. Supports up to 100 TiB per share.'
|
||||
azureBlobContainer:
|
||||
required: false
|
||||
default: 'unity-builds'
|
||||
description: '[Orchestrator] [Experimental] Azure Blob container name for blob-copy storage type.'
|
||||
azureSubscriptionId:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[Orchestrator] [Experimental] Azure subscription ID. Falls back to AZURE_SUBSCRIPTION_ID env var.'
|
||||
azureCpu:
|
||||
required: false
|
||||
default: '4'
|
||||
description: '[Orchestrator] [Experimental] CPU cores for Azure Container Instances (1-16).'
|
||||
azureMemoryGb:
|
||||
required: false
|
||||
default: '16'
|
||||
description: '[Orchestrator] [Experimental] Memory in GB for Azure Container Instances (1-16).'
|
||||
azureDiskSizeGb:
|
||||
required: false
|
||||
default: '100'
|
||||
description:
|
||||
'[Orchestrator] [Experimental] File share quota in GB for Azure Container Instances. Premium shares support up to
|
||||
102400 GB (100 TiB).'
|
||||
azureSubnetId:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[Orchestrator] [Experimental] Azure subnet resource ID for VNet-integrated Container Instances.'
|
||||
default: '.game-ci/sync-state.json'
|
||||
|
||||
outputs:
|
||||
volume:
|
||||
|
||||
1416
dist/index.js
generated
vendored
1416
dist/index.js
generated
vendored
File diff suppressed because it is too large
Load Diff
2
dist/index.js.map
generated
vendored
2
dist/index.js.map
generated
vendored
File diff suppressed because one or more lines are too long
74
src/index.ts
74
src/index.ts
@@ -3,6 +3,8 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
|
||||
import { Cli } from './model/cli/cli';
|
||||
import MacBuilder from './model/mac-builder';
|
||||
import PlatformSetup from './model/platform-setup';
|
||||
import { IncrementalSyncService } from './model/orchestrator/services/sync';
|
||||
import { SyncStrategy } from './model/orchestrator/services/sync/sync-state';
|
||||
|
||||
async function runMain() {
|
||||
try {
|
||||
@@ -23,6 +25,14 @@ async function runMain() {
|
||||
|
||||
if (buildParameters.providerStrategy === 'local') {
|
||||
core.info('Building locally');
|
||||
|
||||
// Apply incremental sync strategy before build
|
||||
const syncStrategy = buildParameters.syncStrategy as SyncStrategy;
|
||||
if (syncStrategy !== 'full') {
|
||||
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
|
||||
await applySyncStrategy(buildParameters, workspace);
|
||||
}
|
||||
|
||||
await PlatformSetup.setup(buildParameters, actionFolder);
|
||||
exitCode =
|
||||
process.platform === 'darwin'
|
||||
@@ -32,6 +42,16 @@ async function runMain() {
|
||||
actionFolder,
|
||||
...buildParameters,
|
||||
});
|
||||
|
||||
// Revert overlays after job completion if configured
|
||||
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||
core.info('[Sync] Reverting overlay changes after job completion');
|
||||
try {
|
||||
await IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
|
||||
} catch (revertError) {
|
||||
core.warning(`[Sync] Overlay revert failed: ${(revertError as Error).message}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await Orchestrator.run(buildParameters, baseImage.toString());
|
||||
exitCode = 0;
|
||||
@@ -50,4 +70,58 @@ async function runMain() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the configured sync strategy to the workspace before build.
|
||||
*/
|
||||
async function applySyncStrategy(buildParameters: BuildParameters, workspace: string): Promise<void> {
|
||||
const strategy = buildParameters.syncStrategy as SyncStrategy;
|
||||
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
|
||||
|
||||
if (resolvedStrategy === 'full') {
|
||||
core.info('[Sync] Resolved to full sync (no incremental state available)');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (resolvedStrategy) {
|
||||
case 'git-delta': {
|
||||
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||
const changedFiles = await IncrementalSyncService.syncGitDelta(
|
||||
workspace,
|
||||
targetReference,
|
||||
buildParameters.syncStatePath,
|
||||
);
|
||||
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
|
||||
break;
|
||||
}
|
||||
case 'direct-input': {
|
||||
if (!buildParameters.syncInputRef) {
|
||||
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
|
||||
}
|
||||
const overlays = await IncrementalSyncService.applyDirectInput(
|
||||
workspace,
|
||||
buildParameters.syncInputRef,
|
||||
buildParameters.syncStorageRemote || undefined,
|
||||
buildParameters.syncStatePath,
|
||||
);
|
||||
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
|
||||
break;
|
||||
}
|
||||
case 'storage-pull': {
|
||||
if (!buildParameters.syncInputRef) {
|
||||
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
|
||||
}
|
||||
const pulledFiles = await IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
|
||||
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||
statePath: buildParameters.syncStatePath,
|
||||
});
|
||||
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
|
||||
}
|
||||
}
|
||||
|
||||
runMain();
|
||||
|
||||
@@ -106,31 +106,11 @@ class BuildParameters {
|
||||
public cacheUnityInstallationOnMac!: boolean;
|
||||
public unityHubVersionOnMac!: string;
|
||||
public dockerWorkspacePath!: string;
|
||||
|
||||
// GCP Cloud Run (Experimental)
|
||||
public gcpProject!: string;
|
||||
public gcpRegion!: string;
|
||||
public gcpStorageType!: string;
|
||||
public gcpBucket!: string;
|
||||
public gcpFilestoreIp!: string;
|
||||
public gcpFilestoreShare!: string;
|
||||
public gcpMachineType!: string;
|
||||
public gcpDiskSizeGb!: string;
|
||||
public gcpServiceAccount!: string;
|
||||
public gcpVpcConnector!: string;
|
||||
|
||||
// Azure Container Instances (Experimental)
|
||||
public azureResourceGroup!: string;
|
||||
public azureLocation!: string;
|
||||
public azureStorageType!: string;
|
||||
public azureStorageAccount!: string;
|
||||
public azureBlobContainer!: string;
|
||||
public azureFileShareName!: string;
|
||||
public azureSubscriptionId!: string;
|
||||
public azureCpu!: string;
|
||||
public azureMemoryGb!: string;
|
||||
public azureDiskSizeGb!: string;
|
||||
public azureSubnetId!: string;
|
||||
public syncStrategy!: string;
|
||||
public syncInputRef!: string;
|
||||
public syncStorageRemote!: string;
|
||||
public syncRevertAfter!: boolean;
|
||||
public syncStatePath!: string;
|
||||
|
||||
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
||||
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
|
||||
@@ -253,27 +233,6 @@ class BuildParameters {
|
||||
inputPullCommand: OrchestratorOptions.inputPullCommand,
|
||||
pullInputList: OrchestratorOptions.pullInputList,
|
||||
kubeStorageClass: OrchestratorOptions.kubeStorageClass,
|
||||
gcpProject: Input.gcpProject,
|
||||
gcpRegion: Input.gcpRegion,
|
||||
gcpStorageType: Input.gcpStorageType,
|
||||
gcpBucket: Input.gcpBucket,
|
||||
gcpFilestoreIp: Input.gcpFilestoreIp,
|
||||
gcpFilestoreShare: Input.gcpFilestoreShare,
|
||||
gcpMachineType: Input.gcpMachineType,
|
||||
gcpDiskSizeGb: Input.gcpDiskSizeGb,
|
||||
gcpServiceAccount: Input.gcpServiceAccount,
|
||||
gcpVpcConnector: Input.gcpVpcConnector,
|
||||
azureResourceGroup: Input.azureResourceGroup,
|
||||
azureLocation: Input.azureLocation,
|
||||
azureStorageType: Input.azureStorageType,
|
||||
azureStorageAccount: Input.azureStorageAccount,
|
||||
azureBlobContainer: Input.azureBlobContainer,
|
||||
azureFileShareName: Input.azureFileShareName,
|
||||
azureSubscriptionId: Input.azureSubscriptionId,
|
||||
azureCpu: Input.azureCpu,
|
||||
azureMemoryGb: Input.azureMemoryGb,
|
||||
azureDiskSizeGb: Input.azureDiskSizeGb,
|
||||
azureSubnetId: Input.azureSubnetId,
|
||||
cacheKey: OrchestratorOptions.cacheKey,
|
||||
maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces),
|
||||
useLargePackages: OrchestratorOptions.useLargePackages,
|
||||
@@ -288,6 +247,11 @@ class BuildParameters {
|
||||
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
||||
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
||||
dockerWorkspacePath: Input.dockerWorkspacePath,
|
||||
syncStrategy: Input.syncStrategy,
|
||||
syncInputRef: Input.syncInputRef,
|
||||
syncStorageRemote: Input.syncStorageRemote,
|
||||
syncRevertAfter: Input.syncRevertAfter,
|
||||
syncStatePath: Input.syncStatePath,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -241,6 +241,28 @@ class Input {
|
||||
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
|
||||
}
|
||||
|
||||
static get syncStrategy(): string {
|
||||
return Input.getInput('syncStrategy') ?? 'full';
|
||||
}
|
||||
|
||||
static get syncInputRef(): string {
|
||||
return Input.getInput('syncInputRef') ?? '';
|
||||
}
|
||||
|
||||
static get syncStorageRemote(): string {
|
||||
return Input.getInput('syncStorageRemote') ?? '';
|
||||
}
|
||||
|
||||
static get syncRevertAfter(): boolean {
|
||||
const input = Input.getInput('syncRevertAfter') ?? 'true';
|
||||
|
||||
return input === 'true';
|
||||
}
|
||||
|
||||
static get syncStatePath(): string {
|
||||
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
|
||||
}
|
||||
|
||||
static get dockerCpuLimit(): string {
|
||||
return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
|
||||
}
|
||||
@@ -282,92 +304,6 @@ class Input {
|
||||
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
|
||||
}
|
||||
|
||||
// GCP Cloud Run (Experimental)
|
||||
static get gcpProject(): string {
|
||||
return Input.getInput('gcpProject') ?? '';
|
||||
}
|
||||
|
||||
static get gcpRegion(): string {
|
||||
return Input.getInput('gcpRegion') ?? '';
|
||||
}
|
||||
|
||||
static get gcpStorageType(): string {
|
||||
return Input.getInput('gcpStorageType') ?? 'gcs-fuse';
|
||||
}
|
||||
|
||||
static get gcpBucket(): string {
|
||||
return Input.getInput('gcpBucket') ?? '';
|
||||
}
|
||||
|
||||
static get gcpFilestoreIp(): string {
|
||||
return Input.getInput('gcpFilestoreIp') ?? '';
|
||||
}
|
||||
|
||||
static get gcpFilestoreShare(): string {
|
||||
return Input.getInput('gcpFilestoreShare') ?? '/share1';
|
||||
}
|
||||
|
||||
static get gcpMachineType(): string {
|
||||
return Input.getInput('gcpMachineType') ?? 'e2-standard-4';
|
||||
}
|
||||
|
||||
static get gcpDiskSizeGb(): string {
|
||||
return Input.getInput('gcpDiskSizeGb') ?? '100';
|
||||
}
|
||||
|
||||
static get gcpServiceAccount(): string {
|
||||
return Input.getInput('gcpServiceAccount') ?? '';
|
||||
}
|
||||
|
||||
static get gcpVpcConnector(): string {
|
||||
return Input.getInput('gcpVpcConnector') ?? '';
|
||||
}
|
||||
|
||||
// Azure Container Instances (Experimental)
|
||||
static get azureResourceGroup(): string {
|
||||
return Input.getInput('azureResourceGroup') ?? '';
|
||||
}
|
||||
|
||||
static get azureLocation(): string {
|
||||
return Input.getInput('azureLocation') ?? '';
|
||||
}
|
||||
|
||||
static get azureStorageType(): string {
|
||||
return Input.getInput('azureStorageType') ?? 'azure-files';
|
||||
}
|
||||
|
||||
static get azureStorageAccount(): string {
|
||||
return Input.getInput('azureStorageAccount') ?? '';
|
||||
}
|
||||
|
||||
static get azureBlobContainer(): string {
|
||||
return Input.getInput('azureBlobContainer') ?? 'unity-builds';
|
||||
}
|
||||
|
||||
static get azureFileShareName(): string {
|
||||
return Input.getInput('azureFileShareName') ?? 'unity-builds';
|
||||
}
|
||||
|
||||
static get azureSubscriptionId(): string {
|
||||
return Input.getInput('azureSubscriptionId') ?? '';
|
||||
}
|
||||
|
||||
static get azureCpu(): string {
|
||||
return Input.getInput('azureCpu') ?? '4';
|
||||
}
|
||||
|
||||
static get azureMemoryGb(): string {
|
||||
return Input.getInput('azureMemoryGb') ?? '16';
|
||||
}
|
||||
|
||||
static get azureDiskSizeGb(): string {
|
||||
return Input.getInput('azureDiskSizeGb') ?? '100';
|
||||
}
|
||||
|
||||
static get azureSubnetId(): string {
|
||||
return Input.getInput('azureSubnetId') ?? '';
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string) {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
|
||||
@@ -13,8 +13,6 @@ import OrchestratorEnvironmentVariable from './options/orchestrator-environment-
|
||||
import TestOrchestrator from './providers/test';
|
||||
import LocalOrchestrator from './providers/local';
|
||||
import LocalDockerOrchestrator from './providers/docker';
|
||||
import GcpCloudRunProvider from './providers/gcp-cloud-run';
|
||||
import AzureAciProvider from './providers/azure-aci';
|
||||
import loadProvider from './providers/provider-loader';
|
||||
import GitHub from '../github';
|
||||
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
|
||||
@@ -160,14 +158,6 @@ class Orchestrator {
|
||||
case 'local':
|
||||
Orchestrator.Provider = new LocalOrchestrator();
|
||||
break;
|
||||
case 'gcp-cloud-run':
|
||||
OrchestratorLogger.log('⚠ EXPERIMENTAL: GCP Cloud Run Jobs provider');
|
||||
Orchestrator.Provider = new GcpCloudRunProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
case 'azure-aci':
|
||||
OrchestratorLogger.log('⚠ EXPERIMENTAL: Azure Container Instances provider');
|
||||
Orchestrator.Provider = new AzureAciProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
default:
|
||||
// Try to load provider using the dynamic loader for unknown providers
|
||||
try {
|
||||
|
||||
@@ -1,536 +0,0 @@
|
||||
/**
|
||||
* Azure Container Instances (ACI) Provider (Experimental)
|
||||
*
|
||||
* Executes Unity builds as Azure Container Instances with configurable storage backends.
|
||||
*
|
||||
* Storage types:
|
||||
* - azure-files: SMB file share mount via Azure Files. Up to 100 TiB per share,
|
||||
* premium throughput. Default.
|
||||
* Requires: azureStorageAccount, azureFileShareName
|
||||
* - blob-copy: Copy artifacts in/out of Azure Blob Storage before/after the build.
|
||||
* No mount overhead, simpler.
|
||||
* Requires: azureStorageAccount, azureBlobContainer
|
||||
* - azure-files-nfs: NFS 4.1 file share mount. True POSIX semantics, no SMB lock overhead,
|
||||
* better for Unity Library caching (many small random reads).
|
||||
* Requires: azureStorageAccount, azureFileShareName, Premium FileStorage,
|
||||
* VNet integration (azureSubnetId)
|
||||
* - in-memory: emptyDir volume (tmpfs). Fastest I/O but volatile, size limited by
|
||||
* container memory allocation.
|
||||
*
|
||||
* Prerequisites:
|
||||
* - Azure CLI authenticated (az login or service principal)
|
||||
* - A resource group for build resources
|
||||
* - Contributor role on the resource group
|
||||
*
|
||||
* @experimental This provider is experimental. APIs and behavior may change.
|
||||
*/
|
||||
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import { Input } from '../../..';
|
||||
import ResourceTracking from '../../services/core/resource-tracking';
|
||||
|
||||
type AzureStorageType = 'azure-files' | 'blob-copy' | 'azure-files-nfs' | 'in-memory';
|
||||
|
||||
class AzureAciProvider implements ProviderInterface {
|
||||
private readonly resourceGroup: string;
|
||||
private readonly location: string;
|
||||
private readonly storageType: AzureStorageType;
|
||||
private readonly storageAccount: string;
|
||||
private readonly blobContainer: string;
|
||||
private readonly fileShareName: string;
|
||||
private readonly subscriptionId: string;
|
||||
private readonly cpu: number;
|
||||
private readonly memoryGb: number;
|
||||
private readonly diskSizeGb: number;
|
||||
private readonly subnetId: string;
|
||||
private buildParameters: BuildParameters;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.buildParameters = buildParameters;
|
||||
this.resourceGroup = buildParameters.azureResourceGroup || process.env.AZURE_RESOURCE_GROUP || '';
|
||||
this.location = buildParameters.azureLocation || Input.region || 'eastus';
|
||||
this.storageType = (buildParameters.azureStorageType || 'azure-files') as AzureStorageType;
|
||||
this.storageAccount = buildParameters.azureStorageAccount || process.env.AZURE_STORAGE_ACCOUNT || '';
|
||||
this.blobContainer = buildParameters.azureBlobContainer || 'unity-builds';
|
||||
this.fileShareName = buildParameters.azureFileShareName || 'unity-builds';
|
||||
this.subscriptionId = buildParameters.azureSubscriptionId || process.env.AZURE_SUBSCRIPTION_ID || '';
|
||||
this.cpu = Number.parseInt(buildParameters.azureCpu || '4', 10);
|
||||
this.memoryGb = Number.parseInt(buildParameters.azureMemoryGb || '16', 10);
|
||||
this.diskSizeGb = Number.parseInt(buildParameters.azureDiskSizeGb || '100', 10);
|
||||
this.subnetId = buildParameters.azureSubnetId || '';
|
||||
|
||||
OrchestratorLogger.log('[Azure ACI] Provider initialized (EXPERIMENTAL)');
|
||||
OrchestratorLogger.log(`[Azure ACI] Resource Group: ${this.resourceGroup || '(not set)'}`);
|
||||
OrchestratorLogger.log(`[Azure ACI] Location: ${this.location}`);
|
||||
OrchestratorLogger.log(`[Azure ACI] Storage: ${this.storageType}`);
|
||||
OrchestratorLogger.log(`[Azure ACI] Resources: ${this.cpu} CPU, ${this.memoryGb}GB RAM`);
|
||||
|
||||
this.validateStorageConfig();
|
||||
}
|
||||
|
||||
private validateStorageConfig(): void {
|
||||
switch (this.storageType) {
|
||||
case 'azure-files':
|
||||
if (!this.storageAccount) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[Azure ACI] Storage type "azure-files" requires azureStorageAccount to be set.',
|
||||
);
|
||||
} else {
|
||||
OrchestratorLogger.log(`[Azure ACI] File Share: ${this.storageAccount}/${this.fileShareName} (SMB)`);
|
||||
}
|
||||
break;
|
||||
case 'azure-files-nfs':
|
||||
if (!this.storageAccount) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[Azure ACI] Storage type "azure-files-nfs" requires azureStorageAccount (Premium FileStorage).',
|
||||
);
|
||||
}
|
||||
if (!this.subnetId) {
|
||||
OrchestratorLogger.logWarning('[Azure ACI] NFS file shares require VNet integration. Set azureSubnetId.');
|
||||
} else {
|
||||
OrchestratorLogger.log(`[Azure ACI] File Share: ${this.storageAccount}/${this.fileShareName} (NFS 4.1)`);
|
||||
}
|
||||
break;
|
||||
case 'blob-copy':
|
||||
if (!this.storageAccount) {
|
||||
OrchestratorLogger.logWarning('[Azure ACI] Storage type "blob-copy" requires azureStorageAccount to be set.');
|
||||
} else {
|
||||
OrchestratorLogger.log(`[Azure ACI] Blob container: ${this.storageAccount}/${this.blobContainer}`);
|
||||
}
|
||||
break;
|
||||
case 'in-memory':
|
||||
OrchestratorLogger.log(
|
||||
`[Azure ACI] In-memory volume (emptyDir): limited by ${this.memoryGb}GB container memory`,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
OrchestratorLogger.logWarning(
|
||||
`[Azure ACI] Unknown storage type '${this.storageType}'. Valid: azure-files, blob-copy, azure-files-nfs, in-memory`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!this.resourceGroup) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[Azure ACI] No resource group specified. Set azureResourceGroup input or AZURE_RESOURCE_GROUP env var.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
OrchestratorLogger.log(`[Azure ACI] Setting up workflow for build ${buildGuid}`);
|
||||
ResourceTracking.logAllocationSummary('azure-aci setup');
|
||||
|
||||
// Verify Azure CLI is available
|
||||
try {
|
||||
await OrchestratorSystem.Run('az version --output json', false, true);
|
||||
OrchestratorLogger.log('[Azure ACI] Azure CLI detected');
|
||||
} catch {
|
||||
throw new Error(
|
||||
'[Azure ACI] Azure CLI not found. Install Azure CLI: https://learn.microsoft.com/en-us/cli/azure/install-azure-cli',
|
||||
);
|
||||
}
|
||||
|
||||
if (this.subscriptionId) {
|
||||
await OrchestratorSystem.Run(`az account set --subscription="${this.subscriptionId}"`);
|
||||
}
|
||||
|
||||
// Ensure resource group exists
|
||||
if (this.resourceGroup) {
|
||||
try {
|
||||
await OrchestratorSystem.Run(`az group show --name "${this.resourceGroup}" --output json`, false, true);
|
||||
OrchestratorLogger.log(`[Azure ACI] Resource group ${this.resourceGroup} exists`);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[Azure ACI] Creating resource group ${this.resourceGroup}`);
|
||||
await OrchestratorSystem.Run(`az group create --name "${this.resourceGroup}" --location "${this.location}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Storage-specific setup
|
||||
switch (this.storageType) {
|
||||
case 'azure-files':
|
||||
await this.setupStorageAccount('Standard_LRS', 'StorageV2');
|
||||
await this.setupFileShare();
|
||||
break;
|
||||
case 'azure-files-nfs':
|
||||
await this.setupStorageAccount('Premium_LRS', 'FileStorage');
|
||||
await this.setupNfsFileShare();
|
||||
break;
|
||||
case 'blob-copy':
|
||||
await this.setupStorageAccount('Standard_LRS', 'StorageV2');
|
||||
await this.setupBlobContainer();
|
||||
break;
|
||||
case 'in-memory':
|
||||
// No storage setup needed
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async setupStorageAccount(sku: string, kind: string): Promise<void> {
|
||||
if (!this.storageAccount || !this.resourceGroup) return;
|
||||
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage account show --name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
OrchestratorLogger.log(`[Azure ACI] Storage account ${this.storageAccount} exists`);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[Azure ACI] Creating storage account ${this.storageAccount} (${sku}, ${kind})`);
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage account create --name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --location "${this.location}" --sku ${sku} --kind ${kind}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async setupFileShare(): Promise<void> {
|
||||
if (!this.storageAccount || !this.resourceGroup) return;
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage share-rm show --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[Azure ACI] Creating file share ${this.fileShareName} (${this.diskSizeGb}GB)`);
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage share-rm create --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --quota ${this.diskSizeGb}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async setupNfsFileShare(): Promise<void> {
|
||||
if (!this.storageAccount || !this.resourceGroup) return;
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage share-rm show --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[Azure ACI] Creating NFS file share ${this.fileShareName} (${this.diskSizeGb}GB)`);
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage share-rm create --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --quota ${this.diskSizeGb} --enabled-protocols NFS`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async setupBlobContainer(): Promise<void> {
|
||||
if (!this.storageAccount || !this.resourceGroup) return;
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage container show --name "${this.blobContainer}" --account-name "${this.storageAccount}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[Azure ACI] Creating blob container ${this.blobContainer}`);
|
||||
await OrchestratorSystem.Run(
|
||||
`az storage container create --name "${this.blobContainer}" --account-name "${this.storageAccount}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async getStorageKey(): Promise<string> {
|
||||
if (!this.storageAccount || !this.resourceGroup) return '';
|
||||
try {
|
||||
const keyJson = await OrchestratorSystem.Run(
|
||||
`az storage account keys list --account-name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
const keys = JSON.parse(keyJson);
|
||||
return keys[0]?.value || '';
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[Azure ACI] Could not get storage key: ${error.message}`);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
private async buildVolumeFlags(mountdir: string): Promise<string> {
|
||||
switch (this.storageType) {
|
||||
case 'azure-files': {
|
||||
const storageKey = await this.getStorageKey();
|
||||
if (!storageKey) return '';
|
||||
return [
|
||||
`--azure-file-volume-account-name "${this.storageAccount}"`,
|
||||
`--azure-file-volume-account-key "${storageKey}"`,
|
||||
`--azure-file-volume-share-name "${this.fileShareName}"`,
|
||||
`--azure-file-volume-mount-path "${mountdir}"`,
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
case 'azure-files-nfs': {
|
||||
// ACI NFS mount uses a YAML deployment template; for CLI we use the same
|
||||
// azure-file-volume flags but the share must be NFS-enabled and
|
||||
// the container must be in a VNet
|
||||
const storageKey = await this.getStorageKey();
|
||||
if (!storageKey) return '';
|
||||
return [
|
||||
`--azure-file-volume-account-name "${this.storageAccount}"`,
|
||||
`--azure-file-volume-account-key "${storageKey}"`,
|
||||
`--azure-file-volume-share-name "${this.fileShareName}"`,
|
||||
`--azure-file-volume-mount-path "${mountdir}"`,
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
case 'in-memory':
|
||||
// ACI emptyDir volumes require YAML deployment; for simplicity we skip
|
||||
// the volume mount and let the container use its own filesystem
|
||||
OrchestratorLogger.log('[Azure ACI] In-memory mode: using container filesystem (no persistent mount)');
|
||||
return '';
|
||||
|
||||
case 'blob-copy':
|
||||
// No volume mount — artifacts are copied in/out via az storage blob commands
|
||||
return '';
|
||||
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(`[Azure ACI] Running task for build ${buildGuid}`);
|
||||
ResourceTracking.logAllocationSummary('azure-aci task');
|
||||
|
||||
const containerName = `unity-build-${buildGuid}`
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.slice(0, 63);
|
||||
|
||||
// Build environment variable flags
|
||||
const allEnvVars = [
|
||||
...environment.map((env) => `${env.name}=${env.value}`),
|
||||
...secrets.map((s) => `${s.EnvironmentVariable}=${s.ParameterValue}`),
|
||||
];
|
||||
const envFlag = allEnvVars.length > 0 ? `--environment-variables ${allEnvVars.map((e) => `"${e}"`).join(' ')}` : '';
|
||||
|
||||
// Build volume flags based on storage type
|
||||
const volumeFlags = await this.buildVolumeFlags(mountdir);
|
||||
|
||||
const subnetFlag = this.subnetId ? `--subnet "${this.subnetId}"` : '';
|
||||
|
||||
// For blob-copy, wrap the user command with copy-in/copy-out steps
|
||||
let effectiveCommands = commands;
|
||||
if (this.storageType === 'blob-copy' && this.storageAccount && commands) {
|
||||
effectiveCommands = [
|
||||
`az storage blob download-batch --destination "${mountdir}" --source "${this.blobContainer}" --account-name "${this.storageAccount}" 2>/dev/null || true`,
|
||||
commands,
|
||||
`az storage blob upload-batch --source "${mountdir}" --destination "${this.blobContainer}" --account-name "${this.storageAccount}" --overwrite`,
|
||||
].join(' && ');
|
||||
}
|
||||
|
||||
const commandFlag = effectiveCommands
|
||||
? `--command-line "/bin/sh -c '${effectiveCommands.replace(/'/g, "'\\''")}'"`
|
||||
: '';
|
||||
|
||||
const createCmd = [
|
||||
'az container create',
|
||||
`--resource-group "${this.resourceGroup}"`,
|
||||
`--name "${containerName}"`,
|
||||
`--image "${image}"`,
|
||||
`--location "${this.location}"`,
|
||||
`--cpu ${this.cpu}`,
|
||||
`--memory ${this.memoryGb}`,
|
||||
'--restart-policy Never',
|
||||
'--os-type Linux',
|
||||
volumeFlags,
|
||||
envFlag,
|
||||
subnetFlag,
|
||||
commandFlag,
|
||||
'--output json',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
try {
|
||||
await OrchestratorSystem.Run(createCmd);
|
||||
OrchestratorLogger.log(
|
||||
`[Azure ACI] Container ${containerName} created (storage: ${this.storageType}), waiting for completion...`,
|
||||
);
|
||||
} catch (error: any) {
|
||||
throw new Error(`[Azure ACI] Failed to create container: ${error.message}`);
|
||||
}
|
||||
|
||||
const output = await this.waitForContainerCompletion(containerName);
|
||||
return output;
|
||||
}
|
||||
|
||||
private async waitForContainerCompletion(containerName: string): Promise<string> {
|
||||
const maxWaitMs = 24 * 60 * 60 * 1000;
|
||||
const pollIntervalMs = 15_000;
|
||||
const startTime = Date.now();
|
||||
let lastLogLength = 0;
|
||||
|
||||
while (Date.now() - startTime < maxWaitMs) {
|
||||
try {
|
||||
const stateJson = await OrchestratorSystem.Run(
|
||||
`az container show --resource-group "${this.resourceGroup}" --name "${containerName}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
const state = JSON.parse(stateJson);
|
||||
const containerState =
|
||||
state.containers?.[0]?.instanceView?.currentState?.state || state.instanceView?.state || 'Unknown';
|
||||
const provisioningState = state.provisioningState || 'Unknown';
|
||||
|
||||
// Stream logs incrementally
|
||||
try {
|
||||
const logs = await OrchestratorSystem.Run(
|
||||
`az container logs --resource-group "${this.resourceGroup}" --name "${containerName}"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
if (logs && logs.length > lastLogLength) {
|
||||
const newLogs = logs.slice(lastLogLength);
|
||||
for (const line of newLogs.split('\n')) {
|
||||
if (line.trim()) {
|
||||
OrchestratorLogger.log(`[Build] ${line}`);
|
||||
}
|
||||
}
|
||||
lastLogLength = logs.length;
|
||||
}
|
||||
} catch {
|
||||
// Logs may not be available yet
|
||||
}
|
||||
|
||||
if (containerState === 'Terminated' || provisioningState === 'Succeeded') {
|
||||
const exitCode = state.containers?.[0]?.instanceView?.currentState?.exitCode;
|
||||
if (exitCode !== undefined && exitCode !== 0) {
|
||||
throw new Error(`[Azure ACI] Container exited with code ${exitCode}`);
|
||||
}
|
||||
OrchestratorLogger.log('[Azure ACI] Container completed successfully');
|
||||
try {
|
||||
return await OrchestratorSystem.Run(
|
||||
`az container logs --resource-group "${this.resourceGroup}" --name "${containerName}"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
if (provisioningState === 'Failed') {
|
||||
const detail =
|
||||
state.containers?.[0]?.instanceView?.currentState?.detailStatus ||
|
||||
state.containers?.[0]?.instanceView?.events?.map((e: any) => e.message).join('; ') ||
|
||||
'Unknown error';
|
||||
throw new Error(`[Azure ACI] Container provisioning failed: ${detail}`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.message?.includes('Container provisioning failed') || error.message?.includes('exited with code')) {
|
||||
throw error;
|
||||
}
|
||||
OrchestratorLogger.logWarning(`[Azure ACI] Polling error: ${error.message}`);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
||||
}
|
||||
|
||||
throw new Error('[Azure ACI] Container execution timed out after 24 hours');
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
OrchestratorLogger.log('[Azure ACI] Cleaning up workflow');
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
filter: string,
|
||||
previewOnly: boolean,
|
||||
olderThan: Number,
|
||||
fullCache: boolean,
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log('[Azure ACI] Garbage collecting old container groups');
|
||||
|
||||
try {
|
||||
const containersJson = await OrchestratorSystem.Run(
|
||||
`az container list --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
const containers = JSON.parse(containersJson || '[]');
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - Number(olderThan));
|
||||
|
||||
let deletedCount = 0;
|
||||
for (const container of containers) {
|
||||
const name = container.name || '';
|
||||
if (!name.startsWith('unity-build-')) continue;
|
||||
|
||||
const createdAt = new Date(container.tags?.createdAt || container.properties?.provisioningState || 0);
|
||||
const state = container.containers?.[0]?.instanceView?.currentState?.state || '';
|
||||
|
||||
if (state === 'Terminated' || createdAt < cutoffDate) {
|
||||
if (previewOnly) {
|
||||
OrchestratorLogger.log(`[Azure ACI] Would delete: ${name}`);
|
||||
} else {
|
||||
await OrchestratorSystem.Run(
|
||||
`az container delete --resource-group "${this.resourceGroup}" --name "${name}" --yes`,
|
||||
);
|
||||
deletedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return `Garbage collected ${deletedCount} Azure container instances`;
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[Azure ACI] Garbage collection failed: ${error.message}`);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
try {
|
||||
const containersJson = await OrchestratorSystem.Run(
|
||||
`az container list --resource-group "${this.resourceGroup}" --output json`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
const containers = JSON.parse(containersJson || '[]');
|
||||
return containers
|
||||
.filter((c: any) => (c.name || '').startsWith('unity-build-'))
|
||||
.map((c: any) => ({ Name: c.name || '' }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('[Azure ACI] listWorkflow not implemented for this experimental provider');
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<string> {
|
||||
throw new Error('[Azure ACI] watchWorkflow not implemented for this experimental provider');
|
||||
}
|
||||
}
|
||||
|
||||
export default AzureAciProvider;
|
||||
@@ -1,435 +0,0 @@
|
||||
/**
|
||||
* Google Cloud Run Jobs Provider (Experimental)
|
||||
*
|
||||
* Executes Unity builds as Cloud Run Jobs with configurable storage backends.
|
||||
*
|
||||
* Storage types:
|
||||
* - gcs-fuse: Mount a GCS bucket as a POSIX filesystem via GCS FUSE sidecar.
|
||||
* Unlimited size, best for large sequential reads/writes.
|
||||
* Requires: gcpBucket
|
||||
* - gcs-copy: Copy artifacts in/out of GCS before/after the build via gsutil.
|
||||
* No mount overhead, simpler, works everywhere.
|
||||
* Requires: gcpBucket
|
||||
* - nfs: Mount a Filestore NFS share. True POSIX semantics, good random I/O,
|
||||
* up to 100 TiB. Best for Library caching (many small random reads).
|
||||
* Requires: gcpFilestoreIp, gcpFilestoreShare
|
||||
* - in-memory: tmpfs volume (emptyDir). Fastest I/O but volatile and limited to 32 GiB.
|
||||
* Good for scratch/temp space during builds.
|
||||
*
|
||||
* Prerequisites:
|
||||
* - Google Cloud SDK authenticated (GOOGLE_APPLICATION_CREDENTIALS or gcloud auth)
|
||||
* - Cloud Run Jobs API enabled
|
||||
* - Service account with roles: Cloud Run Admin, Storage Admin, Logs Viewer
|
||||
*
|
||||
* @experimental This provider is experimental. APIs and behavior may change.
|
||||
*/
|
||||
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import { Input } from '../../..';
|
||||
import ResourceTracking from '../../services/core/resource-tracking';
|
||||
|
||||
type GcpStorageType = 'gcs-fuse' | 'gcs-copy' | 'nfs' | 'in-memory';
|
||||
|
||||
class GcpCloudRunProvider implements ProviderInterface {
|
||||
private readonly project: string;
|
||||
private readonly region: string;
|
||||
private readonly storageType: GcpStorageType;
|
||||
private readonly bucket: string;
|
||||
private readonly filestoreIp: string;
|
||||
private readonly filestoreShare: string;
|
||||
private readonly machineType: string;
|
||||
private readonly diskSizeGb: number;
|
||||
private readonly serviceAccount: string;
|
||||
private readonly vpcConnector: string;
|
||||
private buildParameters: BuildParameters;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.buildParameters = buildParameters;
|
||||
this.project = buildParameters.gcpProject || process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT || '';
|
||||
this.region = buildParameters.gcpRegion || Input.region || 'us-central1';
|
||||
this.storageType = (buildParameters.gcpStorageType || 'gcs-fuse') as GcpStorageType;
|
||||
this.bucket = buildParameters.gcpBucket || '';
|
||||
this.filestoreIp = buildParameters.gcpFilestoreIp || '';
|
||||
this.filestoreShare = buildParameters.gcpFilestoreShare || '/share1';
|
||||
this.machineType = buildParameters.gcpMachineType || 'e2-standard-4';
|
||||
this.diskSizeGb = Number.parseInt(buildParameters.gcpDiskSizeGb || '100', 10);
|
||||
this.serviceAccount = buildParameters.gcpServiceAccount || '';
|
||||
this.vpcConnector = buildParameters.gcpVpcConnector || '';
|
||||
|
||||
OrchestratorLogger.log('[GCP Cloud Run] Provider initialized (EXPERIMENTAL)');
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Project: ${this.project || '(auto-detect)'}`);
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Region: ${this.region}`);
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Storage: ${this.storageType}`);
|
||||
|
||||
this.validateStorageConfig();
|
||||
}
|
||||
|
||||
private validateStorageConfig(): void {
|
||||
switch (this.storageType) {
|
||||
case 'gcs-fuse':
|
||||
case 'gcs-copy':
|
||||
if (!this.bucket) {
|
||||
OrchestratorLogger.logWarning(
|
||||
`[GCP Cloud Run] Storage type '${this.storageType}' requires gcpBucket to be set.`,
|
||||
);
|
||||
} else {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Bucket: gs://${this.bucket}`);
|
||||
}
|
||||
break;
|
||||
case 'nfs':
|
||||
if (!this.filestoreIp) {
|
||||
OrchestratorLogger.logWarning('[GCP Cloud Run] Storage type "nfs" requires gcpFilestoreIp to be set.');
|
||||
} else {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Filestore: ${this.filestoreIp}:${this.filestoreShare}`);
|
||||
}
|
||||
if (!this.vpcConnector) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[GCP Cloud Run] NFS storage usually requires gcpVpcConnector for private network access to Filestore.',
|
||||
);
|
||||
}
|
||||
break;
|
||||
case 'in-memory':
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] In-memory volume: ${Math.min(this.diskSizeGb, 32)} GiB (max 32)`);
|
||||
break;
|
||||
default:
|
||||
OrchestratorLogger.logWarning(
|
||||
`[GCP Cloud Run] Unknown storage type '${this.storageType}'. Valid: gcs-fuse, gcs-copy, nfs, in-memory`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!this.project) {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[GCP Cloud Run] No project specified. Set gcpProject input or GOOGLE_CLOUD_PROJECT env var.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Setting up workflow for build ${buildGuid}`);
|
||||
ResourceTracking.logAllocationSummary('gcp-cloud-run setup');
|
||||
|
||||
// Verify gcloud CLI is available
|
||||
try {
|
||||
await OrchestratorSystem.Run('gcloud --version', false, true);
|
||||
OrchestratorLogger.log('[GCP Cloud Run] gcloud CLI detected');
|
||||
} catch {
|
||||
throw new Error(
|
||||
'[GCP Cloud Run] gcloud CLI not found. Install Google Cloud SDK: https://cloud.google.com/sdk/docs/install',
|
||||
);
|
||||
}
|
||||
|
||||
// Verify Cloud Run Jobs API is enabled
|
||||
try {
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
await OrchestratorSystem.Run(
|
||||
`gcloud services list --enabled --filter="name:run.googleapis.com" ${projectFlag} --format="value(name)"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
OrchestratorLogger.logWarning(
|
||||
'[GCP Cloud Run] Could not verify Cloud Run API status. Ensure run.googleapis.com is enabled.',
|
||||
);
|
||||
}
|
||||
|
||||
// Storage-specific setup
|
||||
if ((this.storageType === 'gcs-fuse' || this.storageType === 'gcs-copy') && this.bucket) {
|
||||
await this.ensureBucketExists();
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureBucketExists(): Promise<void> {
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`gcloud storage buckets describe gs://${this.bucket} --format="value(name)"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Bucket gs://${this.bucket} exists`);
|
||||
} catch {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Creating bucket gs://${this.bucket}`);
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
await OrchestratorSystem.Run(
|
||||
`gcloud storage buckets create gs://${this.bucket} --location=${this.region} ${projectFlag}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private buildVolumeFlags(mountdir: string): { volumeFlags: string; mountFlags: string } {
|
||||
switch (this.storageType) {
|
||||
case 'gcs-fuse':
|
||||
if (!this.bucket) return { volumeFlags: '', mountFlags: '' };
|
||||
return {
|
||||
volumeFlags: `--add-volume=name=gcs-fuse,type=cloud-storage,bucket=${this.bucket}`,
|
||||
mountFlags: `--add-volume-mount=volume=gcs-fuse,mount-path=${mountdir}`,
|
||||
};
|
||||
|
||||
case 'nfs':
|
||||
if (!this.filestoreIp) return { volumeFlags: '', mountFlags: '' };
|
||||
return {
|
||||
volumeFlags: `--add-volume=name=nfs-vol,type=nfs,location=${this.filestoreIp}:${this.filestoreShare}`,
|
||||
mountFlags: `--add-volume-mount=volume=nfs-vol,mount-path=${mountdir}`,
|
||||
};
|
||||
|
||||
case 'in-memory': {
|
||||
const sizeGib = Math.min(this.diskSizeGb, 32);
|
||||
return {
|
||||
volumeFlags: `--add-volume=name=tmpfs-vol,type=in-memory,size-limit=${sizeGib}Gi`,
|
||||
mountFlags: `--add-volume-mount=volume=tmpfs-vol,mount-path=${mountdir}`,
|
||||
};
|
||||
}
|
||||
|
||||
case 'gcs-copy':
|
||||
// No volume mount — artifacts are copied in/out via gsutil commands
|
||||
return { volumeFlags: '', mountFlags: '' };
|
||||
|
||||
default:
|
||||
return { volumeFlags: '', mountFlags: '' };
|
||||
}
|
||||
}
|
||||
|
||||
private async copyArtifactsIn(mountdir: string): Promise<void> {
|
||||
if (this.storageType !== 'gcs-copy' || !this.bucket) return;
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Copying artifacts from gs://${this.bucket} to ${mountdir}`);
|
||||
try {
|
||||
await OrchestratorSystem.Run(`gcloud storage cp -r "gs://${this.bucket}/*" "${mountdir}/" || true`, false, true);
|
||||
} catch {
|
||||
OrchestratorLogger.log('[GCP Cloud Run] No existing artifacts to restore (bucket may be empty)');
|
||||
}
|
||||
}
|
||||
|
||||
private async copyArtifactsOut(mountdir: string): Promise<void> {
|
||||
if (this.storageType !== 'gcs-copy' || !this.bucket) return;
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Uploading artifacts from ${mountdir} to gs://${this.bucket}`);
|
||||
await OrchestratorSystem.Run(`gcloud storage cp -r "${mountdir}/*" "gs://${this.bucket}/"`, false, true);
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Running task for build ${buildGuid}`);
|
||||
ResourceTracking.logAllocationSummary('gcp-cloud-run task');
|
||||
|
||||
const jobName = `unity-build-${buildGuid}`
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.slice(0, 63);
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
|
||||
// Build environment variable flags
|
||||
const envFlags = environment
|
||||
.map((env) => `${env.name}=${env.value}`)
|
||||
.concat(secrets.map((s) => `${s.EnvironmentVariable}=${s.ParameterValue}`));
|
||||
const envString = envFlags.length > 0 ? `--set-env-vars="${envFlags.join(',')}"` : '';
|
||||
|
||||
// Build storage volume flags
|
||||
const { volumeFlags, mountFlags } = this.buildVolumeFlags(mountdir);
|
||||
|
||||
// For gcs-copy, wrap the user command with copy-in/copy-out steps
|
||||
let effectiveCommands = commands;
|
||||
if (this.storageType === 'gcs-copy' && this.bucket && commands) {
|
||||
effectiveCommands = [
|
||||
`gcloud storage cp -r "gs://${this.bucket}/*" "${mountdir}/" 2>/dev/null || true`,
|
||||
commands,
|
||||
`gcloud storage cp -r "${mountdir}/*" "gs://${this.bucket}/"`,
|
||||
].join(' && ');
|
||||
}
|
||||
|
||||
const saFlag = this.serviceAccount ? `--service-account=${this.serviceAccount}` : '';
|
||||
const vpcFlag = this.vpcConnector ? `--vpc-connector=${this.vpcConnector}` : '';
|
||||
|
||||
// Create the Cloud Run Job
|
||||
const createCmd = [
|
||||
'gcloud run jobs create',
|
||||
jobName,
|
||||
`--image=${image}`,
|
||||
`--region=${this.region}`,
|
||||
'--task-timeout=86400s',
|
||||
'--max-retries=0',
|
||||
'--cpu=4',
|
||||
'--memory=16Gi',
|
||||
volumeFlags,
|
||||
mountFlags,
|
||||
envString,
|
||||
saFlag,
|
||||
vpcFlag,
|
||||
projectFlag,
|
||||
'--format=json',
|
||||
'--quiet',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
try {
|
||||
await OrchestratorSystem.Run(createCmd);
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Job ${jobName} created`);
|
||||
} catch (error: any) {
|
||||
if (error.message?.includes('already exists')) {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Job ${jobName} already exists, updating...`);
|
||||
const updateCmd = createCmd.replace('jobs create', 'jobs update');
|
||||
await OrchestratorSystem.Run(updateCmd);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Override the command if provided
|
||||
if (effectiveCommands) {
|
||||
const updateCmd = [
|
||||
'gcloud run jobs update',
|
||||
jobName,
|
||||
`--region=${this.region}`,
|
||||
'--command="/bin/sh"',
|
||||
`--args="-c,${effectiveCommands}"`,
|
||||
projectFlag,
|
||||
'--quiet',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
await OrchestratorSystem.Run(updateCmd);
|
||||
}
|
||||
|
||||
// Execute the job
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Executing job ${jobName} (storage: ${this.storageType})...`);
|
||||
const executeCmd = [
|
||||
'gcloud run jobs execute',
|
||||
jobName,
|
||||
`--region=${this.region}`,
|
||||
projectFlag,
|
||||
'--wait',
|
||||
'--format=json',
|
||||
'--quiet',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
let output = '';
|
||||
try {
|
||||
output = await OrchestratorSystem.Run(executeCmd);
|
||||
OrchestratorLogger.log('[GCP Cloud Run] Job execution completed');
|
||||
} catch (error: any) {
|
||||
await this.streamJobLogs(jobName);
|
||||
throw new Error(`[GCP Cloud Run] Job execution failed: ${error.message}`);
|
||||
}
|
||||
|
||||
await this.streamJobLogs(jobName);
|
||||
return output;
|
||||
}
|
||||
|
||||
private async streamJobLogs(jobName: string): Promise<void> {
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
try {
|
||||
const logs = await OrchestratorSystem.Run(
|
||||
`gcloud logging read "resource.type=cloud_run_job AND resource.labels.job_name=${jobName}" ${projectFlag} --limit=1000 --format="value(textPayload)" --order=asc`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
if (logs) {
|
||||
for (const line of logs.split('\n')) {
|
||||
if (line.trim()) {
|
||||
OrchestratorLogger.log(`[Build] ${line}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
OrchestratorLogger.logWarning('[GCP Cloud Run] Could not retrieve job logs');
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
OrchestratorLogger.log('[GCP Cloud Run] Cleaning up workflow');
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
filter: string,
|
||||
previewOnly: boolean,
|
||||
olderThan: Number,
|
||||
fullCache: boolean,
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log('[GCP Cloud Run] Garbage collecting old jobs');
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
|
||||
try {
|
||||
const jobsJson = await OrchestratorSystem.Run(
|
||||
`gcloud run jobs list --region=${this.region} ${projectFlag} --filter="metadata.name~unity-build-" --format="json(metadata.name,metadata.creationTimestamp)"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
const jobs = JSON.parse(jobsJson || '[]');
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - Number(olderThan));
|
||||
|
||||
let deletedCount = 0;
|
||||
for (const job of jobs) {
|
||||
const createdAt = new Date(job.metadata?.creationTimestamp || 0);
|
||||
if (createdAt < cutoffDate) {
|
||||
const name = job.metadata?.name;
|
||||
if (previewOnly) {
|
||||
OrchestratorLogger.log(`[GCP Cloud Run] Would delete: ${name}`);
|
||||
} else {
|
||||
await OrchestratorSystem.Run(
|
||||
`gcloud run jobs delete ${name} --region=${this.region} ${projectFlag} --quiet`,
|
||||
);
|
||||
deletedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return `Garbage collected ${deletedCount} Cloud Run jobs`;
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[GCP Cloud Run] Garbage collection failed: ${error.message}`);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
const projectFlag = this.project ? `--project=${this.project}` : '';
|
||||
try {
|
||||
const jobsJson = await OrchestratorSystem.Run(
|
||||
`gcloud run jobs list --region=${this.region} ${projectFlag} --filter="metadata.name~unity-build-" --format="json(metadata.name)"`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
const jobs = JSON.parse(jobsJson || '[]');
|
||||
return jobs.map((job: any) => ({ Name: job.metadata?.name || '' }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('[GCP Cloud Run] listWorkflow not implemented for this experimental provider');
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<string> {
|
||||
throw new Error('[GCP Cloud Run] watchWorkflow not implemented for this experimental provider');
|
||||
}
|
||||
}
|
||||
|
||||
export default GcpCloudRunProvider;
|
||||
@@ -62,8 +62,6 @@ export default async function loadProvider(
|
||||
'local-docker': './docker',
|
||||
'local-system': './local',
|
||||
local: './local',
|
||||
'gcp-cloud-run': './gcp-cloud-run',
|
||||
'azure-aci': './azure-aci',
|
||||
};
|
||||
|
||||
modulePath = providerModuleMap[providerSource] || providerSource;
|
||||
@@ -138,7 +136,7 @@ export class ProviderLoader {
|
||||
* @returns string[] - Array of available provider names
|
||||
*/
|
||||
static getAvailableProviders(): string[] {
|
||||
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local', 'gcp-cloud-run', 'azure-aci'];
|
||||
return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local'];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -15,15 +15,24 @@ import BuildParameters from '../../build-parameters';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import OrchestratorOptions from '../options/orchestrator-options';
|
||||
import ResourceTracking from '../services/core/resource-tracking';
|
||||
import { IncrementalSyncService } from '../services/sync';
|
||||
import { SyncStrategy } from '../services/sync/sync-state';
|
||||
|
||||
export class RemoteClient {
|
||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
static async setupRemoteClient() {
|
||||
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
|
||||
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
|
||||
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
|
||||
|
||||
if (syncStrategy !== 'full') {
|
||||
OrchestratorLogger.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
|
||||
await RemoteClient.handleIncrementalSync(syncStrategy);
|
||||
} else if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
|
||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||
}
|
||||
@@ -157,6 +166,20 @@ export class RemoteClient {
|
||||
|
||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||
|
||||
// Revert sync overlays if configured
|
||||
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
|
||||
if (Orchestrator.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||
try {
|
||||
OrchestratorLogger.log('[Sync] Reverting overlay changes after job completion');
|
||||
await IncrementalSyncService.revertOverlays(
|
||||
OrchestratorFolders.repoPathAbsolute,
|
||||
Orchestrator.buildParameters.syncStatePath,
|
||||
);
|
||||
} catch (revertError: any) {
|
||||
RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// WIP - need to give the pod permissions to create config map
|
||||
await RemoteClientLogger.handleLogManagementPostJob();
|
||||
} catch (error: any) {
|
||||
@@ -229,6 +252,78 @@ export class RemoteClient {
|
||||
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
|
||||
*
|
||||
* For git-delta: requires an existing workspace with sync state; fetches and applies
|
||||
* only changed files.
|
||||
*
|
||||
* For direct-input and storage-pull: requires an existing workspace; applies overlay
|
||||
* content on top.
|
||||
*
|
||||
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
|
||||
*/
|
||||
private static async handleIncrementalSync(strategy: SyncStrategy): Promise<void> {
|
||||
const buildParameters = Orchestrator.buildParameters;
|
||||
const workspacePath = OrchestratorFolders.repoPathAbsolute;
|
||||
const statePath = buildParameters.syncStatePath;
|
||||
|
||||
// Resolve strategy — may fall back to 'full' if no state exists
|
||||
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
|
||||
|
||||
if (resolvedStrategy === 'full') {
|
||||
OrchestratorLogger.log('[Sync] Falling back to full bootstrap');
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (resolvedStrategy) {
|
||||
case 'git-delta': {
|
||||
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||
OrchestratorLogger.log(`[Sync] Git delta sync to ${targetReference}`);
|
||||
const changedFiles = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
|
||||
OrchestratorLogger.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
|
||||
break;
|
||||
}
|
||||
case 'direct-input': {
|
||||
const inputReference = buildParameters.syncInputRef;
|
||||
if (!inputReference) {
|
||||
throw new Error('[Sync] direct-input strategy requires syncInputRef');
|
||||
}
|
||||
OrchestratorLogger.log(`[Sync] Applying direct input: ${inputReference}`);
|
||||
await IncrementalSyncService.applyDirectInput(
|
||||
workspacePath,
|
||||
inputReference,
|
||||
buildParameters.syncStorageRemote || undefined,
|
||||
statePath,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'storage-pull': {
|
||||
const storageUri = buildParameters.syncInputRef;
|
||||
if (!storageUri) {
|
||||
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
|
||||
}
|
||||
OrchestratorLogger.log(`[Sync] Storage pull from: ${storageUri}`);
|
||||
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
|
||||
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||
statePath,
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
OrchestratorLogger.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static async bootstrapRepository() {
|
||||
await OrchestratorSystem.Run(
|
||||
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
|
||||
|
||||
315
src/model/orchestrator/services/sync/incremental-sync-service.ts
Normal file
315
src/model/orchestrator/services/sync/incremental-sync-service.ts
Normal file
@@ -0,0 +1,315 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { OrchestratorSystem } from '../core/orchestrator-system';
|
||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||
import { SyncState, SyncStrategy } from './sync-state';
|
||||
import { SyncStateManager } from './sync-state-manager';
|
||||
|
||||
/**
|
||||
* Service for incremental workspace synchronization.
|
||||
*
|
||||
* Supports multiple sync strategies:
|
||||
* - full: Traditional clone + cache restore (default)
|
||||
* - git-delta: Fetch and apply only changed files since last sync
|
||||
* - direct-input: Apply file changes passed as job input (no git push required)
|
||||
* - storage-pull: Fetch changed files from rclone-backed generic storage
|
||||
*/
|
||||
export class IncrementalSyncService {
|
||||
/**
|
||||
* Load sync state from the workspace.
|
||||
*/
|
||||
static loadSyncState(workspacePath: string, statePath?: string): SyncState | undefined {
|
||||
return SyncStateManager.loadState(workspacePath, statePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save sync state to the workspace.
|
||||
*/
|
||||
static saveSyncState(workspacePath: string, state: SyncState, statePath?: string): void {
|
||||
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the appropriate sync strategy based on workspace state and configuration.
|
||||
*/
|
||||
static resolveStrategy(requestedStrategy: SyncStrategy, workspacePath: string, statePath?: string): SyncStrategy {
|
||||
if (requestedStrategy === 'full') {
|
||||
return 'full';
|
||||
}
|
||||
|
||||
// git-delta requires an existing sync state
|
||||
if (requestedStrategy === 'git-delta') {
|
||||
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state) {
|
||||
OrchestratorLogger.log('[Sync] No sync state found, falling back to full sync');
|
||||
|
||||
return 'full';
|
||||
}
|
||||
|
||||
return 'git-delta';
|
||||
}
|
||||
|
||||
return requestedStrategy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a git-delta sync: fetch latest and apply only changed files.
|
||||
*
|
||||
* @param workspacePath - Path to the git workspace
|
||||
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
|
||||
* @param statePath - Optional custom path for sync state file
|
||||
* @returns Number of files changed
|
||||
*/
|
||||
static async syncGitDelta(workspacePath: string, targetReference: string, statePath?: string): Promise<number> {
|
||||
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state) {
|
||||
throw new Error('Cannot git-delta sync without existing sync state');
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
|
||||
|
||||
// Fetch latest
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
|
||||
|
||||
// Get list of changed files
|
||||
const diffOutput = await OrchestratorSystem.Run(
|
||||
`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`,
|
||||
true,
|
||||
);
|
||||
|
||||
const changedFiles = diffOutput.split('\n').filter(Boolean);
|
||||
OrchestratorLogger.log(`[Sync] ${changedFiles.length} file(s) changed`);
|
||||
|
||||
if (changedFiles.length > 0) {
|
||||
// Checkout target ref
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
|
||||
}
|
||||
|
||||
// Update sync state
|
||||
const newState: SyncState = {
|
||||
lastSyncCommit: targetReference,
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
workspaceHash: SyncStateManager.calculateWorkspaceHash(workspacePath),
|
||||
pendingOverlays: state.pendingOverlays,
|
||||
};
|
||||
SyncStateManager.saveState(workspacePath, newState, statePath);
|
||||
|
||||
return changedFiles.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a direct input overlay from a local archive or storage URI.
|
||||
*
|
||||
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
|
||||
* For local paths, the archive is extracted directly.
|
||||
*
|
||||
* @param workspacePath - Path to the workspace
|
||||
* @param inputReference - Local path or storage:// URI to the input archive
|
||||
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
|
||||
* @param statePath - Optional custom path for sync state file
|
||||
* @returns List of overlay paths applied
|
||||
*/
|
||||
static async applyDirectInput(
|
||||
workspacePath: string,
|
||||
inputReference: string,
|
||||
rcloneRemote?: string,
|
||||
statePath?: string,
|
||||
): Promise<string[]> {
|
||||
let localArchive = inputReference;
|
||||
|
||||
// If storage URI, fetch via rclone first
|
||||
if (inputReference.startsWith('storage://')) {
|
||||
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
|
||||
const remote = rcloneRemote || parsed.remote;
|
||||
const remotePath = parsed.path;
|
||||
|
||||
localArchive = path.join(workspacePath, '.game-ci-input-overlay.tar');
|
||||
OrchestratorLogger.log(`[Sync] Fetching input from storage: ${inputReference}`);
|
||||
|
||||
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, path.dirname(localArchive));
|
||||
}
|
||||
|
||||
if (!fs.existsSync(localArchive)) {
|
||||
throw new Error(`Input archive not found: ${localArchive}`);
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[Sync] Applying direct input overlay from ${localArchive}`);
|
||||
|
||||
// Extract overlay
|
||||
await OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
|
||||
|
||||
// Track overlay in sync state
|
||||
const state = SyncStateManager.loadState(workspacePath, statePath) || {
|
||||
lastSyncCommit: '',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
|
||||
state.pendingOverlays.push(localArchive);
|
||||
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
|
||||
return [localArchive];
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a storage-pull sync: pull changed files from an rclone remote.
|
||||
*
|
||||
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
|
||||
* and overlays it onto the workspace. Supports two modes:
|
||||
* - overlay: extract on top of existing workspace (default)
|
||||
* - clean: fresh git checkout, then apply overlay
|
||||
*
|
||||
* @param workspacePath - Path to the workspace
|
||||
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
|
||||
* @param options - Configuration for the storage-pull operation
|
||||
* @returns List of files pulled from storage
|
||||
*/
|
||||
static async syncStoragePull(
|
||||
workspacePath: string,
|
||||
storageUri: string,
|
||||
options: {
|
||||
rcloneRemote?: string;
|
||||
cleanMode?: boolean;
|
||||
syncRevertAfter?: boolean;
|
||||
statePath?: string;
|
||||
} = {},
|
||||
): Promise<string[]> {
|
||||
if (!storageUri.startsWith('storage://')) {
|
||||
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
|
||||
}
|
||||
|
||||
// Verify rclone is available
|
||||
try {
|
||||
await OrchestratorSystem.Run('rclone version', true, true);
|
||||
} catch {
|
||||
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
|
||||
}
|
||||
|
||||
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
|
||||
const remote = options.rcloneRemote || parsed.remote;
|
||||
const remotePath = parsed.path;
|
||||
|
||||
OrchestratorLogger.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
|
||||
|
||||
// Clean mode: reset workspace to clean git state before applying overlay
|
||||
if (options.cleanMode) {
|
||||
OrchestratorLogger.log('[Sync] Clean mode: resetting workspace to HEAD');
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||
}
|
||||
|
||||
// Pull from remote storage directly into workspace
|
||||
const rcloneSource = `${remote}:${remotePath}`;
|
||||
await OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
|
||||
|
||||
// List what was pulled for tracking
|
||||
let pulledFiles: string[] = [];
|
||||
try {
|
||||
const lsOutput = await OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
|
||||
pulledFiles = lsOutput
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
// rclone ls outputs: " <size> <path>"
|
||||
const trimmed = line.trim();
|
||||
const spaceIndex = trimmed.indexOf(' ');
|
||||
|
||||
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
|
||||
})
|
||||
.filter(Boolean);
|
||||
} catch {
|
||||
OrchestratorLogger.logWarning('[Sync] Could not list pulled files from remote');
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
|
||||
|
||||
// Update sync state with overlay tracking
|
||||
const state = SyncStateManager.loadState(workspacePath, options.statePath) || {
|
||||
lastSyncCommit: '',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
|
||||
state.pendingOverlays.push(storageUri);
|
||||
state.lastSyncTimestamp = new Date().toISOString();
|
||||
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
SyncStateManager.saveState(workspacePath, state, options.statePath);
|
||||
|
||||
return pulledFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a storage:// URI into remote and path components.
|
||||
*
|
||||
* Supported formats:
|
||||
* - storage://remote:bucket/path (explicit remote with colon separator)
|
||||
* - storage://remote/path (remote name is first path segment)
|
||||
*
|
||||
* @param uri - The storage:// URI to parse
|
||||
* @returns Object with remote name and path
|
||||
*/
|
||||
static parseStorageUri(uri: string): { remote: string; path: string } {
|
||||
if (!uri.startsWith('storage://')) {
|
||||
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
|
||||
}
|
||||
|
||||
const stripped = uri.replace('storage://', '');
|
||||
|
||||
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
|
||||
const colonIndex = stripped.indexOf(':');
|
||||
if (colonIndex > 0) {
|
||||
return {
|
||||
remote: stripped.slice(0, colonIndex),
|
||||
path: stripped.slice(colonIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
|
||||
const slashIndex = stripped.indexOf('/');
|
||||
if (slashIndex > 0) {
|
||||
return {
|
||||
remote: stripped.slice(0, slashIndex),
|
||||
path: stripped.slice(slashIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
// Just a remote name with no path
|
||||
return {
|
||||
remote: stripped,
|
||||
path: '',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute rclone copy with standard flags.
|
||||
*/
|
||||
private static async executeRcloneCopy(remote: string, remotePath: string, destinationPath: string): Promise<void> {
|
||||
await OrchestratorSystem.Run(
|
||||
`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Revert pending overlays by restoring git state.
|
||||
*/
|
||||
static async revertOverlays(workspacePath: string, statePath?: string): Promise<void> {
|
||||
const state = SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state || state.pendingOverlays.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
|
||||
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||
|
||||
// Clean untracked files from overlays
|
||||
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||
|
||||
state.pendingOverlays = [];
|
||||
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
|
||||
OrchestratorLogger.log('[Sync] Overlays reverted');
|
||||
}
|
||||
}
|
||||
544
src/model/orchestrator/services/sync/incremental-sync.test.ts
Normal file
544
src/model/orchestrator/services/sync/incremental-sync.test.ts
Normal file
@@ -0,0 +1,544 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { IncrementalSyncService } from './incremental-sync-service';
|
||||
import { SyncStateManager } from './sync-state-manager';
|
||||
import { SyncState } from './sync-state';
|
||||
import { OrchestratorSystem } from '../core/orchestrator-system';
|
||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock('node:fs');
|
||||
jest.mock('../core/orchestrator-system');
|
||||
jest.mock('../core/orchestrator-logger');
|
||||
|
||||
const mockFs = fs as jest.Mocked<typeof fs>;
|
||||
const mockSystem = OrchestratorSystem as jest.Mocked<typeof OrchestratorSystem>;
|
||||
const mockLogger = OrchestratorLogger as jest.Mocked<typeof OrchestratorLogger>;
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('IncrementalSyncService', () => {
|
||||
const workspacePath = '/workspace/project';
|
||||
|
||||
describe('parseStorageUri', () => {
|
||||
it('parses storage://remote:bucket/path format', () => {
|
||||
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket/some/path');
|
||||
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/some/path' });
|
||||
});
|
||||
|
||||
it('parses storage://remote/path format', () => {
|
||||
const result = IncrementalSyncService.parseStorageUri('storage://myremote/mybucket/path');
|
||||
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/path' });
|
||||
});
|
||||
|
||||
it('parses storage://remote:bucket with no sub-path', () => {
|
||||
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket');
|
||||
expect(result).toEqual({ remote: 'myremote', path: 'mybucket' });
|
||||
});
|
||||
|
||||
it('handles remote-only URI without path', () => {
|
||||
const result = IncrementalSyncService.parseStorageUri('storage://myremote');
|
||||
expect(result).toEqual({ remote: 'myremote', path: '' });
|
||||
});
|
||||
|
||||
it('throws on invalid URI without storage:// prefix', () => {
|
||||
expect(() => IncrementalSyncService.parseStorageUri('http://example.com')).toThrow('Invalid storage URI');
|
||||
});
|
||||
|
||||
it('throws on empty URI', () => {
|
||||
expect(() => IncrementalSyncService.parseStorageUri('')).toThrow('Invalid storage URI');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveStrategy', () => {
|
||||
it('returns full when full is requested', () => {
|
||||
const result = IncrementalSyncService.resolveStrategy('full', workspacePath);
|
||||
expect(result).toBe('full');
|
||||
});
|
||||
|
||||
it('returns git-delta when sync state exists', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
|
||||
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
|
||||
expect(result).toBe('git-delta');
|
||||
});
|
||||
|
||||
it('falls back to full when git-delta requested but no sync state', () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
|
||||
expect(result).toBe('full');
|
||||
});
|
||||
|
||||
it('returns direct-input as-is', () => {
|
||||
const result = IncrementalSyncService.resolveStrategy('direct-input', workspacePath);
|
||||
expect(result).toBe('direct-input');
|
||||
});
|
||||
|
||||
it('returns storage-pull as-is', () => {
|
||||
const result = IncrementalSyncService.resolveStrategy('storage-pull', workspacePath);
|
||||
expect(result).toBe('storage-pull');
|
||||
});
|
||||
});
|
||||
|
||||
describe('syncGitDelta', () => {
|
||||
const targetReference = 'def456789';
|
||||
|
||||
beforeEach(() => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123456',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
});
|
||||
|
||||
it('fetches and checks out changed files', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||
mockSystem.Run.mockResolvedValueOnce('file1.txt\nfile2.cs\n'); // git diff
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
|
||||
|
||||
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||
|
||||
expect(result).toBe(2);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('git -C "/workspace/project" fetch origin'),
|
||||
true,
|
||||
);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('diff --name-only abc123456..def456789'),
|
||||
true,
|
||||
);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('checkout def456789'), true);
|
||||
});
|
||||
|
||||
it('skips checkout when no files changed', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git diff (empty)
|
||||
|
||||
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||
|
||||
expect(result).toBe(0);
|
||||
|
||||
// Should only have fetch + diff calls, no checkout
|
||||
expect(mockSystem.Run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('throws when no sync state exists', async () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
await expect(IncrementalSyncService.syncGitDelta(workspacePath, targetReference)).rejects.toThrow(
|
||||
'Cannot git-delta sync without existing sync state',
|
||||
);
|
||||
});
|
||||
|
||||
it('saves updated sync state after delta sync', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
|
||||
mockSystem.Run.mockResolvedValueOnce('file1.txt\n'); // git diff
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
|
||||
|
||||
await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
|
||||
|
||||
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||
expect(savedState.lastSyncCommit).toBe(targetReference);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyDirectInput', () => {
|
||||
it('extracts a local archive to workspace', async () => {
|
||||
const archivePath = '/tmp/overlay.tar';
|
||||
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||
if (p === archivePath) return true;
|
||||
|
||||
// State file path does not exist
|
||||
return false;
|
||||
});
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // tar extract
|
||||
|
||||
const result = await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
|
||||
|
||||
expect(result).toEqual([archivePath]);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('tar -xf "/tmp/overlay.tar"'), true);
|
||||
});
|
||||
|
||||
it('fetches archive from storage URI via rclone then extracts', async () => {
|
||||
const storageUri = 'storage://s3remote:builds/overlay.tar';
|
||||
|
||||
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||
const pathString = p.toString();
|
||||
if (pathString.includes('.game-ci-input-overlay.tar')) return true;
|
||||
|
||||
return false;
|
||||
});
|
||||
mockSystem.Run.mockResolvedValue(''); // rclone copy + tar extract
|
||||
|
||||
const result = await IncrementalSyncService.applyDirectInput(workspacePath, storageUri);
|
||||
|
||||
expect(result.length).toBe(1);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('rclone copy "s3remote:builds/overlay.tar"'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when local archive does not exist', async () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
await expect(IncrementalSyncService.applyDirectInput(workspacePath, '/missing/archive.tar')).rejects.toThrow(
|
||||
'Input archive not found',
|
||||
);
|
||||
});
|
||||
|
||||
it('tracks overlay in sync state', async () => {
|
||||
const archivePath = '/tmp/overlay.tar';
|
||||
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
|
||||
if (p === archivePath) return true;
|
||||
|
||||
return false;
|
||||
});
|
||||
mockSystem.Run.mockResolvedValueOnce('');
|
||||
|
||||
await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
|
||||
|
||||
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||
expect(savedState.pendingOverlays).toContain(archivePath);
|
||||
});
|
||||
});
|
||||
|
||||
describe('syncStoragePull', () => {
|
||||
const storageUri = 'storage://s3:game-builds/latest';
|
||||
|
||||
beforeEach(() => {
|
||||
mockFs.existsSync.mockReturnValue(false); // no existing state
|
||||
});
|
||||
|
||||
it('pulls files from rclone remote into workspace', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version check
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||
mockSystem.Run.mockResolvedValueOnce(' 1234 file1.txt\n 5678 dir/file2.cs\n'); // rclone ls
|
||||
|
||||
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||
|
||||
expect(result).toEqual(['file1.txt', 'dir/file2.cs']);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('rclone copy "s3:game-builds/latest" "/workspace/project"'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('uses custom rclone remote when provided', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
|
||||
|
||||
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
|
||||
rcloneRemote: 'custom-remote',
|
||||
});
|
||||
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('rclone copy "custom-remote:game-builds/latest"'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('resets workspace in clean mode before pull', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git checkout -- .
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // git clean -fd
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
|
||||
|
||||
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, { cleanMode: true });
|
||||
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
|
||||
true,
|
||||
);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('git -C "/workspace/project" clean -fd'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws on invalid storage URI', async () => {
|
||||
await expect(IncrementalSyncService.syncStoragePull(workspacePath, 'http://example.com')).rejects.toThrow(
|
||||
'Invalid storage URI',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when rclone binary is not available', async () => {
|
||||
mockSystem.Run.mockRejectedValueOnce(new Error('command not found: rclone'));
|
||||
|
||||
await expect(IncrementalSyncService.syncStoragePull(workspacePath, storageUri)).rejects.toThrow(
|
||||
'rclone binary not found',
|
||||
);
|
||||
});
|
||||
|
||||
it('saves sync state with overlay tracking', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||
mockSystem.Run.mockResolvedValueOnce(' 100 a.txt\n'); // rclone ls
|
||||
|
||||
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||
|
||||
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||
expect(savedState.pendingOverlays).toContain(storageUri);
|
||||
});
|
||||
|
||||
it('handles rclone ls failure gracefully', async () => {
|
||||
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
|
||||
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
|
||||
mockSystem.Run.mockRejectedValueOnce(new Error('ls failed')); // rclone ls fails
|
||||
|
||||
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Could not list pulled files'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('revertOverlays', () => {
|
||||
it('reverts git state and cleans untracked files', async () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: ['/tmp/overlay.tar', 'storage://s3:builds/content'],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
mockSystem.Run.mockResolvedValue('');
|
||||
|
||||
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
|
||||
true,
|
||||
);
|
||||
expect(mockSystem.Run).toHaveBeenCalledWith(
|
||||
expect.stringContaining('git -C "/workspace/project" clean -fd'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('clears pending overlays in saved state', async () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: ['/tmp/overlay.tar'],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
mockSystem.Run.mockResolvedValue('');
|
||||
|
||||
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||
|
||||
expect(mockFs.writeFileSync).toHaveBeenCalled();
|
||||
const writeCall = mockFs.writeFileSync.mock.calls[0];
|
||||
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
|
||||
expect(savedState.pendingOverlays).toEqual([]);
|
||||
});
|
||||
|
||||
it('does nothing when no overlays are pending', async () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
|
||||
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||
|
||||
expect(mockSystem.Run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does nothing when no sync state exists', async () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
await IncrementalSyncService.revertOverlays(workspacePath);
|
||||
|
||||
expect(mockSystem.Run).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('SyncStateManager', () => {
|
||||
const workspacePath = '/workspace/project';
|
||||
|
||||
describe('loadState', () => {
|
||||
it('returns parsed state from default path', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
|
||||
const result = SyncStateManager.loadState(workspacePath);
|
||||
|
||||
expect(result).toEqual(state);
|
||||
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, '.game-ci/sync-state.json'), 'utf8');
|
||||
});
|
||||
|
||||
it('uses custom state path when provided', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
|
||||
|
||||
SyncStateManager.loadState(workspacePath, 'custom/state.json');
|
||||
|
||||
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, 'custom/state.json'), 'utf8');
|
||||
});
|
||||
|
||||
it('returns undefined when state file does not exist', () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
const result = SyncStateManager.loadState(workspacePath);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined and logs warning on malformed JSON', () => {
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockReturnValue('not-valid-json{{{');
|
||||
|
||||
const result = SyncStateManager.loadState(workspacePath);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to load sync state'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('saveState', () => {
|
||||
it('writes state to default path with pretty JSON', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||
pendingOverlays: ['overlay1'],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
|
||||
SyncStateManager.saveState(workspacePath, state);
|
||||
|
||||
expect(mockFs.writeFileSync).toHaveBeenCalledWith(
|
||||
path.join(workspacePath, '.game-ci/sync-state.json'),
|
||||
JSON.stringify(state, undefined, 2),
|
||||
'utf8',
|
||||
);
|
||||
});
|
||||
|
||||
it('creates parent directories if they do not exist', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
SyncStateManager.saveState(workspacePath, state);
|
||||
|
||||
expect(mockFs.mkdirSync).toHaveBeenCalledWith(expect.stringContaining('.game-ci'), { recursive: true });
|
||||
});
|
||||
|
||||
it('logs warning on write failure instead of throwing', () => {
|
||||
const state: SyncState = {
|
||||
lastSyncCommit: 'abc123',
|
||||
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
|
||||
pendingOverlays: [],
|
||||
};
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.writeFileSync.mockImplementation(() => {
|
||||
throw new Error('Permission denied');
|
||||
});
|
||||
|
||||
// Should not throw
|
||||
SyncStateManager.saveState(workspacePath, state);
|
||||
|
||||
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to save sync state'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateWorkspaceHash', () => {
|
||||
it('produces a deterministic SHA-256 hash from workspace files', () => {
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockImplementation((filePath: fs.PathOrFileDescriptor) => {
|
||||
const p = filePath.toString();
|
||||
if (p.includes('ProjectVersion.txt')) return 'm_EditorVersion: 2022.3.10f1';
|
||||
if (p.includes('manifest.json')) return '{"dependencies":{}}';
|
||||
if (p.includes('packages-lock.json')) return '{"dependencies":{}}';
|
||||
if (p.includes('csc.rsp')) return '-nullable+';
|
||||
|
||||
return '';
|
||||
});
|
||||
|
||||
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
|
||||
expect(hash1).toBe(hash2);
|
||||
expect(hash1).toHaveLength(64); // SHA-256 hex digest
|
||||
});
|
||||
|
||||
it('produces different hashes for different workspace content', () => {
|
||||
let callCount = 0;
|
||||
mockFs.existsSync.mockReturnValue(true);
|
||||
mockFs.readFileSync.mockImplementation(() => {
|
||||
callCount++;
|
||||
|
||||
return callCount <= 4 ? 'content-v1' : 'content-v2';
|
||||
});
|
||||
|
||||
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
|
||||
expect(hash1).not.toBe(hash2);
|
||||
});
|
||||
|
||||
it('includes missing file markers in hash for absent files', () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
const hash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
|
||||
expect(hash).toHaveLength(64);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasDrifted', () => {
|
||||
it('returns false when workspace hash matches', () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
const savedHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
|
||||
const result = SyncStateManager.hasDrifted(workspacePath, savedHash);
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true when workspace hash differs', () => {
|
||||
mockFs.existsSync.mockReturnValue(false);
|
||||
|
||||
const result = SyncStateManager.hasDrifted(workspacePath, 'some-old-hash-that-will-not-match');
|
||||
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
3
src/model/orchestrator/services/sync/index.ts
Normal file
3
src/model/orchestrator/services/sync/index.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export { SyncState, SyncStrategy } from './sync-state';
|
||||
export { IncrementalSyncService } from './incremental-sync-service';
|
||||
export { SyncStateManager } from './sync-state-manager';
|
||||
125
src/model/orchestrator/services/sync/sync-state-manager.ts
Normal file
125
src/model/orchestrator/services/sync/sync-state-manager.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
||||
import { SyncState } from './sync-state';
|
||||
|
||||
/**
|
||||
* Manages persistent sync state for incremental workspace updates.
|
||||
*
|
||||
* The sync state tracks what has been synced to a workspace, enabling
|
||||
* delta-based updates instead of full clones. State is stored as a JSON
|
||||
* file in the workspace (default: .game-ci/sync-state.json).
|
||||
*/
|
||||
export class SyncStateManager {
|
||||
static readonly DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
|
||||
|
||||
/**
|
||||
* Key workspace files whose content is hashed for drift detection.
|
||||
* Changes to any of these files indicate the workspace may have been
|
||||
* modified outside of the sync system.
|
||||
*/
|
||||
private static readonly WORKSPACE_HASH_FILES = [
|
||||
'ProjectSettings/ProjectVersion.txt',
|
||||
'Packages/manifest.json',
|
||||
'Packages/packages-lock.json',
|
||||
'Assets/csc.rsp',
|
||||
];
|
||||
|
||||
/**
|
||||
* Load sync state from the workspace.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||
* @returns The loaded sync state, or undefined if no state exists or parsing fails
|
||||
*/
|
||||
static loadState(workspacePath: string, statePath?: string): SyncState | undefined {
|
||||
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(resolvedPath, 'utf8');
|
||||
|
||||
return JSON.parse(content) as SyncState;
|
||||
} catch {
|
||||
OrchestratorLogger.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save sync state to the workspace.
|
||||
*
|
||||
* Creates parent directories if they do not exist.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param state - The sync state to persist
|
||||
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||
*/
|
||||
static saveState(workspacePath: string, state: SyncState, statePath?: string): void {
|
||||
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||
|
||||
try {
|
||||
const directory = path.dirname(resolvedPath);
|
||||
if (!fs.existsSync(directory)) {
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
|
||||
fs.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
|
||||
OrchestratorLogger.log(
|
||||
`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`,
|
||||
);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate a SHA-256 hash of key workspace files for drift detection.
|
||||
*
|
||||
* Hashes the content of known workspace files (ProjectVersion.txt,
|
||||
* manifest.json, etc.) to produce a fingerprint. If the hash changes
|
||||
* between syncs, the workspace may have been modified externally.
|
||||
*
|
||||
* Files that do not exist are skipped (their absence is part of the hash).
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @returns Hex-encoded SHA-256 hash string
|
||||
*/
|
||||
static calculateWorkspaceHash(workspacePath: string): string {
|
||||
const hash = crypto.createHash('sha256');
|
||||
|
||||
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
|
||||
const filePath = path.join(workspacePath, relativePath);
|
||||
try {
|
||||
if (fs.existsSync(filePath)) {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
hash.update(`${relativePath}:${content}`);
|
||||
} else {
|
||||
hash.update(`${relativePath}:__missing__`);
|
||||
}
|
||||
} catch {
|
||||
hash.update(`${relativePath}:__error__`);
|
||||
}
|
||||
}
|
||||
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the workspace has drifted from a previously saved hash.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param savedHash - The previously saved workspace hash to compare against
|
||||
* @returns true if the current workspace hash differs from the saved hash
|
||||
*/
|
||||
static hasDrifted(workspacePath: string, savedHash: string): boolean {
|
||||
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
|
||||
return currentHash !== savedHash;
|
||||
}
|
||||
}
|
||||
19
src/model/orchestrator/services/sync/sync-state.ts
Normal file
19
src/model/orchestrator/services/sync/sync-state.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Persistent sync state for incremental workspace updates.
|
||||
* Stored on the runner to track what has already been synced.
|
||||
*/
|
||||
export interface SyncState {
|
||||
/** Last successfully synced git commit SHA */
|
||||
lastSyncCommit: string;
|
||||
|
||||
/** ISO 8601 timestamp of last sync */
|
||||
lastSyncTimestamp: string;
|
||||
|
||||
/** SHA-256 hash of workspace state (optional) */
|
||||
workspaceHash?: string;
|
||||
|
||||
/** List of overlay paths that haven't been reverted */
|
||||
pendingOverlays: string[];
|
||||
}
|
||||
|
||||
export type SyncStrategy = 'full' | 'git-delta' | 'direct-input' | 'storage-pull';
|
||||
Reference in New Issue
Block a user