Compare commits

..

4 Commits

Author SHA1 Message Date
frostebite
d503b0b61a fix: replace orchestrator-develop branch references with main
The orchestrator-develop branch no longer exists. Update all fallback
clone commands and test fixtures to use main instead.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-09 19:59:58 +00:00
frostebite
07eec62d28 ci: set macOS builds to continue-on-error
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 23:33:04 +00:00
frostebite
4870fb5a5c feat(sync): complete incremental sync protocol with storage-pull, state management, and tests (#799)
- Add storage-pull strategy: rclone-based sync from remote storage with
  overlay and clean modes, URI parsing (storage://remote:bucket/path),
  transfer parallelism, and automatic rclone availability checking
- Add SyncStateManager: persistent state load/save with configurable
  paths, workspace hash calculation via SHA-256 of key project files,
  and drift detection for external modification awareness
- Add action.yml inputs: syncStrategy, syncInputRef, syncStorageRemote,
  syncRevertAfter, syncStatePath with sensible defaults
- Wire sync into Input (5 getters), BuildParameters (5 fields), index.ts
  (local build path), and RemoteClient (orchestrator path) with post-job
  overlay revert when syncRevertAfter is true
- Add 42 unit tests covering all strategies, URI parsing, state
  management, hash calculation, drift detection, error handling, and
  edge cases (missing rclone, invalid URIs, absent state, empty diffs)
- Add root:true to eslintrc to prevent plugin resolution conflicts

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 11:56:43 +00:00
frostebite
3033ee0067 feat(orchestrator): incremental sync protocol — git delta, direct input, and storage-backed sync
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 09:39:53 +00:00
17 changed files with 1757 additions and 1209 deletions

View File

@@ -1,4 +1,5 @@
{
"root": true,
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
"parser": "@typescript-eslint/parser",

View File

@@ -182,8 +182,8 @@ inputs:
required: false
default: ''
description:
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
keys image, secrets (name, value object array), command line string)'
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
the keys image, secrets (name, value object array), command line string)'
awsStackName:
default: 'game-ci'
required: false
@@ -194,15 +194,6 @@ inputs:
description:
'[Orchestrator] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
be configured.'
secretSource:
default: ''
required: false
description:
'[Orchestrator] Premade secret source for pulling build secrets. Supported values: aws-secrets-manager,
aws-parameter-store, gcp-secret-manager, azure-key-vault, hashicorp-vault, hashicorp-vault-kv1,
vault (alias for hashicorp-vault), env. Can also be a custom shell command with {0} placeholder
for the key, or a path to a YAML file defining custom sources. Takes precedence over
inputPullCommand when set.'
resourceTracking:
default: 'false'
required: false
@@ -288,6 +279,24 @@ inputs:
description:
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
fixes.'
syncStrategy:
description: 'Workspace sync strategy: full, git-delta, direct-input, storage-pull'
required: false
default: 'full'
syncInputRef:
description: 'URI for direct-input or storage-pull content (storage://remote/path or file path)'
required: false
syncStorageRemote:
description: 'rclone remote name for storage-backed inputs (defaults to rcloneRemote)'
required: false
syncRevertAfter:
description: 'Revert overlaid changes after job completion'
required: false
default: 'true'
syncStatePath:
description: 'Path to sync state file for delta tracking'
required: false
default: '.game-ci/sync-state.json'
outputs:
volume:

881
dist/index.js generated vendored

File diff suppressed because it is too large Load Diff

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@@ -3,6 +3,8 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup';
import { IncrementalSyncService } from './model/orchestrator/services/sync';
import { SyncStrategy } from './model/orchestrator/services/sync/sync-state';
async function runMain() {
try {
@@ -23,6 +25,14 @@ async function runMain() {
if (buildParameters.providerStrategy === 'local') {
core.info('Building locally');
// Apply incremental sync strategy before build
const syncStrategy = buildParameters.syncStrategy as SyncStrategy;
if (syncStrategy !== 'full') {
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
await applySyncStrategy(buildParameters, workspace);
}
await PlatformSetup.setup(buildParameters, actionFolder);
exitCode =
process.platform === 'darwin'
@@ -32,6 +42,16 @@ async function runMain() {
actionFolder,
...buildParameters,
});
// Revert overlays after job completion if configured
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
core.info('[Sync] Reverting overlay changes after job completion');
try {
await IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
} catch (revertError) {
core.warning(`[Sync] Overlay revert failed: ${(revertError as Error).message}`);
}
}
} else {
await Orchestrator.run(buildParameters, baseImage.toString());
exitCode = 0;
@@ -50,4 +70,58 @@ async function runMain() {
}
}
/**
* Apply the configured sync strategy to the workspace before build.
*/
async function applySyncStrategy(buildParameters: BuildParameters, workspace: string): Promise<void> {
const strategy = buildParameters.syncStrategy as SyncStrategy;
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
if (resolvedStrategy === 'full') {
core.info('[Sync] Resolved to full sync (no incremental state available)');
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
const changedFiles = await IncrementalSyncService.syncGitDelta(
workspace,
targetReference,
buildParameters.syncStatePath,
);
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
break;
}
case 'direct-input': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
}
const overlays = await IncrementalSyncService.applyDirectInput(
workspace,
buildParameters.syncInputRef,
buildParameters.syncStorageRemote || undefined,
buildParameters.syncStatePath,
);
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
break;
}
case 'storage-pull': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
}
const pulledFiles = await IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath: buildParameters.syncStatePath,
});
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
break;
}
default:
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
}
}
runMain();

View File

@@ -106,6 +106,11 @@ class BuildParameters {
public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string;
public dockerWorkspacePath!: string;
public syncStrategy!: string;
public syncInputRef!: string;
public syncStorageRemote!: string;
public syncRevertAfter!: boolean;
public syncStatePath!: string;
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
@@ -242,6 +247,11 @@ class BuildParameters {
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac,
dockerWorkspacePath: Input.dockerWorkspacePath,
syncStrategy: Input.syncStrategy,
syncInputRef: Input.syncInputRef,
syncStorageRemote: Input.syncStorageRemote,
syncRevertAfter: Input.syncRevertAfter,
syncStatePath: Input.syncStatePath,
};
}

View File

@@ -241,6 +241,28 @@ class Input {
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
}
static get syncStrategy(): string {
return Input.getInput('syncStrategy') ?? 'full';
}
static get syncInputRef(): string {
return Input.getInput('syncInputRef') ?? '';
}
static get syncStorageRemote(): string {
return Input.getInput('syncStorageRemote') ?? '';
}
static get syncRevertAfter(): boolean {
const input = Input.getInput('syncRevertAfter') ?? 'true';
return input === 'true';
}
static get syncStatePath(): string {
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
}
static get dockerCpuLimit(): string {
return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
}

View File

@@ -190,10 +190,6 @@ class OrchestratorOptions {
return OrchestratorOptions.getInput('pullInputList')?.split(`,`) || [];
}
static get secretSource(): string {
return OrchestratorOptions.getInput('secretSource') || '';
}
static get inputPullCommand(): string {
const value = OrchestratorOptions.getInput('inputPullCommand');

View File

@@ -1,9 +1,6 @@
import * as core from '@actions/core';
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import OrchestratorOptions from './orchestrator-options';
import { SecretSourceService, validateSecretKey } from '../services/secrets/secret-source-service';
import OrchestratorLogger from '../services/core/orchestrator-logger';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
@@ -16,6 +13,8 @@ const formatFunction = (value: string, arguments_: any[]) => {
class OrchestratorQueryOverride {
static queryOverrides: { [key: string]: string } | undefined;
// TODO accept premade secret sources or custom secret source definition yamls
public static query(key: string, alternativeKey: string) {
if (OrchestratorQueryOverride.queryOverrides && OrchestratorQueryOverride.queryOverrides[key] !== undefined) {
return OrchestratorQueryOverride.queryOverrides[key];
@@ -50,62 +49,14 @@ class OrchestratorQueryOverride {
throw new Error(`Should not be trying to run override query on ${query}`);
}
// Validate the query key before interpolating it into a shell command
validateSecretKey(query);
const result = await GenericInputReader.Run(
return await GenericInputReader.Run(
formatFunction(OrchestratorOptions.inputPullCommand, [{ key: 0, value: query }]),
);
// Mask the fetched secret value so it does not appear in GitHub Actions logs
if (result && result.trim().length > 0) {
core.setSecret(result);
}
return result;
}
/**
* Populate query overrides using either:
* 1. Premade/custom secret sources (via secretSource input), or
* 2. Shell command (via inputPullCommand, legacy approach)
*
* The secretSource input takes precedence if set. It supports:
* - Premade names: 'aws-secrets-manager', 'aws-parameter-store', 'gcp-secret-manager', 'azure-key-vault', 'env'
* - Custom commands: any string containing {0} placeholder
* - YAML file path: a path ending in .yml or .yaml containing custom source definitions
*/
public static async PopulateQueryOverrideInput() {
const queries = OrchestratorOptions.pullInputList;
OrchestratorQueryOverride.queryOverrides = {};
const secretSource = OrchestratorOptions.secretSource;
// Use SecretSourceService if secretSource is configured
if (secretSource) {
OrchestratorLogger.log(`Using secret source: ${secretSource}`);
// YAML file: load definitions and use the first source
if (secretSource.endsWith('.yml') || secretSource.endsWith('.yaml')) {
const definitions = SecretSourceService.loadFromYaml(secretSource);
if (definitions.length > 0) {
OrchestratorLogger.log(`Loaded ${definitions.length} secret source(s) from ${secretSource}`);
for (const key of queries) {
OrchestratorQueryOverride.queryOverrides[key] = await SecretSourceService.fetchSecret(definitions[0], key);
}
}
return;
}
// Premade or custom command source
const results = await SecretSourceService.fetchAll(secretSource, queries);
Object.assign(OrchestratorQueryOverride.queryOverrides, results);
return;
}
// Legacy: use inputPullCommand if set
for (const element of queries) {
if (OrchestratorQueryOverride.shouldUseOverride(element)) {
OrchestratorQueryOverride.queryOverrides[element] = await OrchestratorQueryOverride.queryOverride(element);

View File

@@ -15,15 +15,24 @@ import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import OrchestratorOptions from '../options/orchestrator-options';
import ResourceTracking from '../services/core/resource-tracking';
import { IncrementalSyncService } from '../services/sync';
import { SyncStrategy } from '../services/sync/sync-state';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() {
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
if (!(await RemoteClient.handleRetainedWorkspace())) {
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
if (syncStrategy !== 'full') {
OrchestratorLogger.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
await RemoteClient.handleIncrementalSync(syncStrategy);
} else if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.runCustomHookFiles(`before-build`);
}
@@ -157,6 +166,20 @@ export class RemoteClient {
await RemoteClient.runCustomHookFiles(`after-build`);
// Revert sync overlays if configured
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
if (Orchestrator.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
try {
OrchestratorLogger.log('[Sync] Reverting overlay changes after job completion');
await IncrementalSyncService.revertOverlays(
OrchestratorFolders.repoPathAbsolute,
Orchestrator.buildParameters.syncStatePath,
);
} catch (revertError: any) {
RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
}
}
// WIP - need to give the pod permissions to create config map
await RemoteClientLogger.handleLogManagementPostJob();
} catch (error: any) {
@@ -229,6 +252,78 @@ export class RemoteClient {
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
/**
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
*
* For git-delta: requires an existing workspace with sync state; fetches and applies
* only changed files.
*
* For direct-input and storage-pull: requires an existing workspace; applies overlay
* content on top.
*
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
*/
private static async handleIncrementalSync(strategy: SyncStrategy): Promise<void> {
const buildParameters = Orchestrator.buildParameters;
const workspacePath = OrchestratorFolders.repoPathAbsolute;
const statePath = buildParameters.syncStatePath;
// Resolve strategy — may fall back to 'full' if no state exists
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
if (resolvedStrategy === 'full') {
OrchestratorLogger.log('[Sync] Falling back to full bootstrap');
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
OrchestratorLogger.log(`[Sync] Git delta sync to ${targetReference}`);
const changedFiles = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
OrchestratorLogger.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
break;
}
case 'direct-input': {
const inputReference = buildParameters.syncInputRef;
if (!inputReference) {
throw new Error('[Sync] direct-input strategy requires syncInputRef');
}
OrchestratorLogger.log(`[Sync] Applying direct input: ${inputReference}`);
await IncrementalSyncService.applyDirectInput(
workspacePath,
inputReference,
buildParameters.syncStorageRemote || undefined,
statePath,
);
break;
}
case 'storage-pull': {
const storageUri = buildParameters.syncInputRef;
if (!storageUri) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
}
OrchestratorLogger.log(`[Sync] Storage pull from: ${storageUri}`);
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath,
});
break;
}
default:
OrchestratorLogger.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
}
}
public static async bootstrapRepository() {
await OrchestratorSystem.Run(
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,

View File

@@ -1,446 +0,0 @@
import fs from 'node:fs';
import * as core from '@actions/core';
import { SecretSourceService, validateSecretKey } from './secret-source-service';
jest.mock('node:fs');
jest.mock('@actions/core', () => ({
setSecret: jest.fn(),
info: jest.fn(),
warning: jest.fn(),
error: jest.fn(),
}));
jest.mock('../core/orchestrator-system', () => ({
OrchestratorSystem: {
Run: jest.fn().mockResolvedValue(''),
},
}));
jest.mock('../core/orchestrator-logger', () => ({
__esModule: true,
default: {
log: jest.fn(),
logWarning: jest.fn(),
error: jest.fn(),
},
}));
const mockFs = fs as jest.Mocked<typeof fs>;
describe('SecretSourceService', () => {
beforeEach(() => {
jest.clearAllMocks();
});
describe('validateSecretKey', () => {
it('should accept alphanumeric keys', () => {
expect(validateSecretKey('MY_SECRET_KEY')).toBe('MY_SECRET_KEY');
});
it('should accept keys with hyphens', () => {
expect(validateSecretKey('my-secret-key')).toBe('my-secret-key');
});
it('should accept keys with dots', () => {
expect(validateSecretKey('my.secret.key')).toBe('my.secret.key');
});
it('should accept keys with forward slashes', () => {
expect(validateSecretKey('path/to/secret')).toBe('path/to/secret');
});
it('should accept keys with mixed valid characters', () => {
expect(validateSecretKey('my-app/prod_db.password')).toBe('my-app/prod_db.password');
});
it('should reject keys with semicolons (shell injection)', () => {
expect(() => validateSecretKey('; rm -rf /')).toThrow('Invalid secret key name');
});
it('should reject keys with backticks (command substitution)', () => {
expect(() => validateSecretKey('`whoami`')).toThrow('Invalid secret key name');
});
it('should reject keys with dollar signs (variable expansion)', () => {
expect(() => validateSecretKey('$HOME')).toThrow('Invalid secret key name');
});
it('should reject keys with pipe characters', () => {
expect(() => validateSecretKey('key | cat /etc/passwd')).toThrow('Invalid secret key name');
});
it('should reject keys with ampersands', () => {
expect(() => validateSecretKey('key && echo pwned')).toThrow('Invalid secret key name');
});
it('should reject keys with newlines', () => {
expect(() => validateSecretKey('key\nmalicious')).toThrow('Invalid secret key name');
});
it('should reject keys with quotes', () => {
expect(() => validateSecretKey('"key"')).toThrow('Invalid secret key name');
expect(() => validateSecretKey("'key'")).toThrow('Invalid secret key name');
});
it('should reject keys with parentheses (subshell)', () => {
expect(() => validateSecretKey('$(whoami)')).toThrow('Invalid secret key name');
});
it('should reject empty keys', () => {
expect(() => validateSecretKey('')).toThrow('Invalid secret key name');
});
it('should reject keys with spaces', () => {
expect(() => validateSecretKey('key with spaces')).toThrow('Invalid secret key name');
});
});
describe('isPremadeSource', () => {
it('should return true for aws-secrets-manager', () => {
expect(SecretSourceService.isPremadeSource('aws-secrets-manager')).toBe(true);
});
it('should return true for aws-secret-manager (legacy alias)', () => {
expect(SecretSourceService.isPremadeSource('aws-secret-manager')).toBe(true);
});
it('should return true for aws-parameter-store', () => {
expect(SecretSourceService.isPremadeSource('aws-parameter-store')).toBe(true);
});
it('should return true for gcp-secret-manager', () => {
expect(SecretSourceService.isPremadeSource('gcp-secret-manager')).toBe(true);
});
it('should return true for azure-key-vault', () => {
expect(SecretSourceService.isPremadeSource('azure-key-vault')).toBe(true);
});
it('should return true for hashicorp-vault', () => {
expect(SecretSourceService.isPremadeSource('hashicorp-vault')).toBe(true);
});
it('should return true for hashicorp-vault-kv1', () => {
expect(SecretSourceService.isPremadeSource('hashicorp-vault-kv1')).toBe(true);
});
it('should return true for vault (short alias)', () => {
expect(SecretSourceService.isPremadeSource('vault')).toBe(true);
});
it('should return false for unknown source', () => {
expect(SecretSourceService.isPremadeSource('unknown-source')).toBe(false);
});
});
describe('getAvailableSources', () => {
it('should return all premade source names', () => {
const sources = SecretSourceService.getAvailableSources();
expect(sources).toContain('aws-secrets-manager');
expect(sources).toContain('aws-parameter-store');
expect(sources).toContain('gcp-secret-manager');
expect(sources).toContain('azure-key-vault');
expect(sources).toContain('hashicorp-vault');
expect(sources).toContain('hashicorp-vault-kv1');
expect(sources).toContain('vault');
expect(sources.length).toBeGreaterThanOrEqual(8);
});
});
describe('resolveSource', () => {
it('should resolve premade source by name', () => {
const source = SecretSourceService.resolveSource('aws-secrets-manager');
expect(source).toBeDefined();
expect(source!.name).toBe('aws-secrets-manager');
expect(source!.command).toContain('secretsmanager');
});
it('should resolve custom command with {0} placeholder', () => {
const source = SecretSourceService.resolveSource('vault kv get -field=value secret/{0}');
expect(source).toBeDefined();
expect(source!.name).toBe('custom-command');
expect(source!.command).toContain('{0}');
});
it('should resolve command with spaces as custom command', () => {
const source = SecretSourceService.resolveSource('my-tool get-secret');
expect(source).toBeDefined();
expect(source!.name).toBe('custom-command');
});
it('should return undefined for unknown single-word source', () => {
const source = SecretSourceService.resolveSource('unknown');
expect(source).toBeUndefined();
});
});
describe('fetchSecret', () => {
it('should run the command with {0} replaced by key', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue('my-secret-value');
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
const result = await SecretSourceService.fetchSecret(source, 'MY_SECRET');
expect(result).toBe('my-secret-value');
expect(OrchestratorSystem.Run).toHaveBeenCalledWith(expect.stringContaining('MY_SECRET'), false, true);
});
it('should parse JSON output when parseOutput is json-field', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue(JSON.stringify({ value: 'extracted-secret' }));
const source = {
name: 'test-source',
command: 'fetch {0}',
parseOutput: 'json-field' as const,
jsonField: 'value',
};
const result = await SecretSourceService.fetchSecret(source, 'KEY');
expect(result).toBe('extracted-secret');
});
it('should fall back to raw output on invalid JSON with json-field mode', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue('not-json');
const source = {
name: 'test-source',
command: 'fetch {0}',
parseOutput: 'json-field' as const,
jsonField: 'value',
};
const result = await SecretSourceService.fetchSecret(source, 'KEY');
expect(result).toBe('not-json');
});
it('should return empty string on command failure', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockRejectedValue(new Error('command not found'));
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
const result = await SecretSourceService.fetchSecret(source, 'KEY');
expect(result).toBe('');
});
it('should reject keys with shell injection characters', async () => {
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
await expect(SecretSourceService.fetchSecret(source, '; rm -rf /')).rejects.toThrow('Invalid secret key name');
});
it('should reject keys with command substitution', async () => {
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
await expect(SecretSourceService.fetchSecret(source, '$(whoami)')).rejects.toThrow('Invalid secret key name');
});
it('should reject keys with backtick command substitution', async () => {
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
await expect(SecretSourceService.fetchSecret(source, '`cat /etc/passwd`')).rejects.toThrow(
'Invalid secret key name',
);
});
it('should accept keys with valid path-like patterns', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue('secret-value');
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
const result = await SecretSourceService.fetchSecret(source, 'prod/database/password');
expect(result).toBe('secret-value');
});
it('should mask fetched secret values with core.setSecret', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue('super-secret-value');
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
await SecretSourceService.fetchSecret(source, 'MY_SECRET');
expect(core.setSecret).toHaveBeenCalledWith('super-secret-value');
});
it('should not mask empty secret values', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue('');
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
await SecretSourceService.fetchSecret(source, 'MY_SECRET');
expect(core.setSecret).not.toHaveBeenCalled();
});
it('should mask JSON-extracted secret values', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValue(JSON.stringify({ value: 'json-secret' }));
const source = {
name: 'test-source',
command: 'fetch {0}',
parseOutput: 'json-field' as const,
jsonField: 'value',
};
await SecretSourceService.fetchSecret(source, 'KEY');
expect(core.setSecret).toHaveBeenCalledWith('json-secret');
});
});
describe('fetchFromEnv', () => {
it('should return env var value when set', () => {
process.env.TEST_SECRET_KEY = 'env-value';
const result = SecretSourceService.fetchFromEnv('TEST_SECRET_KEY');
expect(result).toBe('env-value');
delete process.env.TEST_SECRET_KEY;
});
it('should return empty string when env var is not set', () => {
const result = SecretSourceService.fetchFromEnv('NONEXISTENT_KEY_12345');
expect(result).toBe('');
});
it('should mask env var values with core.setSecret', () => {
process.env.TEST_MASK_KEY = 'masked-env-value';
SecretSourceService.fetchFromEnv('TEST_MASK_KEY');
expect(core.setSecret).toHaveBeenCalledWith('masked-env-value');
delete process.env.TEST_MASK_KEY;
});
it('should not mask empty env var values', () => {
const result = SecretSourceService.fetchFromEnv('NONEXISTENT_KEY_99999');
expect(result).toBe('');
expect(core.setSecret).not.toHaveBeenCalled();
});
});
describe('fetchAll', () => {
it('should fetch all keys from env source', async () => {
process.env.KEY_A = 'val-a';
process.env.KEY_B = 'val-b';
const results = await SecretSourceService.fetchAll('env', ['KEY_A', 'KEY_B']);
expect(results.KEY_A).toBe('val-a');
expect(results.KEY_B).toBe('val-b');
delete process.env.KEY_A;
delete process.env.KEY_B;
});
it('should fetch all keys from premade source', async () => {
const { OrchestratorSystem } = require('../core/orchestrator-system');
OrchestratorSystem.Run.mockResolvedValueOnce('secret-1').mockResolvedValueOnce('secret-2');
const results = await SecretSourceService.fetchAll('aws-parameter-store', ['param1', 'param2']);
expect(results.param1).toBe('secret-1');
expect(results.param2).toBe('secret-2');
expect(OrchestratorSystem.Run).toHaveBeenCalledTimes(2);
});
it('should return empty results for unknown source', async () => {
const results = await SecretSourceService.fetchAll('unknown', ['key1']);
expect(results).toEqual({});
});
});
describe('loadFromYaml', () => {
it('should return empty array when file does not exist', () => {
(mockFs.existsSync as jest.Mock).mockReturnValue(false);
const result = SecretSourceService.loadFromYaml('/nonexistent.yml');
expect(result).toEqual([]);
});
it('should parse valid YAML source definitions', () => {
(mockFs.existsSync as jest.Mock).mockReturnValue(true);
(mockFs.readFileSync as jest.Mock).mockReturnValue(`
sources:
- name: my-vault
command: 'vault kv get -field=value secret/{0}'
- name: my-api
command: 'curl -s https://api.example.com/{0}'
parseOutput: json-field
jsonField: secret_value
`);
const result = SecretSourceService.loadFromYaml('/sources.yml');
expect(result).toHaveLength(2);
expect(result[0].name).toBe('my-vault');
expect(result[0].command).toBe('vault kv get -field=value secret/{0}');
expect(result[1].name).toBe('my-api');
expect(result[1].parseOutput).toBe('json-field');
expect(result[1].jsonField).toBe('secret_value');
});
it('should handle YAML with single source', () => {
(mockFs.existsSync as jest.Mock).mockReturnValue(true);
(mockFs.readFileSync as jest.Mock).mockReturnValue(`
sources:
- name: simple
command: echo {0}
`);
const result = SecretSourceService.loadFromYaml('/simple.yml');
expect(result).toHaveLength(1);
expect(result[0].name).toBe('simple');
});
it('should return empty array on parse error', () => {
(mockFs.existsSync as jest.Mock).mockReturnValue(true);
(mockFs.readFileSync as jest.Mock).mockImplementation(() => {
throw new Error('Permission denied');
});
const result = SecretSourceService.loadFromYaml('/error.yml');
expect(result).toEqual([]);
});
});
describe('premade source commands', () => {
it('aws-secrets-manager uses --query SecretString', () => {
const source = SecretSourceService.resolveSource('aws-secrets-manager')!;
expect(source.command).toContain('--query SecretString');
expect(source.command).toContain('--output text');
});
it('aws-parameter-store uses --with-decryption', () => {
const source = SecretSourceService.resolveSource('aws-parameter-store')!;
expect(source.command).toContain('--with-decryption');
expect(source.command).toContain('--query Parameter.Value');
});
it('gcp-secret-manager uses latest version', () => {
const source = SecretSourceService.resolveSource('gcp-secret-manager')!;
expect(source.command).toContain('latest');
});
it('azure-key-vault uses AZURE_VAULT_NAME env var', () => {
const source = SecretSourceService.resolveSource('azure-key-vault')!;
expect(source.command).toContain('$AZURE_VAULT_NAME');
});
it('hashicorp-vault uses vault kv get with VAULT_MOUNT', () => {
const source = SecretSourceService.resolveSource('hashicorp-vault')!;
expect(source.command).toContain('vault kv get');
expect(source.command).toContain('VAULT_MOUNT');
expect(source.command).toContain('-field=value');
});
it('hashicorp-vault-kv1 uses vault read for KV v1', () => {
const source = SecretSourceService.resolveSource('hashicorp-vault-kv1')!;
expect(source.command).toContain('vault read');
expect(source.command).toContain('-field=value');
});
it('vault alias resolves to same command as hashicorp-vault', () => {
const vault = SecretSourceService.resolveSource('vault')!;
const hashicorpVault = SecretSourceService.resolveSource('hashicorp-vault')!;
expect(vault.command).toBe(hashicorpVault.command);
});
});
});

View File

@@ -1,337 +0,0 @@
import fs from 'node:fs';
import * as core from '@actions/core';
import OrchestratorLogger from '../core/orchestrator-logger';
import { OrchestratorSystem } from '../core/orchestrator-system';
/**
* A secret source definition: how to fetch a secret value by key.
*/
export interface SecretSourceDefinition {
name: string;
command: string;
parseOutput?: 'raw' | 'json-field';
jsonField?: string;
}
/**
* Validate that a secret key name contains only safe characters.
* Prevents shell injection when keys are interpolated into commands.
*
* Allowed characters: alphanumeric, hyphens, underscores, dots, forward slashes.
*
* @param key - The secret key name to validate
* @returns The validated key (unchanged)
* @throws Error if the key contains disallowed characters
*/
export function validateSecretKey(key: string): string {
if (!/^[a-zA-Z0-9\-_./]+$/.test(key)) {
throw new Error(
`Invalid secret key name: "${key}". Keys may only contain alphanumeric characters, hyphens, underscores, dots, and forward slashes.`,
);
}
return key;
}
/**
* Mask a secret value so it does not appear in GitHub Actions logs.
* Empty or whitespace-only values are skipped (core.setSecret would be a no-op).
*/
function maskSecretValue(value: string): void {
if (value.trim().length > 0) {
core.setSecret(value);
}
}
/**
* Premade secret sources and custom YAML-based secret source definitions.
*
* Premade sources are string shortcuts that expand to shell commands:
* - `aws-secrets-manager` -- AWS Secrets Manager
* - `aws-parameter-store` -- AWS Systems Manager Parameter Store
* - `gcp-secret-manager` -- Google Cloud Secret Manager
* - `azure-key-vault` -- Azure Key Vault (requires AZURE_VAULT_NAME env var)
* - `hashicorp-vault` -- HashiCorp Vault KV v2 (requires VAULT_ADDR, optionally VAULT_MOUNT)
* - `hashicorp-vault-kv1` -- HashiCorp Vault KV v1 (requires VAULT_ADDR, optionally VAULT_MOUNT)
* - `env` -- Read from environment variables (no shell command needed)
*
* Custom YAML format:
* sources:
* - name: my-vault
* command: 'vault kv get -field=value secret/{0}'
* - name: my-api
* command: 'curl -s https://secrets.example.com/api/{0}'
* parseOutput: json-field
* jsonField: value
*/
export class SecretSourceService {
private static readonly premadeSources: Record<string, SecretSourceDefinition> = {
'aws-secrets-manager': {
name: 'aws-secrets-manager',
command: 'aws secretsmanager get-secret-value --secret-id {0} --query SecretString --output text',
parseOutput: 'raw',
},
'aws-secret-manager': {
// Alias for backward compatibility (original name in inputPullCommand)
name: 'aws-secret-manager',
command: 'aws secretsmanager get-secret-value --secret-id {0} --query SecretString --output text',
parseOutput: 'raw',
},
'aws-parameter-store': {
name: 'aws-parameter-store',
command: 'aws ssm get-parameter --name {0} --with-decryption --query Parameter.Value --output text',
parseOutput: 'raw',
},
'gcp-secret-manager': {
name: 'gcp-secret-manager',
command: 'gcloud secrets versions access latest --secret="{0}"',
parseOutput: 'raw',
},
'azure-key-vault': {
name: 'azure-key-vault',
command: 'az keyvault secret show --vault-name "$AZURE_VAULT_NAME" --name {0} --query value --output tsv',
parseOutput: 'raw',
},
'hashicorp-vault': {
// HashiCorp Vault KV v2 (default). Requires VAULT_ADDR env var.
// Optionally set VAULT_MOUNT to override the mount path (default: 'secret').
// Authentication is handled by VAULT_TOKEN or other Vault auth env vars.
name: 'hashicorp-vault',
command: 'vault kv get -mount="${VAULT_MOUNT:-secret}" -field=value {0}',
parseOutput: 'raw',
},
'hashicorp-vault-kv1': {
// HashiCorp Vault KV v1. Requires VAULT_ADDR env var.
// Optionally set VAULT_MOUNT to override the mount path (default: 'secret').
name: 'hashicorp-vault-kv1',
command: 'vault read -mount="${VAULT_MOUNT:-secret}" -field=value {0}',
parseOutput: 'raw',
},
vault: {
// Short alias for hashicorp-vault (KV v2)
name: 'vault',
command: 'vault kv get -mount="${VAULT_MOUNT:-secret}" -field=value {0}',
parseOutput: 'raw',
},
};
/**
* Check if a source name is a known premade source.
*/
static isPremadeSource(sourceName: string): boolean {
return sourceName in SecretSourceService.premadeSources;
}
/**
* Get the list of available premade source names.
*/
static getAvailableSources(): string[] {
return Object.keys(SecretSourceService.premadeSources);
}
/**
* Resolve a source name to a SecretSourceDefinition.
*
* - If the name matches a premade source, returns that definition.
* - If it looks like a shell command (contains spaces or {0}), wraps it as a custom command.
* - Otherwise, returns undefined.
*/
static resolveSource(sourceName: string): SecretSourceDefinition | undefined {
// Check premade sources
if (SecretSourceService.isPremadeSource(sourceName)) {
return SecretSourceService.premadeSources[sourceName];
}
// If it contains a placeholder or spaces, treat it as a raw command
if (sourceName.includes('{0}') || sourceName.includes(' ')) {
return {
name: 'custom-command',
command: sourceName,
parseOutput: 'raw',
};
}
return undefined;
}
/**
* Load custom secret source definitions from a YAML file.
*
* Expected format:
* sources:
* - name: my-source
* command: 'my-tool get-secret {0}'
* - name: my-api
* command: 'curl -s https://api.example.com/secrets/{0}'
* parseOutput: json-field
* jsonField: value
*/
static loadFromYaml(filePath: string): SecretSourceDefinition[] {
if (!fs.existsSync(filePath)) {
OrchestratorLogger.logWarning(`Secret source YAML not found: ${filePath}`);
return [];
}
try {
const content = fs.readFileSync(filePath, 'utf8');
const parsed = SecretSourceService.parseSimpleYaml(content);
return parsed;
} catch (error: any) {
OrchestratorLogger.logWarning(`Failed to parse secret source YAML: ${error.message}`);
return [];
}
}
/**
* Fetch a secret value using the given source definition.
*
* Validates the key against an allowlist pattern before interpolating it
* into the command string to prevent shell injection. The fetched secret
* value is masked via core.setSecret() so it does not leak in logs.
*
* @param source - The secret source definition to use
* @param key - The secret key to fetch
* @returns The secret value, or empty string on failure
*/
static async fetchSecret(source: SecretSourceDefinition, key: string): Promise<string> {
// Validate the key to prevent shell injection
validateSecretKey(key);
const command = source.command.replace(/\{0\}/g, key);
try {
const output = await OrchestratorSystem.Run(command, false, true);
let value: string;
if (source.parseOutput === 'json-field' && source.jsonField) {
try {
const parsed = JSON.parse(output);
value = parsed[source.jsonField] || '';
} catch {
OrchestratorLogger.logWarning(`Failed to parse JSON output from ${source.name} for key ${key}`);
value = output.trim();
}
} else {
value = output.trim();
}
// Mask the secret value so it does not appear in GitHub Actions logs
maskSecretValue(value);
return value;
} catch (error: any) {
OrchestratorLogger.logWarning(`Failed to fetch secret '${key}' from ${source.name}: ${error.message}`);
return '';
}
}
/**
* Fetch a secret from an environment variable. No shell command needed.
* The value is masked via core.setSecret() so it does not leak in logs.
*/
static fetchFromEnv(key: string): string {
const value = process.env[key] || '';
maskSecretValue(value);
return value;
}
/**
* Resolve a source name and fetch all secrets from it.
*
* @param sourceName - Premade source name, shell command, or 'env'
* @param keys - List of secret keys to fetch
* @returns Map of key -> value
*/
static async fetchAll(sourceName: string, keys: string[]): Promise<Record<string, string>> {
const results: Record<string, string> = {};
if (sourceName === 'env') {
for (const key of keys) {
results[key] = SecretSourceService.fetchFromEnv(key);
}
return results;
}
const source = SecretSourceService.resolveSource(sourceName);
if (!source) {
OrchestratorLogger.logWarning(
`Unknown secret source '${sourceName}'. Available sources: ${SecretSourceService.getAvailableSources().join(
', ',
)}`,
);
return results;
}
OrchestratorLogger.log(`Fetching ${keys.length} secret(s) from ${source.name}`);
for (const key of keys) {
results[key] = await SecretSourceService.fetchSecret(source, key);
}
return results;
}
/**
* Simple YAML parser for secret source definitions.
* Handles the specific structure we expect without requiring a YAML library.
*/
private static parseSimpleYaml(content: string): SecretSourceDefinition[] {
const definitions: SecretSourceDefinition[] = [];
const lines = content.split('\n');
let current: Partial<SecretSourceDefinition> | null = null;
for (const rawLine of lines) {
const line = rawLine.replace(/\r$/, '');
const trimmed = line.trim();
if (trimmed === '' || trimmed.startsWith('#')) continue;
if (trimmed === '- name:' || trimmed.startsWith('- name:')) {
if (current?.name && current?.command) {
definitions.push(current as SecretSourceDefinition);
}
current = {
name: trimmed
.replace('- name:', '')
.trim()
.replace(/^['"]|['"]$/g, ''),
parseOutput: 'raw',
};
continue;
}
if (current && trimmed.startsWith('command:')) {
current.command = trimmed
.replace('command:', '')
.trim()
.replace(/^['"]|['"]$/g, '');
} else if (current && trimmed.startsWith('parseOutput:')) {
const value = trimmed
.replace('parseOutput:', '')
.trim()
.replace(/^['"]|['"]$/g, '');
current.parseOutput = value as 'raw' | 'json-field';
} else if (current && trimmed.startsWith('jsonField:')) {
current.jsonField = trimmed
.replace('jsonField:', '')
.trim()
.replace(/^['"]|['"]$/g, '');
}
}
if (current?.name && current?.command) {
definitions.push(current as SecretSourceDefinition);
}
return definitions;
}
}

View File

@@ -0,0 +1,315 @@
import fs from 'node:fs';
import path from 'node:path';
import { OrchestratorSystem } from '../core/orchestrator-system';
import OrchestratorLogger from '../core/orchestrator-logger';
import { SyncState, SyncStrategy } from './sync-state';
import { SyncStateManager } from './sync-state-manager';
/**
* Service for incremental workspace synchronization.
*
* Supports multiple sync strategies:
* - full: Traditional clone + cache restore (default)
* - git-delta: Fetch and apply only changed files since last sync
* - direct-input: Apply file changes passed as job input (no git push required)
* - storage-pull: Fetch changed files from rclone-backed generic storage
*/
export class IncrementalSyncService {
/**
* Load sync state from the workspace.
*/
static loadSyncState(workspacePath: string, statePath?: string): SyncState | undefined {
return SyncStateManager.loadState(workspacePath, statePath);
}
/**
* Save sync state to the workspace.
*/
static saveSyncState(workspacePath: string, state: SyncState, statePath?: string): void {
SyncStateManager.saveState(workspacePath, state, statePath);
}
/**
* Determine the appropriate sync strategy based on workspace state and configuration.
*/
static resolveStrategy(requestedStrategy: SyncStrategy, workspacePath: string, statePath?: string): SyncStrategy {
if (requestedStrategy === 'full') {
return 'full';
}
// git-delta requires an existing sync state
if (requestedStrategy === 'git-delta') {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
OrchestratorLogger.log('[Sync] No sync state found, falling back to full sync');
return 'full';
}
return 'git-delta';
}
return requestedStrategy;
}
/**
* Execute a git-delta sync: fetch latest and apply only changed files.
*
* @param workspacePath - Path to the git workspace
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
* @param statePath - Optional custom path for sync state file
* @returns Number of files changed
*/
static async syncGitDelta(workspacePath: string, targetReference: string, statePath?: string): Promise<number> {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
throw new Error('Cannot git-delta sync without existing sync state');
}
OrchestratorLogger.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
// Fetch latest
await OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
// Get list of changed files
const diffOutput = await OrchestratorSystem.Run(
`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`,
true,
);
const changedFiles = diffOutput.split('\n').filter(Boolean);
OrchestratorLogger.log(`[Sync] ${changedFiles.length} file(s) changed`);
if (changedFiles.length > 0) {
// Checkout target ref
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
}
// Update sync state
const newState: SyncState = {
lastSyncCommit: targetReference,
lastSyncTimestamp: new Date().toISOString(),
workspaceHash: SyncStateManager.calculateWorkspaceHash(workspacePath),
pendingOverlays: state.pendingOverlays,
};
SyncStateManager.saveState(workspacePath, newState, statePath);
return changedFiles.length;
}
/**
* Apply a direct input overlay from a local archive or storage URI.
*
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
* For local paths, the archive is extracted directly.
*
* @param workspacePath - Path to the workspace
* @param inputReference - Local path or storage:// URI to the input archive
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
* @param statePath - Optional custom path for sync state file
* @returns List of overlay paths applied
*/
static async applyDirectInput(
workspacePath: string,
inputReference: string,
rcloneRemote?: string,
statePath?: string,
): Promise<string[]> {
let localArchive = inputReference;
// If storage URI, fetch via rclone first
if (inputReference.startsWith('storage://')) {
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
const remote = rcloneRemote || parsed.remote;
const remotePath = parsed.path;
localArchive = path.join(workspacePath, '.game-ci-input-overlay.tar');
OrchestratorLogger.log(`[Sync] Fetching input from storage: ${inputReference}`);
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, path.dirname(localArchive));
}
if (!fs.existsSync(localArchive)) {
throw new Error(`Input archive not found: ${localArchive}`);
}
OrchestratorLogger.log(`[Sync] Applying direct input overlay from ${localArchive}`);
// Extract overlay
await OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
// Track overlay in sync state
const state = SyncStateManager.loadState(workspacePath, statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(localArchive);
SyncStateManager.saveState(workspacePath, state, statePath);
return [localArchive];
}
/**
* Execute a storage-pull sync: pull changed files from an rclone remote.
*
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
* and overlays it onto the workspace. Supports two modes:
* - overlay: extract on top of existing workspace (default)
* - clean: fresh git checkout, then apply overlay
*
* @param workspacePath - Path to the workspace
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
* @param options - Configuration for the storage-pull operation
* @returns List of files pulled from storage
*/
static async syncStoragePull(
workspacePath: string,
storageUri: string,
options: {
rcloneRemote?: string;
cleanMode?: boolean;
syncRevertAfter?: boolean;
statePath?: string;
} = {},
): Promise<string[]> {
if (!storageUri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
}
// Verify rclone is available
try {
await OrchestratorSystem.Run('rclone version', true, true);
} catch {
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
}
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
const remote = options.rcloneRemote || parsed.remote;
const remotePath = parsed.path;
OrchestratorLogger.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
// Clean mode: reset workspace to clean git state before applying overlay
if (options.cleanMode) {
OrchestratorLogger.log('[Sync] Clean mode: resetting workspace to HEAD');
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
}
// Pull from remote storage directly into workspace
const rcloneSource = `${remote}:${remotePath}`;
await OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
// List what was pulled for tracking
let pulledFiles: string[] = [];
try {
const lsOutput = await OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
pulledFiles = lsOutput
.split('\n')
.filter(Boolean)
.map((line) => {
// rclone ls outputs: " <size> <path>"
const trimmed = line.trim();
const spaceIndex = trimmed.indexOf(' ');
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
})
.filter(Boolean);
} catch {
OrchestratorLogger.logWarning('[Sync] Could not list pulled files from remote');
}
OrchestratorLogger.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
// Update sync state with overlay tracking
const state = SyncStateManager.loadState(workspacePath, options.statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(storageUri);
state.lastSyncTimestamp = new Date().toISOString();
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
SyncStateManager.saveState(workspacePath, state, options.statePath);
return pulledFiles;
}
/**
* Parse a storage:// URI into remote and path components.
*
* Supported formats:
* - storage://remote:bucket/path (explicit remote with colon separator)
* - storage://remote/path (remote name is first path segment)
*
* @param uri - The storage:// URI to parse
* @returns Object with remote name and path
*/
static parseStorageUri(uri: string): { remote: string; path: string } {
if (!uri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
}
const stripped = uri.replace('storage://', '');
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
const colonIndex = stripped.indexOf(':');
if (colonIndex > 0) {
return {
remote: stripped.slice(0, colonIndex),
path: stripped.slice(colonIndex + 1),
};
}
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
const slashIndex = stripped.indexOf('/');
if (slashIndex > 0) {
return {
remote: stripped.slice(0, slashIndex),
path: stripped.slice(slashIndex + 1),
};
}
// Just a remote name with no path
return {
remote: stripped,
path: '',
};
}
/**
* Execute rclone copy with standard flags.
*/
private static async executeRcloneCopy(remote: string, remotePath: string, destinationPath: string): Promise<void> {
await OrchestratorSystem.Run(
`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`,
true,
);
}
/**
* Revert pending overlays by restoring git state.
*/
static async revertOverlays(workspacePath: string, statePath?: string): Promise<void> {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state || state.pendingOverlays.length === 0) {
return;
}
OrchestratorLogger.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
// Clean untracked files from overlays
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
state.pendingOverlays = [];
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
SyncStateManager.saveState(workspacePath, state, statePath);
OrchestratorLogger.log('[Sync] Overlays reverted');
}
}

View File

@@ -0,0 +1,544 @@
import fs from 'node:fs';
import path from 'node:path';
import { IncrementalSyncService } from './incremental-sync-service';
import { SyncStateManager } from './sync-state-manager';
import { SyncState } from './sync-state';
import { OrchestratorSystem } from '../core/orchestrator-system';
import OrchestratorLogger from '../core/orchestrator-logger';
// Mock dependencies
jest.mock('node:fs');
jest.mock('../core/orchestrator-system');
jest.mock('../core/orchestrator-logger');
const mockFs = fs as jest.Mocked<typeof fs>;
const mockSystem = OrchestratorSystem as jest.Mocked<typeof OrchestratorSystem>;
const mockLogger = OrchestratorLogger as jest.Mocked<typeof OrchestratorLogger>;
afterEach(() => {
jest.clearAllMocks();
jest.restoreAllMocks();
});
describe('IncrementalSyncService', () => {
const workspacePath = '/workspace/project';
describe('parseStorageUri', () => {
it('parses storage://remote:bucket/path format', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket/some/path');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/some/path' });
});
it('parses storage://remote/path format', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote/mybucket/path');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/path' });
});
it('parses storage://remote:bucket with no sub-path', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket' });
});
it('handles remote-only URI without path', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote');
expect(result).toEqual({ remote: 'myremote', path: '' });
});
it('throws on invalid URI without storage:// prefix', () => {
expect(() => IncrementalSyncService.parseStorageUri('http://example.com')).toThrow('Invalid storage URI');
});
it('throws on empty URI', () => {
expect(() => IncrementalSyncService.parseStorageUri('')).toThrow('Invalid storage URI');
});
});
describe('resolveStrategy', () => {
it('returns full when full is requested', () => {
const result = IncrementalSyncService.resolveStrategy('full', workspacePath);
expect(result).toBe('full');
});
it('returns git-delta when sync state exists', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
expect(result).toBe('git-delta');
});
it('falls back to full when git-delta requested but no sync state', () => {
mockFs.existsSync.mockReturnValue(false);
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
expect(result).toBe('full');
});
it('returns direct-input as-is', () => {
const result = IncrementalSyncService.resolveStrategy('direct-input', workspacePath);
expect(result).toBe('direct-input');
});
it('returns storage-pull as-is', () => {
const result = IncrementalSyncService.resolveStrategy('storage-pull', workspacePath);
expect(result).toBe('storage-pull');
});
});
describe('syncGitDelta', () => {
const targetReference = 'def456789';
beforeEach(() => {
const state: SyncState = {
lastSyncCommit: 'abc123456',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
});
it('fetches and checks out changed files', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce('file1.txt\nfile2.cs\n'); // git diff
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(result).toBe(2);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" fetch origin'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('diff --name-only abc123456..def456789'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('checkout def456789'), true);
});
it('skips checkout when no files changed', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce(''); // git diff (empty)
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(result).toBe(0);
// Should only have fetch + diff calls, no checkout
expect(mockSystem.Run).toHaveBeenCalledTimes(2);
});
it('throws when no sync state exists', async () => {
mockFs.existsSync.mockReturnValue(false);
await expect(IncrementalSyncService.syncGitDelta(workspacePath, targetReference)).rejects.toThrow(
'Cannot git-delta sync without existing sync state',
);
});
it('saves updated sync state after delta sync', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce('file1.txt\n'); // git diff
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.lastSyncCommit).toBe(targetReference);
});
});
describe('applyDirectInput', () => {
it('extracts a local archive to workspace', async () => {
const archivePath = '/tmp/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
if (p === archivePath) return true;
// State file path does not exist
return false;
});
mockSystem.Run.mockResolvedValueOnce(''); // tar extract
const result = await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
expect(result).toEqual([archivePath]);
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('tar -xf "/tmp/overlay.tar"'), true);
});
it('fetches archive from storage URI via rclone then extracts', async () => {
const storageUri = 'storage://s3remote:builds/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
const pathString = p.toString();
if (pathString.includes('.game-ci-input-overlay.tar')) return true;
return false;
});
mockSystem.Run.mockResolvedValue(''); // rclone copy + tar extract
const result = await IncrementalSyncService.applyDirectInput(workspacePath, storageUri);
expect(result.length).toBe(1);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "s3remote:builds/overlay.tar"'),
true,
);
});
it('throws when local archive does not exist', async () => {
mockFs.existsSync.mockReturnValue(false);
await expect(IncrementalSyncService.applyDirectInput(workspacePath, '/missing/archive.tar')).rejects.toThrow(
'Input archive not found',
);
});
it('tracks overlay in sync state', async () => {
const archivePath = '/tmp/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
if (p === archivePath) return true;
return false;
});
mockSystem.Run.mockResolvedValueOnce('');
await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toContain(archivePath);
});
});
describe('syncStoragePull', () => {
const storageUri = 'storage://s3:game-builds/latest';
beforeEach(() => {
mockFs.existsSync.mockReturnValue(false); // no existing state
});
it('pulls files from rclone remote into workspace', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version check
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(' 1234 file1.txt\n 5678 dir/file2.cs\n'); // rclone ls
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(result).toEqual(['file1.txt', 'dir/file2.cs']);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "s3:game-builds/latest" "/workspace/project"'),
true,
);
});
it('uses custom rclone remote when provided', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
rcloneRemote: 'custom-remote',
});
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "custom-remote:game-builds/latest"'),
true,
);
});
it('resets workspace in clean mode before pull', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // git checkout -- .
mockSystem.Run.mockResolvedValueOnce(''); // git clean -fd
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, { cleanMode: true });
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" clean -fd'),
true,
);
});
it('throws on invalid storage URI', async () => {
await expect(IncrementalSyncService.syncStoragePull(workspacePath, 'http://example.com')).rejects.toThrow(
'Invalid storage URI',
);
});
it('throws when rclone binary is not available', async () => {
mockSystem.Run.mockRejectedValueOnce(new Error('command not found: rclone'));
await expect(IncrementalSyncService.syncStoragePull(workspacePath, storageUri)).rejects.toThrow(
'rclone binary not found',
);
});
it('saves sync state with overlay tracking', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(' 100 a.txt\n'); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toContain(storageUri);
});
it('handles rclone ls failure gracefully', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockRejectedValueOnce(new Error('ls failed')); // rclone ls fails
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(result).toEqual([]);
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Could not list pulled files'));
});
});
describe('revertOverlays', () => {
it('reverts git state and cleans untracked files', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: ['/tmp/overlay.tar', 'storage://s3:builds/content'],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
mockSystem.Run.mockResolvedValue('');
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" clean -fd'),
true,
);
});
it('clears pending overlays in saved state', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: ['/tmp/overlay.tar'],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
mockSystem.Run.mockResolvedValue('');
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toEqual([]);
});
it('does nothing when no overlays are pending', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).not.toHaveBeenCalled();
});
it('does nothing when no sync state exists', async () => {
mockFs.existsSync.mockReturnValue(false);
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).not.toHaveBeenCalled();
});
});
});
describe('SyncStateManager', () => {
const workspacePath = '/workspace/project';
describe('loadState', () => {
it('returns parsed state from default path', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
const result = SyncStateManager.loadState(workspacePath);
expect(result).toEqual(state);
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, '.game-ci/sync-state.json'), 'utf8');
});
it('uses custom state path when provided', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
SyncStateManager.loadState(workspacePath, 'custom/state.json');
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, 'custom/state.json'), 'utf8');
});
it('returns undefined when state file does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
const result = SyncStateManager.loadState(workspacePath);
expect(result).toBeUndefined();
});
it('returns undefined and logs warning on malformed JSON', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue('not-valid-json{{{');
const result = SyncStateManager.loadState(workspacePath);
expect(result).toBeUndefined();
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to load sync state'));
});
});
describe('saveState', () => {
it('writes state to default path with pretty JSON', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: ['overlay1'],
};
mockFs.existsSync.mockReturnValue(true);
SyncStateManager.saveState(workspacePath, state);
expect(mockFs.writeFileSync).toHaveBeenCalledWith(
path.join(workspacePath, '.game-ci/sync-state.json'),
JSON.stringify(state, undefined, 2),
'utf8',
);
});
it('creates parent directories if they do not exist', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(false);
SyncStateManager.saveState(workspacePath, state);
expect(mockFs.mkdirSync).toHaveBeenCalledWith(expect.stringContaining('.game-ci'), { recursive: true });
});
it('logs warning on write failure instead of throwing', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.writeFileSync.mockImplementation(() => {
throw new Error('Permission denied');
});
// Should not throw
SyncStateManager.saveState(workspacePath, state);
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to save sync state'));
});
});
describe('calculateWorkspaceHash', () => {
it('produces a deterministic SHA-256 hash from workspace files', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockImplementation((filePath: fs.PathOrFileDescriptor) => {
const p = filePath.toString();
if (p.includes('ProjectVersion.txt')) return 'm_EditorVersion: 2022.3.10f1';
if (p.includes('manifest.json')) return '{"dependencies":{}}';
if (p.includes('packages-lock.json')) return '{"dependencies":{}}';
if (p.includes('csc.rsp')) return '-nullable+';
return '';
});
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash1).toBe(hash2);
expect(hash1).toHaveLength(64); // SHA-256 hex digest
});
it('produces different hashes for different workspace content', () => {
let callCount = 0;
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockImplementation(() => {
callCount++;
return callCount <= 4 ? 'content-v1' : 'content-v2';
});
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash1).not.toBe(hash2);
});
it('includes missing file markers in hash for absent files', () => {
mockFs.existsSync.mockReturnValue(false);
const hash = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash).toHaveLength(64);
});
});
describe('hasDrifted', () => {
it('returns false when workspace hash matches', () => {
mockFs.existsSync.mockReturnValue(false);
const savedHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
const result = SyncStateManager.hasDrifted(workspacePath, savedHash);
expect(result).toBe(false);
});
it('returns true when workspace hash differs', () => {
mockFs.existsSync.mockReturnValue(false);
const result = SyncStateManager.hasDrifted(workspacePath, 'some-old-hash-that-will-not-match');
expect(result).toBe(true);
});
});
});

View File

@@ -0,0 +1,3 @@
export { SyncState, SyncStrategy } from './sync-state';
export { IncrementalSyncService } from './incremental-sync-service';
export { SyncStateManager } from './sync-state-manager';

View File

@@ -0,0 +1,125 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import OrchestratorLogger from '../core/orchestrator-logger';
import { SyncState } from './sync-state';
/**
* Manages persistent sync state for incremental workspace updates.
*
* The sync state tracks what has been synced to a workspace, enabling
* delta-based updates instead of full clones. State is stored as a JSON
* file in the workspace (default: .game-ci/sync-state.json).
*/
export class SyncStateManager {
static readonly DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
/**
* Key workspace files whose content is hashed for drift detection.
* Changes to any of these files indicate the workspace may have been
* modified outside of the sync system.
*/
private static readonly WORKSPACE_HASH_FILES = [
'ProjectSettings/ProjectVersion.txt',
'Packages/manifest.json',
'Packages/packages-lock.json',
'Assets/csc.rsp',
];
/**
* Load sync state from the workspace.
*
* @param workspacePath - Root path of the workspace
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
* @returns The loaded sync state, or undefined if no state exists or parsing fails
*/
static loadState(workspacePath: string, statePath?: string): SyncState | undefined {
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
if (!fs.existsSync(resolvedPath)) {
return;
}
try {
const content = fs.readFileSync(resolvedPath, 'utf8');
return JSON.parse(content) as SyncState;
} catch {
OrchestratorLogger.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
return;
}
}
/**
* Save sync state to the workspace.
*
* Creates parent directories if they do not exist.
*
* @param workspacePath - Root path of the workspace
* @param state - The sync state to persist
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
*/
static saveState(workspacePath: string, state: SyncState, statePath?: string): void {
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
try {
const directory = path.dirname(resolvedPath);
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, { recursive: true });
}
fs.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
OrchestratorLogger.log(
`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`,
);
} catch (error: any) {
OrchestratorLogger.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
}
}
/**
* Calculate a SHA-256 hash of key workspace files for drift detection.
*
* Hashes the content of known workspace files (ProjectVersion.txt,
* manifest.json, etc.) to produce a fingerprint. If the hash changes
* between syncs, the workspace may have been modified externally.
*
* Files that do not exist are skipped (their absence is part of the hash).
*
* @param workspacePath - Root path of the workspace
* @returns Hex-encoded SHA-256 hash string
*/
static calculateWorkspaceHash(workspacePath: string): string {
const hash = crypto.createHash('sha256');
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
const filePath = path.join(workspacePath, relativePath);
try {
if (fs.existsSync(filePath)) {
const content = fs.readFileSync(filePath, 'utf8');
hash.update(`${relativePath}:${content}`);
} else {
hash.update(`${relativePath}:__missing__`);
}
} catch {
hash.update(`${relativePath}:__error__`);
}
}
return hash.digest('hex');
}
/**
* Check if the workspace has drifted from a previously saved hash.
*
* @param workspacePath - Root path of the workspace
* @param savedHash - The previously saved workspace hash to compare against
* @returns true if the current workspace hash differs from the saved hash
*/
static hasDrifted(workspacePath: string, savedHash: string): boolean {
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
return currentHash !== savedHash;
}
}

View File

@@ -0,0 +1,19 @@
/**
* Persistent sync state for incremental workspace updates.
* Stored on the runner to track what has already been synced.
*/
export interface SyncState {
/** Last successfully synced git commit SHA */
lastSyncCommit: string;
/** ISO 8601 timestamp of last sync */
lastSyncTimestamp: string;
/** SHA-256 hash of workspace state (optional) */
workspaceHash?: string;
/** List of overlay paths that haven't been reverted */
pendingOverlays: string[];
}
export type SyncStrategy = 'full' | 'git-delta' | 'direct-input' | 'storage-pull';