Compare commits

..

2 Commits

Author SHA1 Message Date
frostebite
3d8edb0711 fix: replace orchestrator-develop branch references with main
The orchestrator-develop branch no longer exists. Update all fallback
clone commands and test fixtures to use main instead.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-09 19:54:16 +00:00
frostebite
9cf65cf5fc feat: add CLAUDE.md for AI-assisted development
Adds a minimal CLAUDE.md covering project architecture, build system,
code conventions, testing, security considerations, and CI workflows.
Provides Claude Code with the context needed to assist with builds,
debugging, orchestrator work, and contributions.

Closes #801

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 18:18:28 +00:00
15 changed files with 85 additions and 1792 deletions

View File

@@ -1,5 +1,4 @@
{
"root": true,
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
"parser": "@typescript-eslint/parser",

View File

@@ -12,7 +12,6 @@ jobs:
buildForAllPlatformsMacOS:
name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
runs-on: macos-latest
continue-on-error: true
strategy:
fail-fast: false
matrix:

80
CLAUDE.md Normal file
View File

@@ -0,0 +1,80 @@
# Unity-Builder
GitHub Action and CLI that builds Unity projects for multiple platforms. Part of the [GameCI](https://game.ci) project.
## Quick Reference
```bash
yarn # install dependencies
yarn build # full build: tsc → ncc bundle (src/ → lib/ → dist/index.js)
yarn test # run all tests (jest)
yarn test:ci # run tests in CI mode (single-threaded, 2min timeout)
yarn lint # prettier + eslint check
yarn format # auto-format with prettier
```
## Architecture
**Entry point:** `src/index.ts` → decides between CLI mode and GitHub Action mode.
**Two execution paths:**
1. **Local builds** — Docker container or native macOS (`src/model/docker.ts`, `src/model/mac-builder.ts`)
2. **Orchestrator builds** — Remote execution on AWS ECS, Kubernetes, or other providers (`src/model/orchestrator/`)
**Key modules:**
| Path | Purpose |
|---|---|
| `src/model/build-parameters.ts` | Central config object — all build settings flow through here |
| `src/model/input.ts` | Input resolution with priority: Action inputs → CLI flags → env override → env vars |
| `src/model/orchestrator/orchestrator.ts` | Remote build orchestration — provider selection, workflow execution |
| `src/model/orchestrator/providers/` | Provider plugin system (AWS, K8s, Docker, Local, Test) |
| `src/model/orchestrator/remote-client/` | Code that runs inside remote containers (caching, hooks, artifacts) |
| `src/model/orchestrator/workflows/` | Build workflow types (standard, custom, async) |
| `src/model/orchestrator/services/` | Logging, locking, resource tracking |
| `src/model/cli/` | CLI mode using commander — dispatches to `@CliFunction`-decorated methods |
| `action.yml` | GitHub Action manifest — all inputs/outputs defined here |
| `dist/index.js` | Bundled output (committed to repo, used by action.yml at runtime) |
**Provider interface:** All providers implement `ProviderInterface` (`providers/provider-interface.ts`) with methods: `setupWorkflow`, `runTaskInWorkflow`, `cleanupWorkflow`, `garbageCollect`, `listResources`, `listWorkflow`, `watchWorkflow`.
**Provider loading:** Providers can be built-in, loaded from npm, cloned from GitHub repos, or loaded from local paths (`provider-loader.ts`).
## Build System
The build pipeline is: `yarn``tsc` (src/ → lib/) → `ncc build lib` (lib/ → dist/index.js).
- **dist/ is committed** — GitHub Actions loads `dist/index.js` directly, no install step on runners
- **Pre-commit hooks** (lefthook) auto-run formatting, linting, related tests, and `yarn build` to keep dist/ in sync
- Runtime: Node 20 (configured via Volta and action.yml `runs.using: node20`)
## Code Conventions
- **Files:** kebab-case (enforced by eslint `unicorn/filename-case`)
- **Code:** camelCase variables/functions, PascalCase classes/types
- **Formatting:** Prettier — 120 char width, single quotes, trailing commas, semicolons
- **Linting:** ESLint with unicorn, github, prettier, jest plugins
- **TypeScript:** strict mode, ES2020 target, CommonJS modules, experimental decorators enabled
- **Blank line before return statements** (enforced)
- **Blank line before block/line comments** (enforced)
- **No `for...in` loops** — use `for...of`
## Testing
- **Framework:** Jest 27 with ts-jest
- **Pattern:** `**/*.test.ts` files colocated with source
- **Orchestrator tests:** Concentrated in `src/model/orchestrator/tests/`
- **Run specific tests:** `yarn test -t "pattern"` or `yarn jest path/to/file.test.ts`
- **Orchestrator integration tests** require `orchestratorTests=true` env var: `cross-env orchestratorTests=true yarn test -i -t "orchestrator"`
## Security
- **Never log, output, or hardcode credentials** — cloud provider secrets (AWS, GCP, K8s), Unity serial keys, keystores, and private tokens must stay in secret inputs
- **Input validation matters** — user-supplied hook commands and custom parameters can be injection vectors; use `shell-quote` for shell escaping
- **Keystore/license data** is base64-encoded in inputs and written to temp files at build time
## CI Workflows
- `integrity-check.yml` — lint, test, build on every push/PR
- `build-tests-{ubuntu,windows,mac}.yml` — matrix builds across Unity versions and platforms
- `orchestrator-integrity.yml` / `orchestrator-async-checks.yml` — orchestrator-specific validation

View File

@@ -182,8 +182,8 @@ inputs:
required: false
default: ''
description:
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
the keys image, secrets (name, value object array), command line string)'
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
keys image, secrets (name, value object array), command line string)'
awsStackName:
default: 'game-ci'
required: false
@@ -279,24 +279,6 @@ inputs:
description:
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
fixes.'
syncStrategy:
description: 'Workspace sync strategy: full, git-delta, direct-input, storage-pull'
required: false
default: 'full'
syncInputRef:
description: 'URI for direct-input or storage-pull content (storage://remote/path or file path)'
required: false
syncStorageRemote:
description: 'rclone remote name for storage-backed inputs (defaults to rcloneRemote)'
required: false
syncRevertAfter:
description: 'Revert overlaid changes after job completion'
required: false
default: 'true'
syncStatePath:
description: 'Path to sync state file for delta tracking'
required: false
default: '.game-ci/sync-state.json'
outputs:
volume:

562
dist/index.js generated vendored
View File

@@ -38,7 +38,6 @@ const model_1 = __nccwpck_require__(41359);
const cli_1 = __nccwpck_require__(55651);
const mac_builder_1 = __importDefault(__nccwpck_require__(39364));
const platform_setup_1 = __importDefault(__nccwpck_require__(64423));
const sync_1 = __nccwpck_require__(98729);
async function runMain() {
try {
if (cli_1.Cli.InitCliMode()) {
@@ -53,12 +52,6 @@ async function runMain() {
let exitCode = -1;
if (buildParameters.providerStrategy === 'local') {
core.info('Building locally');
// Apply incremental sync strategy before build
const syncStrategy = buildParameters.syncStrategy;
if (syncStrategy !== 'full') {
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
await applySyncStrategy(buildParameters, workspace);
}
await platform_setup_1.default.setup(buildParameters, actionFolder);
exitCode =
process.platform === 'darwin'
@@ -68,16 +61,6 @@ async function runMain() {
actionFolder,
...buildParameters,
});
// Revert overlays after job completion if configured
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
core.info('[Sync] Reverting overlay changes after job completion');
try {
await sync_1.IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
}
catch (revertError) {
core.warning(`[Sync] Overlay revert failed: ${revertError.message}`);
}
}
}
else {
await model_1.Orchestrator.run(buildParameters, baseImage.toString());
@@ -95,47 +78,6 @@ async function runMain() {
core.setFailed(error.message);
}
}
/**
* Apply the configured sync strategy to the workspace before build.
*/
async function applySyncStrategy(buildParameters, workspace) {
const strategy = buildParameters.syncStrategy;
const resolvedStrategy = sync_1.IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
if (resolvedStrategy === 'full') {
core.info('[Sync] Resolved to full sync (no incremental state available)');
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
const changedFiles = await sync_1.IncrementalSyncService.syncGitDelta(workspace, targetReference, buildParameters.syncStatePath);
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
break;
}
case 'direct-input': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
}
const overlays = await sync_1.IncrementalSyncService.applyDirectInput(workspace, buildParameters.syncInputRef, buildParameters.syncStorageRemote || undefined, buildParameters.syncStatePath);
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
break;
}
case 'storage-pull': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
}
const pulledFiles = await sync_1.IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath: buildParameters.syncStatePath,
});
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
break;
}
default:
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
}
}
runMain();
@@ -433,11 +375,6 @@ class BuildParameters {
cacheUnityInstallationOnMac: input_1.default.cacheUnityInstallationOnMac,
unityHubVersionOnMac: input_1.default.unityHubVersionOnMac,
dockerWorkspacePath: input_1.default.dockerWorkspacePath,
syncStrategy: input_1.default.syncStrategy,
syncInputRef: input_1.default.syncInputRef,
syncStorageRemote: input_1.default.syncStorageRemote,
syncRevertAfter: input_1.default.syncRevertAfter,
syncStatePath: input_1.default.syncStatePath,
};
}
static parseBuildFile(filename, platform, androidExportType) {
@@ -1858,22 +1795,6 @@ class Input {
static get dockerWorkspacePath() {
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
}
static get syncStrategy() {
return Input.getInput('syncStrategy') ?? 'full';
}
static get syncInputRef() {
return Input.getInput('syncInputRef') ?? '';
}
static get syncStorageRemote() {
return Input.getInput('syncStorageRemote') ?? '';
}
static get syncRevertAfter() {
const input = Input.getInput('syncRevertAfter') ?? 'true';
return input === 'true';
}
static get syncStatePath() {
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
}
static get dockerCpuLimit() {
return Input.getInput('dockerCpuLimit') ?? node_os_1.default.cpus().length.toString();
}
@@ -7839,17 +7760,11 @@ const build_parameters_1 = __importDefault(__nccwpck_require__(80787));
const cli_1 = __nccwpck_require__(55651);
const orchestrator_options_1 = __importDefault(__nccwpck_require__(82473));
const resource_tracking_1 = __importDefault(__nccwpck_require__(42604));
const sync_1 = __nccwpck_require__(98729);
class RemoteClient {
static async setupRemoteClient() {
orchestrator_logger_1.default.log(`bootstrap game ci orchestrator...`);
await resource_tracking_1.default.logDiskUsageSnapshot('remote-cli-pre-build (start)');
const syncStrategy = (orchestrator_1.default.buildParameters.syncStrategy || 'full');
if (syncStrategy !== 'full') {
orchestrator_logger_1.default.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
await RemoteClient.handleIncrementalSync(syncStrategy);
}
else if (!(await RemoteClient.handleRetainedWorkspace())) {
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
@@ -7962,17 +7877,6 @@ class RemoteClient {
}
}
await RemoteClient.runCustomHookFiles(`after-build`);
// Revert sync overlays if configured
const syncStrategy = (orchestrator_1.default.buildParameters.syncStrategy || 'full');
if (orchestrator_1.default.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
try {
orchestrator_logger_1.default.log('[Sync] Reverting overlay changes after job completion');
await sync_1.IncrementalSyncService.revertOverlays(orchestrator_folders_1.OrchestratorFolders.repoPathAbsolute, orchestrator_1.default.buildParameters.syncStatePath);
}
catch (revertError) {
remote_client_logger_1.RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
}
}
// WIP - need to give the pod permissions to create config map
await remote_client_logger_1.RemoteClientLogger.handleLogManagementPostJob();
}
@@ -8041,67 +7945,6 @@ class RemoteClient {
remote_client_logger_1.RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
/**
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
*
* For git-delta: requires an existing workspace with sync state; fetches and applies
* only changed files.
*
* For direct-input and storage-pull: requires an existing workspace; applies overlay
* content on top.
*
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
*/
static async handleIncrementalSync(strategy) {
const buildParameters = orchestrator_1.default.buildParameters;
const workspacePath = orchestrator_folders_1.OrchestratorFolders.repoPathAbsolute;
const statePath = buildParameters.syncStatePath;
// Resolve strategy — may fall back to 'full' if no state exists
const resolvedStrategy = sync_1.IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
if (resolvedStrategy === 'full') {
orchestrator_logger_1.default.log('[Sync] Falling back to full bootstrap');
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
orchestrator_logger_1.default.log(`[Sync] Git delta sync to ${targetReference}`);
const changedFiles = await sync_1.IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
orchestrator_logger_1.default.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
break;
}
case 'direct-input': {
const inputReference = buildParameters.syncInputRef;
if (!inputReference) {
throw new Error('[Sync] direct-input strategy requires syncInputRef');
}
orchestrator_logger_1.default.log(`[Sync] Applying direct input: ${inputReference}`);
await sync_1.IncrementalSyncService.applyDirectInput(workspacePath, inputReference, buildParameters.syncStorageRemote || undefined, statePath);
break;
}
case 'storage-pull': {
const storageUri = buildParameters.syncInputRef;
if (!storageUri) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
}
orchestrator_logger_1.default.log(`[Sync] Storage pull from: ${storageUri}`);
await sync_1.IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath,
});
break;
}
default:
orchestrator_logger_1.default.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
}
}
static async bootstrapRepository() {
await orchestrator_system_1.OrchestratorSystem.Run(`mkdir -p ${orchestrator_folders_1.OrchestratorFolders.ToLinuxFolder(orchestrator_folders_1.OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`);
await orchestrator_system_1.OrchestratorSystem.Run(`mkdir -p ${orchestrator_folders_1.OrchestratorFolders.ToLinuxFolder(orchestrator_folders_1.OrchestratorFolders.cacheFolderForCacheKeyFull)}`);
@@ -9789,409 +9632,6 @@ class ContainerHookService {
exports.ContainerHookService = ContainerHookService;
/***/ }),
/***/ 96920:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.IncrementalSyncService = void 0;
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
const node_path_1 = __importDefault(__nccwpck_require__(49411));
const orchestrator_system_1 = __nccwpck_require__(9744);
const orchestrator_logger_1 = __importDefault(__nccwpck_require__(32549));
const sync_state_manager_1 = __nccwpck_require__(57595);
/**
* Service for incremental workspace synchronization.
*
* Supports multiple sync strategies:
* - full: Traditional clone + cache restore (default)
* - git-delta: Fetch and apply only changed files since last sync
* - direct-input: Apply file changes passed as job input (no git push required)
* - storage-pull: Fetch changed files from rclone-backed generic storage
*/
class IncrementalSyncService {
/**
* Load sync state from the workspace.
*/
static loadSyncState(workspacePath, statePath) {
return sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
}
/**
* Save sync state to the workspace.
*/
static saveSyncState(workspacePath, state, statePath) {
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
}
/**
* Determine the appropriate sync strategy based on workspace state and configuration.
*/
static resolveStrategy(requestedStrategy, workspacePath, statePath) {
if (requestedStrategy === 'full') {
return 'full';
}
// git-delta requires an existing sync state
if (requestedStrategy === 'git-delta') {
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
orchestrator_logger_1.default.log('[Sync] No sync state found, falling back to full sync');
return 'full';
}
return 'git-delta';
}
return requestedStrategy;
}
/**
* Execute a git-delta sync: fetch latest and apply only changed files.
*
* @param workspacePath - Path to the git workspace
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
* @param statePath - Optional custom path for sync state file
* @returns Number of files changed
*/
static async syncGitDelta(workspacePath, targetReference, statePath) {
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
throw new Error('Cannot git-delta sync without existing sync state');
}
orchestrator_logger_1.default.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
// Fetch latest
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
// Get list of changed files
const diffOutput = await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`, true);
const changedFiles = diffOutput.split('\n').filter(Boolean);
orchestrator_logger_1.default.log(`[Sync] ${changedFiles.length} file(s) changed`);
if (changedFiles.length > 0) {
// Checkout target ref
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
}
// Update sync state
const newState = {
lastSyncCommit: targetReference,
lastSyncTimestamp: new Date().toISOString(),
workspaceHash: sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath),
pendingOverlays: state.pendingOverlays,
};
sync_state_manager_1.SyncStateManager.saveState(workspacePath, newState, statePath);
return changedFiles.length;
}
/**
* Apply a direct input overlay from a local archive or storage URI.
*
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
* For local paths, the archive is extracted directly.
*
* @param workspacePath - Path to the workspace
* @param inputReference - Local path or storage:// URI to the input archive
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
* @param statePath - Optional custom path for sync state file
* @returns List of overlay paths applied
*/
static async applyDirectInput(workspacePath, inputReference, rcloneRemote, statePath) {
let localArchive = inputReference;
// If storage URI, fetch via rclone first
if (inputReference.startsWith('storage://')) {
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
const remote = rcloneRemote || parsed.remote;
const remotePath = parsed.path;
localArchive = node_path_1.default.join(workspacePath, '.game-ci-input-overlay.tar');
orchestrator_logger_1.default.log(`[Sync] Fetching input from storage: ${inputReference}`);
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, node_path_1.default.dirname(localArchive));
}
if (!node_fs_1.default.existsSync(localArchive)) {
throw new Error(`Input archive not found: ${localArchive}`);
}
orchestrator_logger_1.default.log(`[Sync] Applying direct input overlay from ${localArchive}`);
// Extract overlay
await orchestrator_system_1.OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
// Track overlay in sync state
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(localArchive);
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
return [localArchive];
}
/**
* Execute a storage-pull sync: pull changed files from an rclone remote.
*
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
* and overlays it onto the workspace. Supports two modes:
* - overlay: extract on top of existing workspace (default)
* - clean: fresh git checkout, then apply overlay
*
* @param workspacePath - Path to the workspace
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
* @param options - Configuration for the storage-pull operation
* @returns List of files pulled from storage
*/
static async syncStoragePull(workspacePath, storageUri, options = {}) {
if (!storageUri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
}
// Verify rclone is available
try {
await orchestrator_system_1.OrchestratorSystem.Run('rclone version', true, true);
}
catch {
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
}
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
const remote = options.rcloneRemote || parsed.remote;
const remotePath = parsed.path;
orchestrator_logger_1.default.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
// Clean mode: reset workspace to clean git state before applying overlay
if (options.cleanMode) {
orchestrator_logger_1.default.log('[Sync] Clean mode: resetting workspace to HEAD');
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
}
// Pull from remote storage directly into workspace
const rcloneSource = `${remote}:${remotePath}`;
await orchestrator_system_1.OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
// List what was pulled for tracking
let pulledFiles = [];
try {
const lsOutput = await orchestrator_system_1.OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
pulledFiles = lsOutput
.split('\n')
.filter(Boolean)
.map((line) => {
// rclone ls outputs: " <size> <path>"
const trimmed = line.trim();
const spaceIndex = trimmed.indexOf(' ');
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
})
.filter(Boolean);
}
catch {
orchestrator_logger_1.default.logWarning('[Sync] Could not list pulled files from remote');
}
orchestrator_logger_1.default.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
// Update sync state with overlay tracking
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, options.statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(storageUri);
state.lastSyncTimestamp = new Date().toISOString();
state.workspaceHash = sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath);
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, options.statePath);
return pulledFiles;
}
/**
* Parse a storage:// URI into remote and path components.
*
* Supported formats:
* - storage://remote:bucket/path (explicit remote with colon separator)
* - storage://remote/path (remote name is first path segment)
*
* @param uri - The storage:// URI to parse
* @returns Object with remote name and path
*/
static parseStorageUri(uri) {
if (!uri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
}
const stripped = uri.replace('storage://', '');
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
const colonIndex = stripped.indexOf(':');
if (colonIndex > 0) {
return {
remote: stripped.slice(0, colonIndex),
path: stripped.slice(colonIndex + 1),
};
}
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
const slashIndex = stripped.indexOf('/');
if (slashIndex > 0) {
return {
remote: stripped.slice(0, slashIndex),
path: stripped.slice(slashIndex + 1),
};
}
// Just a remote name with no path
return {
remote: stripped,
path: '',
};
}
/**
* Execute rclone copy with standard flags.
*/
static async executeRcloneCopy(remote, remotePath, destinationPath) {
await orchestrator_system_1.OrchestratorSystem.Run(`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`, true);
}
/**
* Revert pending overlays by restoring git state.
*/
static async revertOverlays(workspacePath, statePath) {
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
if (!state || state.pendingOverlays.length === 0) {
return;
}
orchestrator_logger_1.default.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
// Clean untracked files from overlays
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
state.pendingOverlays = [];
state.workspaceHash = sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath);
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
orchestrator_logger_1.default.log('[Sync] Overlays reverted');
}
}
exports.IncrementalSyncService = IncrementalSyncService;
/***/ }),
/***/ 98729:
/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => {
"use strict";
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.SyncStateManager = exports.IncrementalSyncService = void 0;
var incremental_sync_service_1 = __nccwpck_require__(96920);
Object.defineProperty(exports, "IncrementalSyncService", ({ enumerable: true, get: function () { return incremental_sync_service_1.IncrementalSyncService; } }));
var sync_state_manager_1 = __nccwpck_require__(57595);
Object.defineProperty(exports, "SyncStateManager", ({ enumerable: true, get: function () { return sync_state_manager_1.SyncStateManager; } }));
/***/ }),
/***/ 57595:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.SyncStateManager = void 0;
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
const node_path_1 = __importDefault(__nccwpck_require__(49411));
const node_crypto_1 = __importDefault(__nccwpck_require__(6005));
const orchestrator_logger_1 = __importDefault(__nccwpck_require__(32549));
/**
* Manages persistent sync state for incremental workspace updates.
*
* The sync state tracks what has been synced to a workspace, enabling
* delta-based updates instead of full clones. State is stored as a JSON
* file in the workspace (default: .game-ci/sync-state.json).
*/
class SyncStateManager {
/**
* Load sync state from the workspace.
*
* @param workspacePath - Root path of the workspace
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
* @returns The loaded sync state, or undefined if no state exists or parsing fails
*/
static loadState(workspacePath, statePath) {
const resolvedPath = node_path_1.default.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
if (!node_fs_1.default.existsSync(resolvedPath)) {
return;
}
try {
const content = node_fs_1.default.readFileSync(resolvedPath, 'utf8');
return JSON.parse(content);
}
catch {
orchestrator_logger_1.default.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
return;
}
}
/**
* Save sync state to the workspace.
*
* Creates parent directories if they do not exist.
*
* @param workspacePath - Root path of the workspace
* @param state - The sync state to persist
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
*/
static saveState(workspacePath, state, statePath) {
const resolvedPath = node_path_1.default.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
try {
const directory = node_path_1.default.dirname(resolvedPath);
if (!node_fs_1.default.existsSync(directory)) {
node_fs_1.default.mkdirSync(directory, { recursive: true });
}
node_fs_1.default.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
orchestrator_logger_1.default.log(`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`);
}
catch (error) {
orchestrator_logger_1.default.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
}
}
/**
* Calculate a SHA-256 hash of key workspace files for drift detection.
*
* Hashes the content of known workspace files (ProjectVersion.txt,
* manifest.json, etc.) to produce a fingerprint. If the hash changes
* between syncs, the workspace may have been modified externally.
*
* Files that do not exist are skipped (their absence is part of the hash).
*
* @param workspacePath - Root path of the workspace
* @returns Hex-encoded SHA-256 hash string
*/
static calculateWorkspaceHash(workspacePath) {
const hash = node_crypto_1.default.createHash('sha256');
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
const filePath = node_path_1.default.join(workspacePath, relativePath);
try {
if (node_fs_1.default.existsSync(filePath)) {
const content = node_fs_1.default.readFileSync(filePath, 'utf8');
hash.update(`${relativePath}:${content}`);
}
else {
hash.update(`${relativePath}:__missing__`);
}
}
catch {
hash.update(`${relativePath}:__error__`);
}
}
return hash.digest('hex');
}
/**
* Check if the workspace has drifted from a previously saved hash.
*
* @param workspacePath - Root path of the workspace
* @param savedHash - The previously saved workspace hash to compare against
* @returns true if the current workspace hash differs from the saved hash
*/
static hasDrifted(workspacePath, savedHash) {
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
return currentHash !== savedHash;
}
}
exports.SyncStateManager = SyncStateManager;
SyncStateManager.DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
/**
* Key workspace files whose content is hashed for drift detection.
* Changes to any of these files indicate the workspace may have been
* modified outside of the sync system.
*/
SyncStateManager.WORKSPACE_HASH_FILES = [
'ProjectSettings/ProjectVersion.txt',
'Packages/manifest.json',
'Packages/packages-lock.json',
'Assets/csc.rsp',
];
/***/ }),
/***/ 23451:

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@@ -3,8 +3,6 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup';
import { IncrementalSyncService } from './model/orchestrator/services/sync';
import { SyncStrategy } from './model/orchestrator/services/sync/sync-state';
async function runMain() {
try {
@@ -25,14 +23,6 @@ async function runMain() {
if (buildParameters.providerStrategy === 'local') {
core.info('Building locally');
// Apply incremental sync strategy before build
const syncStrategy = buildParameters.syncStrategy as SyncStrategy;
if (syncStrategy !== 'full') {
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
await applySyncStrategy(buildParameters, workspace);
}
await PlatformSetup.setup(buildParameters, actionFolder);
exitCode =
process.platform === 'darwin'
@@ -42,16 +32,6 @@ async function runMain() {
actionFolder,
...buildParameters,
});
// Revert overlays after job completion if configured
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
core.info('[Sync] Reverting overlay changes after job completion');
try {
await IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
} catch (revertError) {
core.warning(`[Sync] Overlay revert failed: ${(revertError as Error).message}`);
}
}
} else {
await Orchestrator.run(buildParameters, baseImage.toString());
exitCode = 0;
@@ -70,58 +50,4 @@ async function runMain() {
}
}
/**
* Apply the configured sync strategy to the workspace before build.
*/
async function applySyncStrategy(buildParameters: BuildParameters, workspace: string): Promise<void> {
const strategy = buildParameters.syncStrategy as SyncStrategy;
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
if (resolvedStrategy === 'full') {
core.info('[Sync] Resolved to full sync (no incremental state available)');
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
const changedFiles = await IncrementalSyncService.syncGitDelta(
workspace,
targetReference,
buildParameters.syncStatePath,
);
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
break;
}
case 'direct-input': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
}
const overlays = await IncrementalSyncService.applyDirectInput(
workspace,
buildParameters.syncInputRef,
buildParameters.syncStorageRemote || undefined,
buildParameters.syncStatePath,
);
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
break;
}
case 'storage-pull': {
if (!buildParameters.syncInputRef) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
}
const pulledFiles = await IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath: buildParameters.syncStatePath,
});
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
break;
}
default:
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
}
}
runMain();

View File

@@ -106,11 +106,6 @@ class BuildParameters {
public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string;
public dockerWorkspacePath!: string;
public syncStrategy!: string;
public syncInputRef!: string;
public syncStorageRemote!: string;
public syncRevertAfter!: boolean;
public syncStatePath!: string;
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
@@ -247,11 +242,6 @@ class BuildParameters {
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac,
dockerWorkspacePath: Input.dockerWorkspacePath,
syncStrategy: Input.syncStrategy,
syncInputRef: Input.syncInputRef,
syncStorageRemote: Input.syncStorageRemote,
syncRevertAfter: Input.syncRevertAfter,
syncStatePath: Input.syncStatePath,
};
}

View File

@@ -241,28 +241,6 @@ class Input {
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
}
static get syncStrategy(): string {
return Input.getInput('syncStrategy') ?? 'full';
}
static get syncInputRef(): string {
return Input.getInput('syncInputRef') ?? '';
}
static get syncStorageRemote(): string {
return Input.getInput('syncStorageRemote') ?? '';
}
static get syncRevertAfter(): boolean {
const input = Input.getInput('syncRevertAfter') ?? 'true';
return input === 'true';
}
static get syncStatePath(): string {
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
}
static get dockerCpuLimit(): string {
return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
}

View File

@@ -15,24 +15,15 @@ import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import OrchestratorOptions from '../options/orchestrator-options';
import ResourceTracking from '../services/core/resource-tracking';
import { IncrementalSyncService } from '../services/sync';
import { SyncStrategy } from '../services/sync/sync-state';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() {
OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
if (syncStrategy !== 'full') {
OrchestratorLogger.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
await RemoteClient.handleIncrementalSync(syncStrategy);
} else if (!(await RemoteClient.handleRetainedWorkspace())) {
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.runCustomHookFiles(`before-build`);
}
@@ -166,20 +157,6 @@ export class RemoteClient {
await RemoteClient.runCustomHookFiles(`after-build`);
// Revert sync overlays if configured
const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
if (Orchestrator.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
try {
OrchestratorLogger.log('[Sync] Reverting overlay changes after job completion');
await IncrementalSyncService.revertOverlays(
OrchestratorFolders.repoPathAbsolute,
Orchestrator.buildParameters.syncStatePath,
);
} catch (revertError: any) {
RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
}
}
// WIP - need to give the pod permissions to create config map
await RemoteClientLogger.handleLogManagementPostJob();
} catch (error: any) {
@@ -252,78 +229,6 @@ export class RemoteClient {
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
/**
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
*
* For git-delta: requires an existing workspace with sync state; fetches and applies
* only changed files.
*
* For direct-input and storage-pull: requires an existing workspace; applies overlay
* content on top.
*
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
*/
private static async handleIncrementalSync(strategy: SyncStrategy): Promise<void> {
const buildParameters = Orchestrator.buildParameters;
const workspacePath = OrchestratorFolders.repoPathAbsolute;
const statePath = buildParameters.syncStatePath;
// Resolve strategy — may fall back to 'full' if no state exists
const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
if (resolvedStrategy === 'full') {
OrchestratorLogger.log('[Sync] Falling back to full bootstrap');
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
return;
}
switch (resolvedStrategy) {
case 'git-delta': {
const targetReference = buildParameters.gitSha || buildParameters.branch;
OrchestratorLogger.log(`[Sync] Git delta sync to ${targetReference}`);
const changedFiles = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
OrchestratorLogger.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
break;
}
case 'direct-input': {
const inputReference = buildParameters.syncInputRef;
if (!inputReference) {
throw new Error('[Sync] direct-input strategy requires syncInputRef');
}
OrchestratorLogger.log(`[Sync] Applying direct input: ${inputReference}`);
await IncrementalSyncService.applyDirectInput(
workspacePath,
inputReference,
buildParameters.syncStorageRemote || undefined,
statePath,
);
break;
}
case 'storage-pull': {
const storageUri = buildParameters.syncInputRef;
if (!storageUri) {
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
}
OrchestratorLogger.log(`[Sync] Storage pull from: ${storageUri}`);
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
rcloneRemote: buildParameters.syncStorageRemote || undefined,
syncRevertAfter: buildParameters.syncRevertAfter,
statePath,
});
break;
}
default:
OrchestratorLogger.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
}
}
public static async bootstrapRepository() {
await OrchestratorSystem.Run(
`mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,

View File

@@ -1,315 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import { OrchestratorSystem } from '../core/orchestrator-system';
import OrchestratorLogger from '../core/orchestrator-logger';
import { SyncState, SyncStrategy } from './sync-state';
import { SyncStateManager } from './sync-state-manager';
/**
* Service for incremental workspace synchronization.
*
* Supports multiple sync strategies:
* - full: Traditional clone + cache restore (default)
* - git-delta: Fetch and apply only changed files since last sync
* - direct-input: Apply file changes passed as job input (no git push required)
* - storage-pull: Fetch changed files from rclone-backed generic storage
*/
export class IncrementalSyncService {
/**
* Load sync state from the workspace.
*/
static loadSyncState(workspacePath: string, statePath?: string): SyncState | undefined {
return SyncStateManager.loadState(workspacePath, statePath);
}
/**
* Save sync state to the workspace.
*/
static saveSyncState(workspacePath: string, state: SyncState, statePath?: string): void {
SyncStateManager.saveState(workspacePath, state, statePath);
}
/**
* Determine the appropriate sync strategy based on workspace state and configuration.
*/
static resolveStrategy(requestedStrategy: SyncStrategy, workspacePath: string, statePath?: string): SyncStrategy {
if (requestedStrategy === 'full') {
return 'full';
}
// git-delta requires an existing sync state
if (requestedStrategy === 'git-delta') {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
OrchestratorLogger.log('[Sync] No sync state found, falling back to full sync');
return 'full';
}
return 'git-delta';
}
return requestedStrategy;
}
/**
* Execute a git-delta sync: fetch latest and apply only changed files.
*
* @param workspacePath - Path to the git workspace
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
* @param statePath - Optional custom path for sync state file
* @returns Number of files changed
*/
static async syncGitDelta(workspacePath: string, targetReference: string, statePath?: string): Promise<number> {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state) {
throw new Error('Cannot git-delta sync without existing sync state');
}
OrchestratorLogger.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
// Fetch latest
await OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
// Get list of changed files
const diffOutput = await OrchestratorSystem.Run(
`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`,
true,
);
const changedFiles = diffOutput.split('\n').filter(Boolean);
OrchestratorLogger.log(`[Sync] ${changedFiles.length} file(s) changed`);
if (changedFiles.length > 0) {
// Checkout target ref
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
}
// Update sync state
const newState: SyncState = {
lastSyncCommit: targetReference,
lastSyncTimestamp: new Date().toISOString(),
workspaceHash: SyncStateManager.calculateWorkspaceHash(workspacePath),
pendingOverlays: state.pendingOverlays,
};
SyncStateManager.saveState(workspacePath, newState, statePath);
return changedFiles.length;
}
/**
* Apply a direct input overlay from a local archive or storage URI.
*
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
* For local paths, the archive is extracted directly.
*
* @param workspacePath - Path to the workspace
* @param inputReference - Local path or storage:// URI to the input archive
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
* @param statePath - Optional custom path for sync state file
* @returns List of overlay paths applied
*/
static async applyDirectInput(
workspacePath: string,
inputReference: string,
rcloneRemote?: string,
statePath?: string,
): Promise<string[]> {
let localArchive = inputReference;
// If storage URI, fetch via rclone first
if (inputReference.startsWith('storage://')) {
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
const remote = rcloneRemote || parsed.remote;
const remotePath = parsed.path;
localArchive = path.join(workspacePath, '.game-ci-input-overlay.tar');
OrchestratorLogger.log(`[Sync] Fetching input from storage: ${inputReference}`);
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, path.dirname(localArchive));
}
if (!fs.existsSync(localArchive)) {
throw new Error(`Input archive not found: ${localArchive}`);
}
OrchestratorLogger.log(`[Sync] Applying direct input overlay from ${localArchive}`);
// Extract overlay
await OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
// Track overlay in sync state
const state = SyncStateManager.loadState(workspacePath, statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(localArchive);
SyncStateManager.saveState(workspacePath, state, statePath);
return [localArchive];
}
/**
* Execute a storage-pull sync: pull changed files from an rclone remote.
*
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
* and overlays it onto the workspace. Supports two modes:
* - overlay: extract on top of existing workspace (default)
* - clean: fresh git checkout, then apply overlay
*
* @param workspacePath - Path to the workspace
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
* @param options - Configuration for the storage-pull operation
* @returns List of files pulled from storage
*/
static async syncStoragePull(
workspacePath: string,
storageUri: string,
options: {
rcloneRemote?: string;
cleanMode?: boolean;
syncRevertAfter?: boolean;
statePath?: string;
} = {},
): Promise<string[]> {
if (!storageUri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
}
// Verify rclone is available
try {
await OrchestratorSystem.Run('rclone version', true, true);
} catch {
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
}
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
const remote = options.rcloneRemote || parsed.remote;
const remotePath = parsed.path;
OrchestratorLogger.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
// Clean mode: reset workspace to clean git state before applying overlay
if (options.cleanMode) {
OrchestratorLogger.log('[Sync] Clean mode: resetting workspace to HEAD');
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
}
// Pull from remote storage directly into workspace
const rcloneSource = `${remote}:${remotePath}`;
await OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
// List what was pulled for tracking
let pulledFiles: string[] = [];
try {
const lsOutput = await OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
pulledFiles = lsOutput
.split('\n')
.filter(Boolean)
.map((line) => {
// rclone ls outputs: " <size> <path>"
const trimmed = line.trim();
const spaceIndex = trimmed.indexOf(' ');
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
})
.filter(Boolean);
} catch {
OrchestratorLogger.logWarning('[Sync] Could not list pulled files from remote');
}
OrchestratorLogger.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
// Update sync state with overlay tracking
const state = SyncStateManager.loadState(workspacePath, options.statePath) || {
lastSyncCommit: '',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
state.pendingOverlays.push(storageUri);
state.lastSyncTimestamp = new Date().toISOString();
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
SyncStateManager.saveState(workspacePath, state, options.statePath);
return pulledFiles;
}
/**
* Parse a storage:// URI into remote and path components.
*
* Supported formats:
* - storage://remote:bucket/path (explicit remote with colon separator)
* - storage://remote/path (remote name is first path segment)
*
* @param uri - The storage:// URI to parse
* @returns Object with remote name and path
*/
static parseStorageUri(uri: string): { remote: string; path: string } {
if (!uri.startsWith('storage://')) {
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
}
const stripped = uri.replace('storage://', '');
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
const colonIndex = stripped.indexOf(':');
if (colonIndex > 0) {
return {
remote: stripped.slice(0, colonIndex),
path: stripped.slice(colonIndex + 1),
};
}
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
const slashIndex = stripped.indexOf('/');
if (slashIndex > 0) {
return {
remote: stripped.slice(0, slashIndex),
path: stripped.slice(slashIndex + 1),
};
}
// Just a remote name with no path
return {
remote: stripped,
path: '',
};
}
/**
* Execute rclone copy with standard flags.
*/
private static async executeRcloneCopy(remote: string, remotePath: string, destinationPath: string): Promise<void> {
await OrchestratorSystem.Run(
`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`,
true,
);
}
/**
* Revert pending overlays by restoring git state.
*/
static async revertOverlays(workspacePath: string, statePath?: string): Promise<void> {
const state = SyncStateManager.loadState(workspacePath, statePath);
if (!state || state.pendingOverlays.length === 0) {
return;
}
OrchestratorLogger.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
await OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
// Clean untracked files from overlays
await OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
state.pendingOverlays = [];
state.workspaceHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
SyncStateManager.saveState(workspacePath, state, statePath);
OrchestratorLogger.log('[Sync] Overlays reverted');
}
}

View File

@@ -1,544 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import { IncrementalSyncService } from './incremental-sync-service';
import { SyncStateManager } from './sync-state-manager';
import { SyncState } from './sync-state';
import { OrchestratorSystem } from '../core/orchestrator-system';
import OrchestratorLogger from '../core/orchestrator-logger';
// Mock dependencies
jest.mock('node:fs');
jest.mock('../core/orchestrator-system');
jest.mock('../core/orchestrator-logger');
const mockFs = fs as jest.Mocked<typeof fs>;
const mockSystem = OrchestratorSystem as jest.Mocked<typeof OrchestratorSystem>;
const mockLogger = OrchestratorLogger as jest.Mocked<typeof OrchestratorLogger>;
afterEach(() => {
jest.clearAllMocks();
jest.restoreAllMocks();
});
describe('IncrementalSyncService', () => {
const workspacePath = '/workspace/project';
describe('parseStorageUri', () => {
it('parses storage://remote:bucket/path format', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket/some/path');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/some/path' });
});
it('parses storage://remote/path format', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote/mybucket/path');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket/path' });
});
it('parses storage://remote:bucket with no sub-path', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote:mybucket');
expect(result).toEqual({ remote: 'myremote', path: 'mybucket' });
});
it('handles remote-only URI without path', () => {
const result = IncrementalSyncService.parseStorageUri('storage://myremote');
expect(result).toEqual({ remote: 'myremote', path: '' });
});
it('throws on invalid URI without storage:// prefix', () => {
expect(() => IncrementalSyncService.parseStorageUri('http://example.com')).toThrow('Invalid storage URI');
});
it('throws on empty URI', () => {
expect(() => IncrementalSyncService.parseStorageUri('')).toThrow('Invalid storage URI');
});
});
describe('resolveStrategy', () => {
it('returns full when full is requested', () => {
const result = IncrementalSyncService.resolveStrategy('full', workspacePath);
expect(result).toBe('full');
});
it('returns git-delta when sync state exists', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
expect(result).toBe('git-delta');
});
it('falls back to full when git-delta requested but no sync state', () => {
mockFs.existsSync.mockReturnValue(false);
const result = IncrementalSyncService.resolveStrategy('git-delta', workspacePath);
expect(result).toBe('full');
});
it('returns direct-input as-is', () => {
const result = IncrementalSyncService.resolveStrategy('direct-input', workspacePath);
expect(result).toBe('direct-input');
});
it('returns storage-pull as-is', () => {
const result = IncrementalSyncService.resolveStrategy('storage-pull', workspacePath);
expect(result).toBe('storage-pull');
});
});
describe('syncGitDelta', () => {
const targetReference = 'def456789';
beforeEach(() => {
const state: SyncState = {
lastSyncCommit: 'abc123456',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
});
it('fetches and checks out changed files', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce('file1.txt\nfile2.cs\n'); // git diff
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(result).toBe(2);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" fetch origin'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('diff --name-only abc123456..def456789'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('checkout def456789'), true);
});
it('skips checkout when no files changed', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce(''); // git diff (empty)
const result = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(result).toBe(0);
// Should only have fetch + diff calls, no checkout
expect(mockSystem.Run).toHaveBeenCalledTimes(2);
});
it('throws when no sync state exists', async () => {
mockFs.existsSync.mockReturnValue(false);
await expect(IncrementalSyncService.syncGitDelta(workspacePath, targetReference)).rejects.toThrow(
'Cannot git-delta sync without existing sync state',
);
});
it('saves updated sync state after delta sync', async () => {
mockSystem.Run.mockResolvedValueOnce(''); // git fetch
mockSystem.Run.mockResolvedValueOnce('file1.txt\n'); // git diff
mockSystem.Run.mockResolvedValueOnce(''); // git checkout
await IncrementalSyncService.syncGitDelta(workspacePath, targetReference);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.lastSyncCommit).toBe(targetReference);
});
});
describe('applyDirectInput', () => {
it('extracts a local archive to workspace', async () => {
const archivePath = '/tmp/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
if (p === archivePath) return true;
// State file path does not exist
return false;
});
mockSystem.Run.mockResolvedValueOnce(''); // tar extract
const result = await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
expect(result).toEqual([archivePath]);
expect(mockSystem.Run).toHaveBeenCalledWith(expect.stringContaining('tar -xf "/tmp/overlay.tar"'), true);
});
it('fetches archive from storage URI via rclone then extracts', async () => {
const storageUri = 'storage://s3remote:builds/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
const pathString = p.toString();
if (pathString.includes('.game-ci-input-overlay.tar')) return true;
return false;
});
mockSystem.Run.mockResolvedValue(''); // rclone copy + tar extract
const result = await IncrementalSyncService.applyDirectInput(workspacePath, storageUri);
expect(result.length).toBe(1);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "s3remote:builds/overlay.tar"'),
true,
);
});
it('throws when local archive does not exist', async () => {
mockFs.existsSync.mockReturnValue(false);
await expect(IncrementalSyncService.applyDirectInput(workspacePath, '/missing/archive.tar')).rejects.toThrow(
'Input archive not found',
);
});
it('tracks overlay in sync state', async () => {
const archivePath = '/tmp/overlay.tar';
mockFs.existsSync.mockImplementation((p: fs.PathLike) => {
if (p === archivePath) return true;
return false;
});
mockSystem.Run.mockResolvedValueOnce('');
await IncrementalSyncService.applyDirectInput(workspacePath, archivePath);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toContain(archivePath);
});
});
describe('syncStoragePull', () => {
const storageUri = 'storage://s3:game-builds/latest';
beforeEach(() => {
mockFs.existsSync.mockReturnValue(false); // no existing state
});
it('pulls files from rclone remote into workspace', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version check
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(' 1234 file1.txt\n 5678 dir/file2.cs\n'); // rclone ls
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(result).toEqual(['file1.txt', 'dir/file2.cs']);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "s3:game-builds/latest" "/workspace/project"'),
true,
);
});
it('uses custom rclone remote when provided', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
rcloneRemote: 'custom-remote',
});
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('rclone copy "custom-remote:game-builds/latest"'),
true,
);
});
it('resets workspace in clean mode before pull', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // git checkout -- .
mockSystem.Run.mockResolvedValueOnce(''); // git clean -fd
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(''); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, { cleanMode: true });
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" clean -fd'),
true,
);
});
it('throws on invalid storage URI', async () => {
await expect(IncrementalSyncService.syncStoragePull(workspacePath, 'http://example.com')).rejects.toThrow(
'Invalid storage URI',
);
});
it('throws when rclone binary is not available', async () => {
mockSystem.Run.mockRejectedValueOnce(new Error('command not found: rclone'));
await expect(IncrementalSyncService.syncStoragePull(workspacePath, storageUri)).rejects.toThrow(
'rclone binary not found',
);
});
it('saves sync state with overlay tracking', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockResolvedValueOnce(' 100 a.txt\n'); // rclone ls
await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toContain(storageUri);
});
it('handles rclone ls failure gracefully', async () => {
mockSystem.Run.mockResolvedValueOnce('rclone v1.60.0'); // version
mockSystem.Run.mockResolvedValueOnce(''); // rclone copy
mockSystem.Run.mockRejectedValueOnce(new Error('ls failed')); // rclone ls fails
const result = await IncrementalSyncService.syncStoragePull(workspacePath, storageUri);
expect(result).toEqual([]);
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Could not list pulled files'));
});
});
describe('revertOverlays', () => {
it('reverts git state and cleans untracked files', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: ['/tmp/overlay.tar', 'storage://s3:builds/content'],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
mockSystem.Run.mockResolvedValue('');
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" checkout -- .'),
true,
);
expect(mockSystem.Run).toHaveBeenCalledWith(
expect.stringContaining('git -C "/workspace/project" clean -fd'),
true,
);
});
it('clears pending overlays in saved state', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: ['/tmp/overlay.tar'],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
mockSystem.Run.mockResolvedValue('');
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockFs.writeFileSync).toHaveBeenCalled();
const writeCall = mockFs.writeFileSync.mock.calls[0];
const savedState = JSON.parse(writeCall[1] as string) as SyncState;
expect(savedState.pendingOverlays).toEqual([]);
});
it('does nothing when no overlays are pending', async () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: new Date().toISOString(),
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).not.toHaveBeenCalled();
});
it('does nothing when no sync state exists', async () => {
mockFs.existsSync.mockReturnValue(false);
await IncrementalSyncService.revertOverlays(workspacePath);
expect(mockSystem.Run).not.toHaveBeenCalled();
});
});
});
describe('SyncStateManager', () => {
const workspacePath = '/workspace/project';
describe('loadState', () => {
it('returns parsed state from default path', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
const result = SyncStateManager.loadState(workspacePath);
expect(result).toEqual(state);
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, '.game-ci/sync-state.json'), 'utf8');
});
it('uses custom state path when provided', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue(JSON.stringify(state));
SyncStateManager.loadState(workspacePath, 'custom/state.json');
expect(mockFs.readFileSync).toHaveBeenCalledWith(path.join(workspacePath, 'custom/state.json'), 'utf8');
});
it('returns undefined when state file does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
const result = SyncStateManager.loadState(workspacePath);
expect(result).toBeUndefined();
});
it('returns undefined and logs warning on malformed JSON', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockReturnValue('not-valid-json{{{');
const result = SyncStateManager.loadState(workspacePath);
expect(result).toBeUndefined();
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to load sync state'));
});
});
describe('saveState', () => {
it('writes state to default path with pretty JSON', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: ['overlay1'],
};
mockFs.existsSync.mockReturnValue(true);
SyncStateManager.saveState(workspacePath, state);
expect(mockFs.writeFileSync).toHaveBeenCalledWith(
path.join(workspacePath, '.game-ci/sync-state.json'),
JSON.stringify(state, undefined, 2),
'utf8',
);
});
it('creates parent directories if they do not exist', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(false);
SyncStateManager.saveState(workspacePath, state);
expect(mockFs.mkdirSync).toHaveBeenCalledWith(expect.stringContaining('.game-ci'), { recursive: true });
});
it('logs warning on write failure instead of throwing', () => {
const state: SyncState = {
lastSyncCommit: 'abc123',
lastSyncTimestamp: '2026-01-01T00:00:00.000Z',
pendingOverlays: [],
};
mockFs.existsSync.mockReturnValue(true);
mockFs.writeFileSync.mockImplementation(() => {
throw new Error('Permission denied');
});
// Should not throw
SyncStateManager.saveState(workspacePath, state);
expect(mockLogger.logWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to save sync state'));
});
});
describe('calculateWorkspaceHash', () => {
it('produces a deterministic SHA-256 hash from workspace files', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockImplementation((filePath: fs.PathOrFileDescriptor) => {
const p = filePath.toString();
if (p.includes('ProjectVersion.txt')) return 'm_EditorVersion: 2022.3.10f1';
if (p.includes('manifest.json')) return '{"dependencies":{}}';
if (p.includes('packages-lock.json')) return '{"dependencies":{}}';
if (p.includes('csc.rsp')) return '-nullable+';
return '';
});
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash1).toBe(hash2);
expect(hash1).toHaveLength(64); // SHA-256 hex digest
});
it('produces different hashes for different workspace content', () => {
let callCount = 0;
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockImplementation(() => {
callCount++;
return callCount <= 4 ? 'content-v1' : 'content-v2';
});
const hash1 = SyncStateManager.calculateWorkspaceHash(workspacePath);
const hash2 = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash1).not.toBe(hash2);
});
it('includes missing file markers in hash for absent files', () => {
mockFs.existsSync.mockReturnValue(false);
const hash = SyncStateManager.calculateWorkspaceHash(workspacePath);
expect(hash).toHaveLength(64);
});
});
describe('hasDrifted', () => {
it('returns false when workspace hash matches', () => {
mockFs.existsSync.mockReturnValue(false);
const savedHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
const result = SyncStateManager.hasDrifted(workspacePath, savedHash);
expect(result).toBe(false);
});
it('returns true when workspace hash differs', () => {
mockFs.existsSync.mockReturnValue(false);
const result = SyncStateManager.hasDrifted(workspacePath, 'some-old-hash-that-will-not-match');
expect(result).toBe(true);
});
});
});

View File

@@ -1,3 +0,0 @@
export { SyncState, SyncStrategy } from './sync-state';
export { IncrementalSyncService } from './incremental-sync-service';
export { SyncStateManager } from './sync-state-manager';

View File

@@ -1,125 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import OrchestratorLogger from '../core/orchestrator-logger';
import { SyncState } from './sync-state';
/**
* Manages persistent sync state for incremental workspace updates.
*
* The sync state tracks what has been synced to a workspace, enabling
* delta-based updates instead of full clones. State is stored as a JSON
* file in the workspace (default: .game-ci/sync-state.json).
*/
export class SyncStateManager {
static readonly DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
/**
* Key workspace files whose content is hashed for drift detection.
* Changes to any of these files indicate the workspace may have been
* modified outside of the sync system.
*/
private static readonly WORKSPACE_HASH_FILES = [
'ProjectSettings/ProjectVersion.txt',
'Packages/manifest.json',
'Packages/packages-lock.json',
'Assets/csc.rsp',
];
/**
* Load sync state from the workspace.
*
* @param workspacePath - Root path of the workspace
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
* @returns The loaded sync state, or undefined if no state exists or parsing fails
*/
static loadState(workspacePath: string, statePath?: string): SyncState | undefined {
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
if (!fs.existsSync(resolvedPath)) {
return;
}
try {
const content = fs.readFileSync(resolvedPath, 'utf8');
return JSON.parse(content) as SyncState;
} catch {
OrchestratorLogger.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
return;
}
}
/**
* Save sync state to the workspace.
*
* Creates parent directories if they do not exist.
*
* @param workspacePath - Root path of the workspace
* @param state - The sync state to persist
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
*/
static saveState(workspacePath: string, state: SyncState, statePath?: string): void {
const resolvedPath = path.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
try {
const directory = path.dirname(resolvedPath);
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, { recursive: true });
}
fs.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
OrchestratorLogger.log(
`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`,
);
} catch (error: any) {
OrchestratorLogger.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
}
}
/**
* Calculate a SHA-256 hash of key workspace files for drift detection.
*
* Hashes the content of known workspace files (ProjectVersion.txt,
* manifest.json, etc.) to produce a fingerprint. If the hash changes
* between syncs, the workspace may have been modified externally.
*
* Files that do not exist are skipped (their absence is part of the hash).
*
* @param workspacePath - Root path of the workspace
* @returns Hex-encoded SHA-256 hash string
*/
static calculateWorkspaceHash(workspacePath: string): string {
const hash = crypto.createHash('sha256');
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
const filePath = path.join(workspacePath, relativePath);
try {
if (fs.existsSync(filePath)) {
const content = fs.readFileSync(filePath, 'utf8');
hash.update(`${relativePath}:${content}`);
} else {
hash.update(`${relativePath}:__missing__`);
}
} catch {
hash.update(`${relativePath}:__error__`);
}
}
return hash.digest('hex');
}
/**
* Check if the workspace has drifted from a previously saved hash.
*
* @param workspacePath - Root path of the workspace
* @param savedHash - The previously saved workspace hash to compare against
* @returns true if the current workspace hash differs from the saved hash
*/
static hasDrifted(workspacePath: string, savedHash: string): boolean {
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
return currentHash !== savedHash;
}
}

View File

@@ -1,19 +0,0 @@
/**
* Persistent sync state for incremental workspace updates.
* Stored on the runner to track what has already been synced.
*/
export interface SyncState {
/** Last successfully synced git commit SHA */
lastSyncCommit: string;
/** ISO 8601 timestamp of last sync */
lastSyncTimestamp: string;
/** SHA-256 hash of workspace state (optional) */
workspaceHash?: string;
/** List of overlay paths that haven't been reverted */
pendingOverlays: string[];
}
export type SyncStrategy = 'full' | 'git-delta' | 'direct-input' | 'storage-pull';