mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-16 13:06:53 -07:00
feat(sync): complete incremental sync protocol with storage-pull, state management, and tests (#799)
- Add storage-pull strategy: rclone-based sync from remote storage with overlay and clean modes, URI parsing (storage://remote:bucket/path), transfer parallelism, and automatic rclone availability checking - Add SyncStateManager: persistent state load/save with configurable paths, workspace hash calculation via SHA-256 of key project files, and drift detection for external modification awareness - Add action.yml inputs: syncStrategy, syncInputRef, syncStorageRemote, syncRevertAfter, syncStatePath with sensible defaults - Wire sync into Input (5 getters), BuildParameters (5 fields), index.ts (local build path), and RemoteClient (orchestrator path) with post-job overlay revert when syncRevertAfter is true - Add 42 unit tests covering all strategies, URI parsing, state management, hash calculation, drift detection, error handling, and edge cases (missing rclone, invalid URIs, absent state, empty diffs) - Add root:true to eslintrc to prevent plugin resolution conflicts Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
frostebite
+561
-1
@@ -38,6 +38,7 @@ const model_1 = __nccwpck_require__(41359);
|
||||
const cli_1 = __nccwpck_require__(55651);
|
||||
const mac_builder_1 = __importDefault(__nccwpck_require__(39364));
|
||||
const platform_setup_1 = __importDefault(__nccwpck_require__(64423));
|
||||
const sync_1 = __nccwpck_require__(98729);
|
||||
async function runMain() {
|
||||
try {
|
||||
if (cli_1.Cli.InitCliMode()) {
|
||||
@@ -52,6 +53,12 @@ async function runMain() {
|
||||
let exitCode = -1;
|
||||
if (buildParameters.providerStrategy === 'local') {
|
||||
core.info('Building locally');
|
||||
// Apply incremental sync strategy before build
|
||||
const syncStrategy = buildParameters.syncStrategy;
|
||||
if (syncStrategy !== 'full') {
|
||||
core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
|
||||
await applySyncStrategy(buildParameters, workspace);
|
||||
}
|
||||
await platform_setup_1.default.setup(buildParameters, actionFolder);
|
||||
exitCode =
|
||||
process.platform === 'darwin'
|
||||
@@ -61,6 +68,16 @@ async function runMain() {
|
||||
actionFolder,
|
||||
...buildParameters,
|
||||
});
|
||||
// Revert overlays after job completion if configured
|
||||
if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||
core.info('[Sync] Reverting overlay changes after job completion');
|
||||
try {
|
||||
await sync_1.IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
|
||||
}
|
||||
catch (revertError) {
|
||||
core.warning(`[Sync] Overlay revert failed: ${revertError.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
await model_1.Orchestrator.run(buildParameters, baseImage.toString());
|
||||
@@ -78,6 +95,47 @@ async function runMain() {
|
||||
core.setFailed(error.message);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Apply the configured sync strategy to the workspace before build.
|
||||
*/
|
||||
async function applySyncStrategy(buildParameters, workspace) {
|
||||
const strategy = buildParameters.syncStrategy;
|
||||
const resolvedStrategy = sync_1.IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
|
||||
if (resolvedStrategy === 'full') {
|
||||
core.info('[Sync] Resolved to full sync (no incremental state available)');
|
||||
return;
|
||||
}
|
||||
switch (resolvedStrategy) {
|
||||
case 'git-delta': {
|
||||
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||
const changedFiles = await sync_1.IncrementalSyncService.syncGitDelta(workspace, targetReference, buildParameters.syncStatePath);
|
||||
core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
|
||||
break;
|
||||
}
|
||||
case 'direct-input': {
|
||||
if (!buildParameters.syncInputRef) {
|
||||
throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
|
||||
}
|
||||
const overlays = await sync_1.IncrementalSyncService.applyDirectInput(workspace, buildParameters.syncInputRef, buildParameters.syncStorageRemote || undefined, buildParameters.syncStatePath);
|
||||
core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
|
||||
break;
|
||||
}
|
||||
case 'storage-pull': {
|
||||
if (!buildParameters.syncInputRef) {
|
||||
throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
|
||||
}
|
||||
const pulledFiles = await sync_1.IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
|
||||
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||
statePath: buildParameters.syncStatePath,
|
||||
});
|
||||
core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
|
||||
}
|
||||
}
|
||||
runMain();
|
||||
|
||||
|
||||
@@ -375,6 +433,11 @@ class BuildParameters {
|
||||
cacheUnityInstallationOnMac: input_1.default.cacheUnityInstallationOnMac,
|
||||
unityHubVersionOnMac: input_1.default.unityHubVersionOnMac,
|
||||
dockerWorkspacePath: input_1.default.dockerWorkspacePath,
|
||||
syncStrategy: input_1.default.syncStrategy,
|
||||
syncInputRef: input_1.default.syncInputRef,
|
||||
syncStorageRemote: input_1.default.syncStorageRemote,
|
||||
syncRevertAfter: input_1.default.syncRevertAfter,
|
||||
syncStatePath: input_1.default.syncStatePath,
|
||||
};
|
||||
}
|
||||
static parseBuildFile(filename, platform, androidExportType) {
|
||||
@@ -1795,6 +1858,22 @@ class Input {
|
||||
static get dockerWorkspacePath() {
|
||||
return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
|
||||
}
|
||||
static get syncStrategy() {
|
||||
return Input.getInput('syncStrategy') ?? 'full';
|
||||
}
|
||||
static get syncInputRef() {
|
||||
return Input.getInput('syncInputRef') ?? '';
|
||||
}
|
||||
static get syncStorageRemote() {
|
||||
return Input.getInput('syncStorageRemote') ?? '';
|
||||
}
|
||||
static get syncRevertAfter() {
|
||||
const input = Input.getInput('syncRevertAfter') ?? 'true';
|
||||
return input === 'true';
|
||||
}
|
||||
static get syncStatePath() {
|
||||
return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
|
||||
}
|
||||
static get dockerCpuLimit() {
|
||||
return Input.getInput('dockerCpuLimit') ?? node_os_1.default.cpus().length.toString();
|
||||
}
|
||||
@@ -7760,11 +7839,17 @@ const build_parameters_1 = __importDefault(__nccwpck_require__(80787));
|
||||
const cli_1 = __nccwpck_require__(55651);
|
||||
const orchestrator_options_1 = __importDefault(__nccwpck_require__(82473));
|
||||
const resource_tracking_1 = __importDefault(__nccwpck_require__(42604));
|
||||
const sync_1 = __nccwpck_require__(98729);
|
||||
class RemoteClient {
|
||||
static async setupRemoteClient() {
|
||||
orchestrator_logger_1.default.log(`bootstrap game ci orchestrator...`);
|
||||
await resource_tracking_1.default.logDiskUsageSnapshot('remote-cli-pre-build (start)');
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
const syncStrategy = (orchestrator_1.default.buildParameters.syncStrategy || 'full');
|
||||
if (syncStrategy !== 'full') {
|
||||
orchestrator_logger_1.default.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
|
||||
await RemoteClient.handleIncrementalSync(syncStrategy);
|
||||
}
|
||||
else if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||
@@ -7877,6 +7962,17 @@ class RemoteClient {
|
||||
}
|
||||
}
|
||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||
// Revert sync overlays if configured
|
||||
const syncStrategy = (orchestrator_1.default.buildParameters.syncStrategy || 'full');
|
||||
if (orchestrator_1.default.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
|
||||
try {
|
||||
orchestrator_logger_1.default.log('[Sync] Reverting overlay changes after job completion');
|
||||
await sync_1.IncrementalSyncService.revertOverlays(orchestrator_folders_1.OrchestratorFolders.repoPathAbsolute, orchestrator_1.default.buildParameters.syncStatePath);
|
||||
}
|
||||
catch (revertError) {
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
|
||||
}
|
||||
}
|
||||
// WIP - need to give the pod permissions to create config map
|
||||
await remote_client_logger_1.RemoteClientLogger.handleLogManagementPostJob();
|
||||
}
|
||||
@@ -7945,6 +8041,67 @@ class RemoteClient {
|
||||
remote_client_logger_1.RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Handle incremental sync strategies (git-delta, direct-input, storage-pull).
|
||||
*
|
||||
* For git-delta: requires an existing workspace with sync state; fetches and applies
|
||||
* only changed files.
|
||||
*
|
||||
* For direct-input and storage-pull: requires an existing workspace; applies overlay
|
||||
* content on top.
|
||||
*
|
||||
* Falls back to full bootstrapRepository() if incremental sync cannot proceed.
|
||||
*/
|
||||
static async handleIncrementalSync(strategy) {
|
||||
const buildParameters = orchestrator_1.default.buildParameters;
|
||||
const workspacePath = orchestrator_folders_1.OrchestratorFolders.repoPathAbsolute;
|
||||
const statePath = buildParameters.syncStatePath;
|
||||
// Resolve strategy — may fall back to 'full' if no state exists
|
||||
const resolvedStrategy = sync_1.IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
|
||||
if (resolvedStrategy === 'full') {
|
||||
orchestrator_logger_1.default.log('[Sync] Falling back to full bootstrap');
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
return;
|
||||
}
|
||||
switch (resolvedStrategy) {
|
||||
case 'git-delta': {
|
||||
const targetReference = buildParameters.gitSha || buildParameters.branch;
|
||||
orchestrator_logger_1.default.log(`[Sync] Git delta sync to ${targetReference}`);
|
||||
const changedFiles = await sync_1.IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
|
||||
orchestrator_logger_1.default.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
|
||||
break;
|
||||
}
|
||||
case 'direct-input': {
|
||||
const inputReference = buildParameters.syncInputRef;
|
||||
if (!inputReference) {
|
||||
throw new Error('[Sync] direct-input strategy requires syncInputRef');
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Applying direct input: ${inputReference}`);
|
||||
await sync_1.IncrementalSyncService.applyDirectInput(workspacePath, inputReference, buildParameters.syncStorageRemote || undefined, statePath);
|
||||
break;
|
||||
}
|
||||
case 'storage-pull': {
|
||||
const storageUri = buildParameters.syncInputRef;
|
||||
if (!storageUri) {
|
||||
throw new Error('[Sync] storage-pull strategy requires syncInputRef');
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Storage pull from: ${storageUri}`);
|
||||
await sync_1.IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
|
||||
rcloneRemote: buildParameters.syncStorageRemote || undefined,
|
||||
syncRevertAfter: buildParameters.syncRevertAfter,
|
||||
statePath,
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
orchestrator_logger_1.default.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
}
|
||||
}
|
||||
static async bootstrapRepository() {
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`mkdir -p ${orchestrator_folders_1.OrchestratorFolders.ToLinuxFolder(orchestrator_folders_1.OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`);
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`mkdir -p ${orchestrator_folders_1.OrchestratorFolders.ToLinuxFolder(orchestrator_folders_1.OrchestratorFolders.cacheFolderForCacheKeyFull)}`);
|
||||
@@ -9632,6 +9789,409 @@ class ContainerHookService {
|
||||
exports.ContainerHookService = ContainerHookService;
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
/***/ 96920:
|
||||
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
|
||||
|
||||
"use strict";
|
||||
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||
exports.IncrementalSyncService = void 0;
|
||||
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
|
||||
const node_path_1 = __importDefault(__nccwpck_require__(49411));
|
||||
const orchestrator_system_1 = __nccwpck_require__(9744);
|
||||
const orchestrator_logger_1 = __importDefault(__nccwpck_require__(32549));
|
||||
const sync_state_manager_1 = __nccwpck_require__(57595);
|
||||
/**
|
||||
* Service for incremental workspace synchronization.
|
||||
*
|
||||
* Supports multiple sync strategies:
|
||||
* - full: Traditional clone + cache restore (default)
|
||||
* - git-delta: Fetch and apply only changed files since last sync
|
||||
* - direct-input: Apply file changes passed as job input (no git push required)
|
||||
* - storage-pull: Fetch changed files from rclone-backed generic storage
|
||||
*/
|
||||
class IncrementalSyncService {
|
||||
/**
|
||||
* Load sync state from the workspace.
|
||||
*/
|
||||
static loadSyncState(workspacePath, statePath) {
|
||||
return sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
|
||||
}
|
||||
/**
|
||||
* Save sync state to the workspace.
|
||||
*/
|
||||
static saveSyncState(workspacePath, state, statePath) {
|
||||
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
}
|
||||
/**
|
||||
* Determine the appropriate sync strategy based on workspace state and configuration.
|
||||
*/
|
||||
static resolveStrategy(requestedStrategy, workspacePath, statePath) {
|
||||
if (requestedStrategy === 'full') {
|
||||
return 'full';
|
||||
}
|
||||
// git-delta requires an existing sync state
|
||||
if (requestedStrategy === 'git-delta') {
|
||||
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state) {
|
||||
orchestrator_logger_1.default.log('[Sync] No sync state found, falling back to full sync');
|
||||
return 'full';
|
||||
}
|
||||
return 'git-delta';
|
||||
}
|
||||
return requestedStrategy;
|
||||
}
|
||||
/**
|
||||
* Execute a git-delta sync: fetch latest and apply only changed files.
|
||||
*
|
||||
* @param workspacePath - Path to the git workspace
|
||||
* @param targetReference - Git ref to sync to (commit SHA, branch, tag)
|
||||
* @param statePath - Optional custom path for sync state file
|
||||
* @returns Number of files changed
|
||||
*/
|
||||
static async syncGitDelta(workspacePath, targetReference, statePath) {
|
||||
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state) {
|
||||
throw new Error('Cannot git-delta sync without existing sync state');
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Git delta: ${state.lastSyncCommit.slice(0, 8)} -> ${targetReference.slice(0, 8)}`);
|
||||
// Fetch latest
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" fetch origin`, true);
|
||||
// Get list of changed files
|
||||
const diffOutput = await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" diff --name-only ${state.lastSyncCommit}..${targetReference}`, true);
|
||||
const changedFiles = diffOutput.split('\n').filter(Boolean);
|
||||
orchestrator_logger_1.default.log(`[Sync] ${changedFiles.length} file(s) changed`);
|
||||
if (changedFiles.length > 0) {
|
||||
// Checkout target ref
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout ${targetReference}`, true);
|
||||
}
|
||||
// Update sync state
|
||||
const newState = {
|
||||
lastSyncCommit: targetReference,
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
workspaceHash: sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath),
|
||||
pendingOverlays: state.pendingOverlays,
|
||||
};
|
||||
sync_state_manager_1.SyncStateManager.saveState(workspacePath, newState, statePath);
|
||||
return changedFiles.length;
|
||||
}
|
||||
/**
|
||||
* Apply a direct input overlay from a local archive or storage URI.
|
||||
*
|
||||
* For storage URIs (storage://remote:bucket/path), the archive is fetched via rclone.
|
||||
* For local paths, the archive is extracted directly.
|
||||
*
|
||||
* @param workspacePath - Path to the workspace
|
||||
* @param inputReference - Local path or storage:// URI to the input archive
|
||||
* @param rcloneRemote - rclone remote name for storage:// URIs (optional, uses URI-embedded remote)
|
||||
* @param statePath - Optional custom path for sync state file
|
||||
* @returns List of overlay paths applied
|
||||
*/
|
||||
static async applyDirectInput(workspacePath, inputReference, rcloneRemote, statePath) {
|
||||
let localArchive = inputReference;
|
||||
// If storage URI, fetch via rclone first
|
||||
if (inputReference.startsWith('storage://')) {
|
||||
const parsed = IncrementalSyncService.parseStorageUri(inputReference);
|
||||
const remote = rcloneRemote || parsed.remote;
|
||||
const remotePath = parsed.path;
|
||||
localArchive = node_path_1.default.join(workspacePath, '.game-ci-input-overlay.tar');
|
||||
orchestrator_logger_1.default.log(`[Sync] Fetching input from storage: ${inputReference}`);
|
||||
await IncrementalSyncService.executeRcloneCopy(remote, remotePath, node_path_1.default.dirname(localArchive));
|
||||
}
|
||||
if (!node_fs_1.default.existsSync(localArchive)) {
|
||||
throw new Error(`Input archive not found: ${localArchive}`);
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Applying direct input overlay from ${localArchive}`);
|
||||
// Extract overlay
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`tar -xf "${localArchive}" -C "${workspacePath}"`, true);
|
||||
// Track overlay in sync state
|
||||
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath) || {
|
||||
lastSyncCommit: '',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
state.pendingOverlays.push(localArchive);
|
||||
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
return [localArchive];
|
||||
}
|
||||
/**
|
||||
* Execute a storage-pull sync: pull changed files from an rclone remote.
|
||||
*
|
||||
* This strategy fetches content from a remote storage backend (S3, GCS, Azure, etc.)
|
||||
* and overlays it onto the workspace. Supports two modes:
|
||||
* - overlay: extract on top of existing workspace (default)
|
||||
* - clean: fresh git checkout, then apply overlay
|
||||
*
|
||||
* @param workspacePath - Path to the workspace
|
||||
* @param storageUri - storage://remote:bucket/path URI pointing to remote content
|
||||
* @param options - Configuration for the storage-pull operation
|
||||
* @returns List of files pulled from storage
|
||||
*/
|
||||
static async syncStoragePull(workspacePath, storageUri, options = {}) {
|
||||
if (!storageUri.startsWith('storage://')) {
|
||||
throw new Error(`Invalid storage URI: ${storageUri}. Must start with storage://`);
|
||||
}
|
||||
// Verify rclone is available
|
||||
try {
|
||||
await orchestrator_system_1.OrchestratorSystem.Run('rclone version', true, true);
|
||||
}
|
||||
catch {
|
||||
throw new Error('rclone binary not found. Install rclone to use storage-pull sync strategy.');
|
||||
}
|
||||
const parsed = IncrementalSyncService.parseStorageUri(storageUri);
|
||||
const remote = options.rcloneRemote || parsed.remote;
|
||||
const remotePath = parsed.path;
|
||||
orchestrator_logger_1.default.log(`[Sync] Storage pull: ${remote}:${remotePath} -> ${workspacePath}`);
|
||||
// Clean mode: reset workspace to clean git state before applying overlay
|
||||
if (options.cleanMode) {
|
||||
orchestrator_logger_1.default.log('[Sync] Clean mode: resetting workspace to HEAD');
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||
}
|
||||
// Pull from remote storage directly into workspace
|
||||
const rcloneSource = `${remote}:${remotePath}`;
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`rclone copy "${rcloneSource}" "${workspacePath}" --transfers 8 --checkers 16`, true);
|
||||
// List what was pulled for tracking
|
||||
let pulledFiles = [];
|
||||
try {
|
||||
const lsOutput = await orchestrator_system_1.OrchestratorSystem.Run(`rclone ls "${rcloneSource}"`, true, true);
|
||||
pulledFiles = lsOutput
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
// rclone ls outputs: " <size> <path>"
|
||||
const trimmed = line.trim();
|
||||
const spaceIndex = trimmed.indexOf(' ');
|
||||
return spaceIndex >= 0 ? trimmed.slice(spaceIndex + 1).trim() : trimmed;
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
catch {
|
||||
orchestrator_logger_1.default.logWarning('[Sync] Could not list pulled files from remote');
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Pulled ${pulledFiles.length} file(s) from storage`);
|
||||
// Update sync state with overlay tracking
|
||||
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, options.statePath) || {
|
||||
lastSyncCommit: '',
|
||||
lastSyncTimestamp: new Date().toISOString(),
|
||||
pendingOverlays: [],
|
||||
};
|
||||
state.pendingOverlays.push(storageUri);
|
||||
state.lastSyncTimestamp = new Date().toISOString();
|
||||
state.workspaceHash = sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, options.statePath);
|
||||
return pulledFiles;
|
||||
}
|
||||
/**
|
||||
* Parse a storage:// URI into remote and path components.
|
||||
*
|
||||
* Supported formats:
|
||||
* - storage://remote:bucket/path (explicit remote with colon separator)
|
||||
* - storage://remote/path (remote name is first path segment)
|
||||
*
|
||||
* @param uri - The storage:// URI to parse
|
||||
* @returns Object with remote name and path
|
||||
*/
|
||||
static parseStorageUri(uri) {
|
||||
if (!uri.startsWith('storage://')) {
|
||||
throw new Error(`Invalid storage URI: ${uri}. Must start with storage://`);
|
||||
}
|
||||
const stripped = uri.replace('storage://', '');
|
||||
// Check for explicit remote:path format (e.g., "myremote:bucket/path")
|
||||
const colonIndex = stripped.indexOf(':');
|
||||
if (colonIndex > 0) {
|
||||
return {
|
||||
remote: stripped.slice(0, colonIndex),
|
||||
path: stripped.slice(colonIndex + 1),
|
||||
};
|
||||
}
|
||||
// Fallback: first segment is remote name (e.g., "myremote/bucket/path")
|
||||
const slashIndex = stripped.indexOf('/');
|
||||
if (slashIndex > 0) {
|
||||
return {
|
||||
remote: stripped.slice(0, slashIndex),
|
||||
path: stripped.slice(slashIndex + 1),
|
||||
};
|
||||
}
|
||||
// Just a remote name with no path
|
||||
return {
|
||||
remote: stripped,
|
||||
path: '',
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Execute rclone copy with standard flags.
|
||||
*/
|
||||
static async executeRcloneCopy(remote, remotePath, destinationPath) {
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`rclone copy "${remote}:${remotePath}" "${destinationPath}" --transfers 8 --checkers 16`, true);
|
||||
}
|
||||
/**
|
||||
* Revert pending overlays by restoring git state.
|
||||
*/
|
||||
static async revertOverlays(workspacePath, statePath) {
|
||||
const state = sync_state_manager_1.SyncStateManager.loadState(workspacePath, statePath);
|
||||
if (!state || state.pendingOverlays.length === 0) {
|
||||
return;
|
||||
}
|
||||
orchestrator_logger_1.default.log(`[Sync] Reverting ${state.pendingOverlays.length} overlay(s)`);
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" checkout -- .`, true);
|
||||
// Clean untracked files from overlays
|
||||
await orchestrator_system_1.OrchestratorSystem.Run(`git -C "${workspacePath}" clean -fd`, true);
|
||||
state.pendingOverlays = [];
|
||||
state.workspaceHash = sync_state_manager_1.SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
sync_state_manager_1.SyncStateManager.saveState(workspacePath, state, statePath);
|
||||
orchestrator_logger_1.default.log('[Sync] Overlays reverted');
|
||||
}
|
||||
}
|
||||
exports.IncrementalSyncService = IncrementalSyncService;
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
/***/ 98729:
|
||||
/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => {
|
||||
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||
exports.SyncStateManager = exports.IncrementalSyncService = void 0;
|
||||
var incremental_sync_service_1 = __nccwpck_require__(96920);
|
||||
Object.defineProperty(exports, "IncrementalSyncService", ({ enumerable: true, get: function () { return incremental_sync_service_1.IncrementalSyncService; } }));
|
||||
var sync_state_manager_1 = __nccwpck_require__(57595);
|
||||
Object.defineProperty(exports, "SyncStateManager", ({ enumerable: true, get: function () { return sync_state_manager_1.SyncStateManager; } }));
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
/***/ 57595:
|
||||
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
|
||||
|
||||
"use strict";
|
||||
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||
exports.SyncStateManager = void 0;
|
||||
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
|
||||
const node_path_1 = __importDefault(__nccwpck_require__(49411));
|
||||
const node_crypto_1 = __importDefault(__nccwpck_require__(6005));
|
||||
const orchestrator_logger_1 = __importDefault(__nccwpck_require__(32549));
|
||||
/**
|
||||
* Manages persistent sync state for incremental workspace updates.
|
||||
*
|
||||
* The sync state tracks what has been synced to a workspace, enabling
|
||||
* delta-based updates instead of full clones. State is stored as a JSON
|
||||
* file in the workspace (default: .game-ci/sync-state.json).
|
||||
*/
|
||||
class SyncStateManager {
|
||||
/**
|
||||
* Load sync state from the workspace.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||
* @returns The loaded sync state, or undefined if no state exists or parsing fails
|
||||
*/
|
||||
static loadState(workspacePath, statePath) {
|
||||
const resolvedPath = node_path_1.default.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||
if (!node_fs_1.default.existsSync(resolvedPath)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const content = node_fs_1.default.readFileSync(resolvedPath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
}
|
||||
catch {
|
||||
orchestrator_logger_1.default.logWarning(`[SyncState] Failed to load sync state from ${resolvedPath}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Save sync state to the workspace.
|
||||
*
|
||||
* Creates parent directories if they do not exist.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param state - The sync state to persist
|
||||
* @param statePath - Relative path to the state file (default: .game-ci/sync-state.json)
|
||||
*/
|
||||
static saveState(workspacePath, state, statePath) {
|
||||
const resolvedPath = node_path_1.default.join(workspacePath, statePath || SyncStateManager.DEFAULT_STATE_PATH);
|
||||
try {
|
||||
const directory = node_path_1.default.dirname(resolvedPath);
|
||||
if (!node_fs_1.default.existsSync(directory)) {
|
||||
node_fs_1.default.mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
node_fs_1.default.writeFileSync(resolvedPath, JSON.stringify(state, undefined, 2), 'utf8');
|
||||
orchestrator_logger_1.default.log(`[SyncState] State saved: commit=${state.lastSyncCommit}, overlays=${state.pendingOverlays.length}`);
|
||||
}
|
||||
catch (error) {
|
||||
orchestrator_logger_1.default.logWarning(`[SyncState] Failed to save sync state: ${error.message}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Calculate a SHA-256 hash of key workspace files for drift detection.
|
||||
*
|
||||
* Hashes the content of known workspace files (ProjectVersion.txt,
|
||||
* manifest.json, etc.) to produce a fingerprint. If the hash changes
|
||||
* between syncs, the workspace may have been modified externally.
|
||||
*
|
||||
* Files that do not exist are skipped (their absence is part of the hash).
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @returns Hex-encoded SHA-256 hash string
|
||||
*/
|
||||
static calculateWorkspaceHash(workspacePath) {
|
||||
const hash = node_crypto_1.default.createHash('sha256');
|
||||
for (const relativePath of SyncStateManager.WORKSPACE_HASH_FILES) {
|
||||
const filePath = node_path_1.default.join(workspacePath, relativePath);
|
||||
try {
|
||||
if (node_fs_1.default.existsSync(filePath)) {
|
||||
const content = node_fs_1.default.readFileSync(filePath, 'utf8');
|
||||
hash.update(`${relativePath}:${content}`);
|
||||
}
|
||||
else {
|
||||
hash.update(`${relativePath}:__missing__`);
|
||||
}
|
||||
}
|
||||
catch {
|
||||
hash.update(`${relativePath}:__error__`);
|
||||
}
|
||||
}
|
||||
return hash.digest('hex');
|
||||
}
|
||||
/**
|
||||
* Check if the workspace has drifted from a previously saved hash.
|
||||
*
|
||||
* @param workspacePath - Root path of the workspace
|
||||
* @param savedHash - The previously saved workspace hash to compare against
|
||||
* @returns true if the current workspace hash differs from the saved hash
|
||||
*/
|
||||
static hasDrifted(workspacePath, savedHash) {
|
||||
const currentHash = SyncStateManager.calculateWorkspaceHash(workspacePath);
|
||||
return currentHash !== savedHash;
|
||||
}
|
||||
}
|
||||
exports.SyncStateManager = SyncStateManager;
|
||||
SyncStateManager.DEFAULT_STATE_PATH = '.game-ci/sync-state.json';
|
||||
/**
|
||||
* Key workspace files whose content is hashed for drift detection.
|
||||
* Changes to any of these files indicate the workspace may have been
|
||||
* modified outside of the sync system.
|
||||
*/
|
||||
SyncStateManager.WORKSPACE_HASH_FILES = [
|
||||
'ProjectSettings/ProjectVersion.txt',
|
||||
'Packages/manifest.json',
|
||||
'Packages/packages-lock.json',
|
||||
'Assets/csc.rsp',
|
||||
];
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
/***/ 23451:
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user