feat(reliability): implement build reliability service with git integrity, reserved filename cleanup, and build archival

Adds BuildReliabilityService with the following capabilities:
- checkGitIntegrity(): runs git fsck --no-dangling and parses output for corruption
- cleanStaleLockFiles(): removes stale .lock files older than 10 minutes
- validateSubmoduleBackingStores(): validates .git files point to valid backing stores
- recoverCorruptedRepo(): orchestrates fsck, lock cleanup, re-fetch, retry fsck
- cleanReservedFilenames(): removes Windows reserved filenames (con, prn, aux, nul, com1-9, lpt1-9)
- archiveBuildOutput(): creates tar.gz archive of build output
- enforceRetention(): deletes archives older than retention period
- configureGitEnvironment(): sets GIT_TERMINAL_PROMPT=0, http.postBuffer, core.longpaths

Wired into action.yml as opt-in inputs, with pre-build integrity checks and
post-build archival in the main entry point.

Includes 29 unit tests covering success and failure cases for all methods.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
frostebite
2026-03-05 11:44:15 +00:00
parent 4f07508484
commit 47670cf3ce
7 changed files with 1177 additions and 180 deletions

View File

@@ -182,8 +182,8 @@ inputs:
required: false required: false
default: '' default: ''
description: description:
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the '[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
keys image, secrets (name, value object array), command line string)' the keys image, secrets (name, value object array), command line string)'
awsStackName: awsStackName:
default: 'game-ci' default: 'game-ci'
required: false required: false
@@ -280,29 +280,29 @@ inputs:
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or '[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
fixes.' fixes.'
gitIntegrityCheck: gitIntegrityCheck:
default: 'false' description: 'Run git integrity checks before build (fsck, lock cleanup, submodule validation)'
required: false required: false
description: '[Orchestrator] Enable pre-checkout git integrity verification for self-hosted runners' default: 'false'
gitAutoRecover: gitAutoRecover:
default: 'true' description: 'Attempt automatic recovery if git corruption is detected'
required: false required: false
description: '[Orchestrator] Automatically recover from detected git corruption (requires gitIntegrityCheck)' default: 'false'
cleanReservedFilenames: cleanReservedFilenames:
default: 'false' description: 'Remove Windows reserved filenames that cause Unity import loops'
required: false required: false
description: '[Orchestrator] Remove Windows reserved filenames (nul, con, prn, etc.) from Assets before build' default: 'false'
buildArchiveEnabled: buildArchiveEnabled:
description: 'Archive build output after successful build'
required: false
default: 'false' default: 'false'
required: false
description: '[Orchestrator] Enable post-build output archival with retention policy'
buildArchivePath: buildArchivePath:
default: '' description: 'Path to store build archives'
required: false required: false
description: '[Orchestrator] Path to archive location for build outputs' default: './build-archives'
buildArchiveRetention: buildArchiveRetention:
default: '3' description: 'Days to retain build archives before cleanup'
required: false required: false
description: '[Orchestrator] Number of builds to retain per platform in the archive' default: '30'
outputs: outputs:
volume: volume:

480
dist/index.js generated vendored
View File

@@ -38,6 +38,7 @@ const model_1 = __nccwpck_require__(41359);
const cli_1 = __nccwpck_require__(55651); const cli_1 = __nccwpck_require__(55651);
const mac_builder_1 = __importDefault(__nccwpck_require__(39364)); const mac_builder_1 = __importDefault(__nccwpck_require__(39364));
const platform_setup_1 = __importDefault(__nccwpck_require__(64423)); const platform_setup_1 = __importDefault(__nccwpck_require__(64423));
const reliability_1 = __nccwpck_require__(9842);
async function runMain() { async function runMain() {
try { try {
if (cli_1.Cli.InitCliMode()) { if (cli_1.Cli.InitCliMode()) {
@@ -46,9 +47,32 @@ async function runMain() {
} }
model_1.Action.checkCompatibility(); model_1.Action.checkCompatibility();
model_1.Cache.verify(); model_1.Cache.verify();
// Always configure git environment for CI reliability
reliability_1.BuildReliabilityService.configureGitEnvironment();
const { workspace, actionFolder } = model_1.Action; const { workspace, actionFolder } = model_1.Action;
const buildParameters = await model_1.BuildParameters.create(); const buildParameters = await model_1.BuildParameters.create();
const baseImage = new model_1.ImageTag(buildParameters); const baseImage = new model_1.ImageTag(buildParameters);
// Pre-build reliability checks
if (buildParameters.gitIntegrityCheck) {
core.info('Running git integrity checks...');
const isHealthy = reliability_1.BuildReliabilityService.checkGitIntegrity(workspace);
reliability_1.BuildReliabilityService.cleanStaleLockFiles(workspace);
reliability_1.BuildReliabilityService.validateSubmoduleBackingStores(workspace);
if (buildParameters.cleanReservedFilenames) {
reliability_1.BuildReliabilityService.cleanReservedFilenames(buildParameters.projectPath);
}
if (!isHealthy && buildParameters.gitAutoRecover) {
core.info('Git corruption detected, attempting automatic recovery...');
const recovered = reliability_1.BuildReliabilityService.recoverCorruptedRepo(workspace);
if (!recovered) {
core.warning('Automatic recovery failed. Build may encounter issues.');
}
}
}
else if (buildParameters.cleanReservedFilenames) {
// cleanReservedFilenames can run independently of gitIntegrityCheck
reliability_1.BuildReliabilityService.cleanReservedFilenames(buildParameters.projectPath);
}
let exitCode = -1; let exitCode = -1;
if (buildParameters.providerStrategy === 'local') { if (buildParameters.providerStrategy === 'local') {
core.info('Building locally'); core.info('Building locally');
@@ -66,6 +90,12 @@ async function runMain() {
await model_1.Orchestrator.run(buildParameters, baseImage.toString()); await model_1.Orchestrator.run(buildParameters, baseImage.toString());
exitCode = 0; exitCode = 0;
} }
// Post-build: archive and enforce retention
if (buildParameters.buildArchiveEnabled && exitCode === 0) {
core.info('Archiving build output...');
reliability_1.BuildReliabilityService.archiveBuildOutput(buildParameters.buildPath, buildParameters.buildArchivePath);
reliability_1.BuildReliabilityService.enforceRetention(buildParameters.buildArchivePath, buildParameters.buildArchiveRetention);
}
// Set output // Set output
await model_1.Output.setBuildVersion(buildParameters.buildVersion); await model_1.Output.setBuildVersion(buildParameters.buildVersion);
await model_1.Output.setAndroidVersionCode(buildParameters.androidVersionCode); await model_1.Output.setAndroidVersionCode(buildParameters.androidVersionCode);
@@ -1833,26 +1863,26 @@ class Input {
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false'; return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
} }
static get gitIntegrityCheck() { static get gitIntegrityCheck() {
const input = Input.getInput('gitIntegrityCheck') ?? false; const input = Input.getInput('gitIntegrityCheck') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get gitAutoRecover() { static get gitAutoRecover() {
const input = Input.getInput('gitAutoRecover') ?? 'true'; const input = Input.getInput('gitAutoRecover') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get cleanReservedFilenames() { static get cleanReservedFilenames() {
const input = Input.getInput('cleanReservedFilenames') ?? false; const input = Input.getInput('cleanReservedFilenames') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get buildArchiveEnabled() { static get buildArchiveEnabled() {
const input = Input.getInput('buildArchiveEnabled') ?? false; const input = Input.getInput('buildArchiveEnabled') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get buildArchivePath() { static get buildArchivePath() {
return Input.getInput('buildArchivePath') ?? ''; return Input.getInput('buildArchivePath') ?? './build-archives';
} }
static get buildArchiveRetention() { static get buildArchiveRetention() {
return Number.parseInt(Input.getInput('buildArchiveRetention') ?? '3', 10); return Number.parseInt(Input.getInput('buildArchiveRetention') ?? '30', 10);
} }
static ToEnvVarFormat(input) { static ToEnvVarFormat(input) {
if (input.toUpperCase() === input) { if (input.toUpperCase() === input) {
@@ -9660,6 +9690,444 @@ class ContainerHookService {
exports.ContainerHookService = ContainerHookService; exports.ContainerHookService = ContainerHookService;
/***/ }),
/***/ 37347:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.BuildReliabilityService = void 0;
const node_child_process_1 = __nccwpck_require__(17718);
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
const node_path_1 = __importDefault(__nccwpck_require__(49411));
const core = __importStar(__nccwpck_require__(42186));
/**
* Build reliability features for hardening CI pipelines.
* Provides git integrity checks, stale lock cleanup, submodule validation,
* reserved filename removal, build archival, and git environment configuration.
* All features are opt-in and fail gracefully (warnings only).
*/
class BuildReliabilityService {
/**
* Run git fsck to check repository integrity.
* Returns true if the repo is healthy, false if corruption detected.
*/
static checkGitIntegrity(repoPath = '.') {
core.info(`[Reliability] Checking git integrity in ${repoPath}`);
try {
const output = (0, node_child_process_1.execSync)(`git -C "${repoPath}" fsck --no-dangling`, {
encoding: 'utf8',
timeout: 120000,
stdio: ['pipe', 'pipe', 'pipe'],
});
// Parse output for corruption indicators
const corruptionPatterns = [
/broken link/i,
/missing (blob|tree|commit|tag)/i,
/dangling/i,
/corrupt/i,
/error in /i,
];
for (const pattern of corruptionPatterns) {
if (pattern.test(output)) {
core.warning(`[Reliability] Git integrity check found issues: ${output.trim()}`);
return false;
}
}
core.info('[Reliability] Git integrity check passed');
return true;
}
catch (error) {
// execSync throws on non-zero exit code
const stderr = error.stderr?.toString() ?? error.message;
core.warning(`[Reliability] Git integrity check failed: ${stderr}`);
return false;
}
}
/**
* Remove stale .lock files from the .git directory.
* Only removes lock files older than 10 minutes to avoid interfering with active operations.
* Returns the number of lock files removed.
*/
static cleanStaleLockFiles(repoPath = '.') {
const gitDir = node_path_1.default.join(repoPath, '.git');
if (!node_fs_1.default.existsSync(gitDir) || !node_fs_1.default.statSync(gitDir).isDirectory()) {
return 0;
}
core.info(`[Reliability] Scanning for stale lock files in ${gitDir}`);
const now = Date.now();
let removed = 0;
const cleanDirectory = (directory) => {
if (!node_fs_1.default.existsSync(directory))
return;
try {
const entries = node_fs_1.default.readdirSync(directory, { withFileTypes: true });
for (const entry of entries) {
const fullPath = node_path_1.default.join(directory, entry.name);
if (entry.isDirectory()) {
cleanDirectory(fullPath);
}
else if (entry.name.endsWith('.lock')) {
// Check if it is a known lock file location OR under refs/
const relativePath = node_path_1.default.relative(gitDir, fullPath);
const isKnownLock = BuildReliabilityService.LOCK_FILE_NAMES.has(entry.name);
const isRefsLock = relativePath.startsWith('refs' + node_path_1.default.sep);
if (isKnownLock || isRefsLock) {
try {
const stat = node_fs_1.default.statSync(fullPath);
const ageMs = now - stat.mtimeMs;
if (ageMs > BuildReliabilityService.LOCK_FILE_MAX_AGE_MS) {
node_fs_1.default.unlinkSync(fullPath);
removed++;
core.info(`[Reliability] Removed stale lock file (age: ${Math.round(ageMs / 1000)}s): ${relativePath}`);
}
else {
core.info(`[Reliability] Lock file is recent (age: ${Math.round(ageMs / 1000)}s), skipping: ${relativePath}`);
}
}
catch {
core.warning(`[Reliability] Could not remove lock file: ${fullPath}`);
}
}
}
}
}
catch {
// Directory not accessible
}
};
cleanDirectory(gitDir);
if (removed > 0) {
core.info(`[Reliability] Cleaned ${removed} stale lock file(s)`);
}
else {
core.info('[Reliability] No stale lock files found');
}
return removed;
}
/**
* Validate that submodule .git files point to existing backing stores
* under .git/modules/. Returns list of submodule paths with broken backing stores.
*/
static validateSubmoduleBackingStores(repoPath = '.') {
const broken = [];
const gitmodulesPath = node_path_1.default.join(repoPath, '.gitmodules');
if (!node_fs_1.default.existsSync(gitmodulesPath)) {
core.info('[Reliability] No .gitmodules found, skipping submodule validation');
return broken;
}
core.info(`[Reliability] Validating submodule backing stores in ${repoPath}`);
try {
const content = node_fs_1.default.readFileSync(gitmodulesPath, 'utf8');
const pathMatches = content.matchAll(/path\s*=\s*(.+)/g);
for (const match of pathMatches) {
const submodulePath = match[1].trim();
const gitFile = node_path_1.default.join(repoPath, submodulePath, '.git');
if (!node_fs_1.default.existsSync(gitFile)) {
// Submodule not initialized -- not necessarily broken
continue;
}
try {
const stat = node_fs_1.default.statSync(gitFile);
if (stat.isFile()) {
// .git is a file -- should contain "gitdir: <path>"
const gitFileContent = node_fs_1.default.readFileSync(gitFile, 'utf8').trim();
const gitdirMatch = gitFileContent.match(/^gitdir:\s*(.+)$/);
if (gitdirMatch) {
const backingStore = node_path_1.default.resolve(node_path_1.default.join(repoPath, submodulePath), gitdirMatch[1]);
if (!node_fs_1.default.existsSync(backingStore)) {
broken.push(submodulePath);
core.warning(`[Reliability] Submodule ${submodulePath} has broken backing store: ${backingStore}`);
}
else {
core.info(`[Reliability] Submodule ${submodulePath} backing store OK`);
}
}
else {
broken.push(submodulePath);
core.warning(`[Reliability] Submodule ${submodulePath} .git file has invalid format`);
}
}
}
catch {
// Can't read .git file
core.warning(`[Reliability] Could not read .git file for submodule: ${submodulePath}`);
}
}
}
catch (error) {
core.warning(`[Reliability] Could not read .gitmodules: ${error.message}`);
}
if (broken.length > 0) {
core.warning(`[Reliability] ${broken.length} submodule(s) have broken backing stores`);
}
else {
core.info('[Reliability] All submodule backing stores are valid');
}
return broken;
}
/**
* Orchestrate recovery of a corrupted repository.
* Sequence: fsck -> clean locks -> re-fetch -> retry fsck.
* Returns true if recovery succeeded.
*/
static recoverCorruptedRepo(repoPath = '.') {
core.warning(`[Reliability] Attempting automatic recovery for ${repoPath}`);
// Step 1: Clean stale lock files that may be preventing operations
const locksRemoved = BuildReliabilityService.cleanStaleLockFiles(repoPath);
if (locksRemoved > 0) {
core.info(`[Reliability] Recovery: cleaned ${locksRemoved} lock file(s)`);
}
// Step 2: Re-fetch to restore missing objects
try {
core.info('[Reliability] Recovery: re-fetching from remote');
(0, node_child_process_1.execSync)(`git -C "${repoPath}" fetch --all`, {
encoding: 'utf8',
timeout: 300000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Recovery: fetch completed');
}
catch (error) {
core.warning(`[Reliability] Recovery: fetch failed: ${error.stderr?.toString() ?? error.message}`);
}
// Step 3: Retry fsck
const healthy = BuildReliabilityService.checkGitIntegrity(repoPath);
if (healthy) {
core.info('[Reliability] Recovery succeeded -- repository is healthy');
}
else {
core.warning('[Reliability] Recovery failed -- repository still has integrity issues');
}
return healthy;
}
/**
* Scan a directory tree for files/directories with Windows reserved names.
* These names (con, prn, aux, nul, com1-9, lpt1-9) with any extension
* cause Unity asset importer infinite loops on Windows.
* Returns list of paths that were removed.
*/
static cleanReservedFilenames(projectPath) {
const assetsPath = node_path_1.default.join(projectPath, 'Assets');
if (!node_fs_1.default.existsSync(assetsPath)) {
core.info(`[Reliability] No Assets directory found at ${assetsPath}, skipping reserved filename scan`);
return [];
}
core.info(`[Reliability] Scanning for reserved filenames in ${assetsPath}`);
const cleaned = [];
const scanDirectory = (directory) => {
try {
const entries = node_fs_1.default.readdirSync(directory, { withFileTypes: true });
for (const entry of entries) {
const nameWithoutExtension = entry.name.split('.')[0].toLowerCase();
const fullPath = node_path_1.default.join(directory, entry.name);
if (BuildReliabilityService.RESERVED_NAMES.has(nameWithoutExtension)) {
try {
if (entry.isDirectory()) {
node_fs_1.default.rmSync(fullPath, { recursive: true, force: true });
}
else {
node_fs_1.default.unlinkSync(fullPath);
}
cleaned.push(fullPath);
core.warning(`[Reliability] Removed reserved filename: ${fullPath}`);
}
catch {
core.warning(`[Reliability] Could not remove reserved filename: ${fullPath}`);
}
}
else if (entry.isDirectory()) {
scanDirectory(fullPath);
}
}
}
catch {
// Directory not accessible
}
};
scanDirectory(assetsPath);
if (cleaned.length > 0) {
core.warning(`[Reliability] Cleaned ${cleaned.length} reserved filename(s)`);
}
else {
core.info('[Reliability] No reserved filenames found');
}
return cleaned;
}
/**
* Create a tar.gz archive of build output.
*/
static archiveBuildOutput(sourcePath, archivePath) {
if (!node_fs_1.default.existsSync(sourcePath)) {
core.info(`[Reliability] No build output to archive at ${sourcePath}`);
return;
}
node_fs_1.default.mkdirSync(archivePath, { recursive: true });
const timestamp = new Date().toISOString().replace(/[.:]/g, '-');
const archiveFile = node_path_1.default.join(archivePath, `build-${timestamp}.tar.gz`);
try {
(0, node_child_process_1.execSync)(`tar -czf "${archiveFile}" -C "${node_path_1.default.dirname(sourcePath)}" "${node_path_1.default.basename(sourcePath)}"`, {
encoding: 'utf8',
timeout: 600000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info(`[Reliability] Build output archived to ${archiveFile}`);
}
catch (error) {
core.warning(`[Reliability] Failed to archive build output: ${error.stderr?.toString() ?? error.message}`);
}
}
/**
* Enforce retention policy -- delete archives older than the retention period.
* Returns the number of old archives removed.
*/
static enforceRetention(archivePath, retentionDays) {
if (!node_fs_1.default.existsSync(archivePath)) {
return 0;
}
const now = Date.now();
const retentionMs = retentionDays * 24 * 60 * 60 * 1000;
let removed = 0;
try {
const entries = node_fs_1.default.readdirSync(archivePath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = node_path_1.default.join(archivePath, entry.name);
try {
const stat = node_fs_1.default.statSync(fullPath);
const ageMs = now - stat.mtimeMs;
if (ageMs > retentionMs) {
if (entry.isDirectory()) {
node_fs_1.default.rmSync(fullPath, { recursive: true, force: true });
}
else {
node_fs_1.default.unlinkSync(fullPath);
}
removed++;
core.info(`[Reliability] Removed old archive: ${entry.name} (age: ${Math.round(ageMs / (24 * 60 * 60 * 1000))} days)`);
}
}
catch {
core.warning(`[Reliability] Could not process archive entry: ${fullPath}`);
}
}
}
catch {
core.warning(`[Reliability] Could not read archive directory: ${archivePath}`);
return 0;
}
if (removed > 0) {
core.info(`[Reliability] Retention enforced: removed ${removed} old archive(s), retention: ${retentionDays} days`);
}
return removed;
}
/**
* Configure git environment variables for CI reliability.
* Sets GIT_TERMINAL_PROMPT=0, increases http.postBuffer, enables core.longpaths.
*/
static configureGitEnvironment() {
core.info('[Reliability] Configuring git environment for CI');
// Prevent git from prompting for credentials (hangs in CI)
process.env.GIT_TERMINAL_PROMPT = '0';
core.info('[Reliability] Set GIT_TERMINAL_PROMPT=0');
try {
// Increase http.postBuffer to 500MB for large pushes
(0, node_child_process_1.execSync)('git config --global http.postBuffer 524288000', {
encoding: 'utf8',
timeout: 10000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Set http.postBuffer=524288000 (500MB)');
}
catch (error) {
core.warning(`[Reliability] Could not set http.postBuffer: ${error.message}`);
}
try {
// Enable long paths on Windows
(0, node_child_process_1.execSync)('git config --global core.longpaths true', {
encoding: 'utf8',
timeout: 10000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Set core.longpaths=true');
}
catch (error) {
core.warning(`[Reliability] Could not set core.longpaths: ${error.message}`);
}
}
}
exports.BuildReliabilityService = BuildReliabilityService;
// Windows reserved device names that cause Unity asset importer infinite loops
BuildReliabilityService.RESERVED_NAMES = new Set([
'con',
'prn',
'aux',
'nul',
'com1',
'com2',
'com3',
'com4',
'com5',
'com6',
'com7',
'com8',
'com9',
'lpt1',
'lpt2',
'lpt3',
'lpt4',
'lpt5',
'lpt6',
'lpt7',
'lpt8',
'lpt9',
]);
// Lock files to look for in the .git directory
BuildReliabilityService.LOCK_FILE_NAMES = new Set(['index.lock', 'shallow.lock', 'config.lock', 'HEAD.lock']);
// Maximum age in milliseconds before a lock file is considered stale (10 minutes)
BuildReliabilityService.LOCK_FILE_MAX_AGE_MS = 10 * 60 * 1000;
/***/ }),
/***/ 9842:
/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => {
"use strict";
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.BuildReliabilityService = void 0;
var build_reliability_service_1 = __nccwpck_require__(37347);
Object.defineProperty(exports, "BuildReliabilityService", ({ enumerable: true, get: function () { return build_reliability_service_1.BuildReliabilityService; } }));
/***/ }), /***/ }),
/***/ 23451: /***/ 23451:

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@@ -3,6 +3,7 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
import { Cli } from './model/cli/cli'; import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder'; import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup'; import PlatformSetup from './model/platform-setup';
import { BuildReliabilityService } from './model/orchestrator/services/reliability';
async function runMain() { async function runMain() {
try { try {
@@ -14,11 +15,38 @@ async function runMain() {
Action.checkCompatibility(); Action.checkCompatibility();
Cache.verify(); Cache.verify();
// Always configure git environment for CI reliability
BuildReliabilityService.configureGitEnvironment();
const { workspace, actionFolder } = Action; const { workspace, actionFolder } = Action;
const buildParameters = await BuildParameters.create(); const buildParameters = await BuildParameters.create();
const baseImage = new ImageTag(buildParameters); const baseImage = new ImageTag(buildParameters);
// Pre-build reliability checks
if (buildParameters.gitIntegrityCheck) {
core.info('Running git integrity checks...');
const isHealthy = BuildReliabilityService.checkGitIntegrity(workspace);
BuildReliabilityService.cleanStaleLockFiles(workspace);
BuildReliabilityService.validateSubmoduleBackingStores(workspace);
if (buildParameters.cleanReservedFilenames) {
BuildReliabilityService.cleanReservedFilenames(buildParameters.projectPath);
}
if (!isHealthy && buildParameters.gitAutoRecover) {
core.info('Git corruption detected, attempting automatic recovery...');
const recovered = BuildReliabilityService.recoverCorruptedRepo(workspace);
if (!recovered) {
core.warning('Automatic recovery failed. Build may encounter issues.');
}
}
} else if (buildParameters.cleanReservedFilenames) {
// cleanReservedFilenames can run independently of gitIntegrityCheck
BuildReliabilityService.cleanReservedFilenames(buildParameters.projectPath);
}
let exitCode = -1; let exitCode = -1;
if (buildParameters.providerStrategy === 'local') { if (buildParameters.providerStrategy === 'local') {
@@ -37,6 +65,13 @@ async function runMain() {
exitCode = 0; exitCode = 0;
} }
// Post-build: archive and enforce retention
if (buildParameters.buildArchiveEnabled && exitCode === 0) {
core.info('Archiving build output...');
BuildReliabilityService.archiveBuildOutput(buildParameters.buildPath, buildParameters.buildArchivePath);
BuildReliabilityService.enforceRetention(buildParameters.buildArchivePath, buildParameters.buildArchiveRetention);
}
// Set output // Set output
await Output.setBuildVersion(buildParameters.buildVersion); await Output.setBuildVersion(buildParameters.buildVersion);
await Output.setAndroidVersionCode(buildParameters.androidVersionCode); await Output.setAndroidVersionCode(buildParameters.androidVersionCode);

View File

@@ -283,35 +283,35 @@ class Input {
} }
static get gitIntegrityCheck(): boolean { static get gitIntegrityCheck(): boolean {
const input = Input.getInput('gitIntegrityCheck') ?? false; const input = Input.getInput('gitIntegrityCheck') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get gitAutoRecover(): boolean { static get gitAutoRecover(): boolean {
const input = Input.getInput('gitAutoRecover') ?? 'true'; const input = Input.getInput('gitAutoRecover') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get cleanReservedFilenames(): boolean { static get cleanReservedFilenames(): boolean {
const input = Input.getInput('cleanReservedFilenames') ?? false; const input = Input.getInput('cleanReservedFilenames') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get buildArchiveEnabled(): boolean { static get buildArchiveEnabled(): boolean {
const input = Input.getInput('buildArchiveEnabled') ?? false; const input = Input.getInput('buildArchiveEnabled') ?? 'false';
return input === 'true'; return input === 'true';
} }
static get buildArchivePath(): string { static get buildArchivePath(): string {
return Input.getInput('buildArchivePath') ?? ''; return Input.getInput('buildArchivePath') ?? './build-archives';
} }
static get buildArchiveRetention(): number { static get buildArchiveRetention(): number {
return Number.parseInt(Input.getInput('buildArchiveRetention') ?? '3', 10); return Number.parseInt(Input.getInput('buildArchiveRetention') ?? '30', 10);
} }
public static ToEnvVarFormat(input: string) { public static ToEnvVarFormat(input: string) {

View File

@@ -1,11 +1,17 @@
import { execSync } from 'node:child_process';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path';
import { BuildReliabilityService } from './build-reliability-service'; import { BuildReliabilityService } from './build-reliability-service';
// Mock dependencies // Mock dependencies
jest.mock('node:child_process');
jest.mock('node:fs'); jest.mock('node:fs');
jest.mock('../core/orchestrator-logger'); jest.mock('@actions/core', () => ({
jest.mock('../core/orchestrator-system'); info: jest.fn(),
warning: jest.fn(),
}));
const mockExecSync = execSync as jest.MockedFunction<typeof execSync>;
const mockFs = fs as jest.Mocked<typeof fs>; const mockFs = fs as jest.Mocked<typeof fs>;
describe('BuildReliabilityService', () => { describe('BuildReliabilityService', () => {
@@ -13,42 +19,441 @@ describe('BuildReliabilityService', () => {
jest.clearAllMocks(); jest.clearAllMocks();
}); });
describe('cleanStaleLockFiles', () => { // =========================================================================
it('should return 0 when .git directory does not exist', async () => { // checkGitIntegrity
mockFs.existsSync.mockReturnValue(false); // =========================================================================
const result = await BuildReliabilityService.cleanStaleLockFiles('/repo');
expect(result).toBe(0); describe('checkGitIntegrity', () => {
it('should return true when fsck succeeds with clean output', () => {
mockExecSync.mockReturnValue('');
const result = BuildReliabilityService.checkGitIntegrity('/repo');
expect(result).toBe(true);
expect(mockExecSync).toHaveBeenCalledWith(
'git -C "/repo" fsck --no-dangling',
expect.objectContaining({ encoding: 'utf8' }),
);
});
it('should return false when fsck output contains corruption indicators', () => {
mockExecSync.mockReturnValue('broken link from tree abc123');
const result = BuildReliabilityService.checkGitIntegrity('/repo');
expect(result).toBe(false);
});
it('should return false when fsck output contains missing objects', () => {
mockExecSync.mockReturnValue('missing blob abc123');
const result = BuildReliabilityService.checkGitIntegrity('/repo');
expect(result).toBe(false);
});
it('should return false when execSync throws (non-zero exit code)', () => {
mockExecSync.mockImplementation(() => {
const error: any = new Error('fsck failed');
error.stderr = Buffer.from('error: bad object HEAD');
throw error;
});
const result = BuildReliabilityService.checkGitIntegrity('/repo');
expect(result).toBe(false);
});
it('should use current directory when no repoPath provided', () => {
mockExecSync.mockReturnValue('');
BuildReliabilityService.checkGitIntegrity();
expect(mockExecSync).toHaveBeenCalledWith('git -C "." fsck --no-dangling', expect.anything());
}); });
}); });
describe('cleanReservedFilenames', () => { // =========================================================================
it('should return empty array when Assets directory does not exist', async () => { // cleanStaleLockFiles
// =========================================================================
describe('cleanStaleLockFiles', () => {
it('should return 0 when .git directory does not exist', () => {
mockFs.existsSync.mockReturnValue(false); mockFs.existsSync.mockReturnValue(false);
const result = await BuildReliabilityService.cleanReservedFilenames('/project'); const result = BuildReliabilityService.cleanStaleLockFiles('/repo');
expect(result).toEqual([]); expect(result).toBe(0);
});
it('should remove lock files older than 10 minutes', () => {
const now = Date.now();
const oldTime = now - 15 * 60 * 1000; // 15 minutes ago
mockFs.existsSync.mockReturnValue(true);
mockFs.statSync.mockImplementation((filePath: any) => {
if (filePath === path.join('/repo', '.git')) {
return { isDirectory: () => true } as fs.Stats;
}
return { mtimeMs: oldTime } as fs.Stats;
});
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/repo', '.git')) {
return [
{ name: 'index.lock', isDirectory: () => false },
{ name: 'HEAD.lock', isDirectory: () => false },
] as any;
}
return [];
});
mockFs.unlinkSync.mockReturnValue(undefined);
const result = BuildReliabilityService.cleanStaleLockFiles('/repo');
expect(result).toBe(2);
expect(mockFs.unlinkSync).toHaveBeenCalledTimes(2);
});
it('should NOT remove lock files younger than 10 minutes', () => {
const now = Date.now();
const recentTime = now - 2 * 60 * 1000; // 2 minutes ago
mockFs.existsSync.mockReturnValue(true);
mockFs.statSync.mockImplementation((filePath: any) => {
if (filePath === path.join('/repo', '.git')) {
return { isDirectory: () => true } as fs.Stats;
}
return { mtimeMs: recentTime } as fs.Stats;
});
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/repo', '.git')) {
return [{ name: 'index.lock', isDirectory: () => false }] as any;
}
return [];
});
const result = BuildReliabilityService.cleanStaleLockFiles('/repo');
expect(result).toBe(0);
expect(mockFs.unlinkSync).not.toHaveBeenCalled();
});
it('should recursively scan refs directory for lock files', () => {
const now = Date.now();
const oldTime = now - 15 * 60 * 1000;
mockFs.existsSync.mockReturnValue(true);
mockFs.statSync.mockImplementation((filePath: any) => {
if (filePath === path.join('/repo', '.git')) {
return { isDirectory: () => true } as fs.Stats;
}
return { mtimeMs: oldTime } as fs.Stats;
});
mockFs.readdirSync.mockImplementation((dir: any) => {
const gitDir = path.join('/repo', '.git');
if (dir === gitDir) {
return [{ name: 'refs', isDirectory: () => true }] as any;
}
if (dir === path.join(gitDir, 'refs')) {
return [{ name: 'heads', isDirectory: () => true }] as any;
}
if (dir === path.join(gitDir, 'refs', 'heads')) {
return [{ name: 'main.lock', isDirectory: () => false }] as any;
}
return [];
});
mockFs.unlinkSync.mockReturnValue(undefined);
const result = BuildReliabilityService.cleanStaleLockFiles('/repo');
expect(result).toBe(1);
}); });
}); });
// =========================================================================
// validateSubmoduleBackingStores
// =========================================================================
describe('validateSubmoduleBackingStores', () => { describe('validateSubmoduleBackingStores', () => {
it('should return empty array when .gitmodules does not exist', async () => { it('should return empty array when .gitmodules does not exist', () => {
mockFs.existsSync.mockReturnValue(false); mockFs.existsSync.mockReturnValue(false);
const result = await BuildReliabilityService.validateSubmoduleBackingStores('/repo'); const result = BuildReliabilityService.validateSubmoduleBackingStores('/repo');
expect(result).toEqual([]);
});
it('should detect broken backing store for submodule', () => {
mockFs.existsSync.mockImplementation((p: any) => {
if (p === path.join('/repo', '.gitmodules')) return true;
if (p === path.join('/repo', 'lib/sub', '.git')) return true;
// Backing store does not exist
return false;
});
mockFs.readFileSync.mockImplementation((p: any) => {
if (p === path.join('/repo', '.gitmodules')) {
return '[submodule "sub"]\n\tpath = lib/sub\n\turl = https://example.com/sub.git';
}
if (p === path.join('/repo', 'lib/sub', '.git')) {
return 'gitdir: ../../.git/modules/lib/sub';
}
return '';
});
mockFs.statSync.mockReturnValue({ isFile: () => true } as fs.Stats);
const result = BuildReliabilityService.validateSubmoduleBackingStores('/repo');
expect(result).toContain('lib/sub');
});
it('should return empty array when all submodule backing stores are valid', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFileSync.mockImplementation((p: any) => {
if (p === path.join('/repo', '.gitmodules')) {
return '[submodule "sub"]\n\tpath = lib/sub\n\turl = https://example.com/sub.git';
}
if (p === path.join('/repo', 'lib/sub', '.git')) {
return 'gitdir: ../../.git/modules/lib/sub';
}
return '';
});
mockFs.statSync.mockReturnValue({ isFile: () => true } as fs.Stats);
const result = BuildReliabilityService.validateSubmoduleBackingStores('/repo');
expect(result).toEqual([]); expect(result).toEqual([]);
}); });
}); });
describe('enforceRetention', () => { // =========================================================================
it('should return 0 when archive path does not exist', async () => { // recoverCorruptedRepo
// =========================================================================
describe('recoverCorruptedRepo', () => {
it('should orchestrate fsck cleanup and re-fetch, returning true on success', () => {
// cleanStaleLockFiles: no .git dir
mockFs.existsSync.mockReturnValue(false); mockFs.existsSync.mockReturnValue(false);
const result = await BuildReliabilityService.enforceRetention('/archive', 3); mockFs.statSync.mockReturnValue({ isDirectory: () => false } as fs.Stats);
// fetch succeeds, then fsck succeeds
mockExecSync.mockReturnValue('');
const result = BuildReliabilityService.recoverCorruptedRepo('/repo');
expect(result).toBe(true);
// Should have called fetch
expect(mockExecSync).toHaveBeenCalledWith('git -C "/repo" fetch --all', expect.anything());
});
it('should return false when recovery fails to restore integrity', () => {
mockFs.existsSync.mockReturnValue(false);
mockFs.statSync.mockReturnValue({ isDirectory: () => false } as fs.Stats);
// fetch succeeds, but fsck fails
mockExecSync.mockImplementation((cmd: any) => {
if (typeof cmd === 'string' && cmd.includes('fetch')) return '';
if (typeof cmd === 'string' && cmd.includes('fsck')) {
return 'missing blob abc123';
}
return '';
});
const result = BuildReliabilityService.recoverCorruptedRepo('/repo');
expect(result).toBe(false);
});
it('should continue recovery even when fetch fails', () => {
mockFs.existsSync.mockReturnValue(false);
mockFs.statSync.mockReturnValue({ isDirectory: () => false } as fs.Stats);
let callCount = 0;
mockExecSync.mockImplementation((cmd: any) => {
callCount++;
if (typeof cmd === 'string' && cmd.includes('fetch')) {
throw new Error('network error');
}
// fsck call
return '';
});
const result = BuildReliabilityService.recoverCorruptedRepo('/repo');
// Should still attempt fsck after failed fetch
expect(callCount).toBeGreaterThanOrEqual(2);
});
});
// =========================================================================
// cleanReservedFilenames
// =========================================================================
describe('cleanReservedFilenames', () => {
it('should return empty array when Assets directory does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
const result = BuildReliabilityService.cleanReservedFilenames('/project');
expect(result).toEqual([]);
});
it('should remove files with reserved names (con, prn, aux, nul)', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/project', 'Assets')) {
return [
{ name: 'con.txt', isDirectory: () => false },
{ name: 'PRN.meta', isDirectory: () => false },
{ name: 'aux.shader', isDirectory: () => false },
{ name: 'nul.png', isDirectory: () => false },
{ name: 'valid-file.cs', isDirectory: () => false },
] as any;
}
return [];
});
mockFs.unlinkSync.mockReturnValue(undefined);
const result = BuildReliabilityService.cleanReservedFilenames('/project');
expect(result).toHaveLength(4);
expect(mockFs.unlinkSync).toHaveBeenCalledTimes(4);
});
it('should remove directories with reserved names', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/project', 'Assets')) {
return [{ name: 'com1', isDirectory: () => true }] as any;
}
return [];
});
mockFs.rmSync.mockReturnValue(undefined);
const result = BuildReliabilityService.cleanReservedFilenames('/project');
expect(result).toHaveLength(1);
expect(mockFs.rmSync).toHaveBeenCalledWith(path.join('/project', 'Assets', 'com1'), {
recursive: true,
force: true,
});
});
it('should detect COM1 through COM9 and LPT1 through LPT9', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/project', 'Assets')) {
return [
{ name: 'com1.txt', isDirectory: () => false },
{ name: 'COM9.meta', isDirectory: () => false },
{ name: 'lpt1.dat', isDirectory: () => false },
{ name: 'LPT9.log', isDirectory: () => false },
] as any;
}
return [];
});
mockFs.unlinkSync.mockReturnValue(undefined);
const result = BuildReliabilityService.cleanReservedFilenames('/project');
expect(result).toHaveLength(4);
});
it('should not remove files that merely contain reserved names as substrings', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockImplementation((dir: any) => {
if (dir === path.join('/project', 'Assets')) {
return [
{ name: 'controller.cs', isDirectory: () => false },
{ name: 'printer-utils.cs', isDirectory: () => false },
{ name: 'auxiliary.shader', isDirectory: () => false },
] as any;
}
return [];
});
const result = BuildReliabilityService.cleanReservedFilenames('/project');
expect(result).toHaveLength(0);
});
});
// =========================================================================
// archiveBuildOutput
// =========================================================================
describe('archiveBuildOutput', () => {
it('should skip archiving when source path does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
BuildReliabilityService.archiveBuildOutput('/builds/output', '/archives');
expect(mockExecSync).not.toHaveBeenCalled();
});
it('should create archive directory and tar.gz output', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.mkdirSync.mockReturnValue(undefined as any);
mockExecSync.mockReturnValue('');
BuildReliabilityService.archiveBuildOutput('/builds/output', '/archives');
expect(mockFs.mkdirSync).toHaveBeenCalledWith('/archives', { recursive: true });
expect(mockExecSync).toHaveBeenCalledWith(expect.stringContaining('tar -czf'), expect.anything());
});
});
// =========================================================================
// enforceRetention
// =========================================================================
describe('enforceRetention', () => {
it('should return 0 when archive path does not exist', () => {
mockFs.existsSync.mockReturnValue(false);
const result = BuildReliabilityService.enforceRetention('/archive', 30);
expect(result).toBe(0);
});
it('should remove archives older than retention period', () => {
const now = Date.now();
const oldTime = now - 45 * 24 * 60 * 60 * 1000; // 45 days ago
const recentTime = now - 5 * 24 * 60 * 60 * 1000; // 5 days ago
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockReturnValue([
{ name: 'build-old.tar.gz', isDirectory: () => false },
{ name: 'build-recent.tar.gz', isDirectory: () => false },
] as any);
mockFs.statSync.mockImplementation((p: any) => {
if ((p as string).includes('old')) {
return { mtimeMs: oldTime } as fs.Stats;
}
return { mtimeMs: recentTime } as fs.Stats;
});
mockFs.unlinkSync.mockReturnValue(undefined);
const result = BuildReliabilityService.enforceRetention('/archive', 30);
expect(result).toBe(1);
expect(mockFs.unlinkSync).toHaveBeenCalledTimes(1);
});
it('should keep all archives within retention period', () => {
const now = Date.now();
const recentTime = now - 5 * 24 * 60 * 60 * 1000;
mockFs.existsSync.mockReturnValue(true);
mockFs.readdirSync.mockReturnValue([
{ name: 'build-1.tar.gz', isDirectory: () => false },
{ name: 'build-2.tar.gz', isDirectory: () => false },
] as any);
mockFs.statSync.mockReturnValue({ mtimeMs: recentTime } as fs.Stats);
const result = BuildReliabilityService.enforceRetention('/archive', 30);
expect(result).toBe(0); expect(result).toBe(0);
}); });
}); });
// =========================================================================
// configureGitEnvironment
// =========================================================================
describe('configureGitEnvironment', () => { describe('configureGitEnvironment', () => {
it('should return GIT_CONFIG_NOSYSTEM=1', () => { it('should set GIT_TERMINAL_PROMPT=0 in process.env', () => {
const environment = BuildReliabilityService.configureGitEnvironment(); mockExecSync.mockReturnValue('');
expect(environment.GIT_CONFIG_NOSYSTEM).toBe('1'); BuildReliabilityService.configureGitEnvironment();
expect(process.env.GIT_TERMINAL_PROMPT).toBe('0');
});
it('should configure http.postBuffer via git config', () => {
mockExecSync.mockReturnValue('');
BuildReliabilityService.configureGitEnvironment();
expect(mockExecSync).toHaveBeenCalledWith('git config --global http.postBuffer 524288000', expect.anything());
});
it('should configure core.longpaths via git config', () => {
mockExecSync.mockReturnValue('');
BuildReliabilityService.configureGitEnvironment();
expect(mockExecSync).toHaveBeenCalledWith('git config --global core.longpaths true', expect.anything());
});
it('should warn but not throw when git config commands fail', () => {
const core = require('@actions/core');
mockExecSync.mockImplementation(() => {
throw new Error('git config failed');
});
// Should not throw
expect(() => BuildReliabilityService.configureGitEnvironment()).not.toThrow();
expect(core.warning).toHaveBeenCalled();
}); });
}); });
}); });

View File

@@ -1,10 +1,12 @@
import { execSync } from 'node:child_process';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import OrchestratorLogger from '../core/orchestrator-logger'; import * as core from '@actions/core';
import { OrchestratorSystem } from '../core/orchestrator-system';
/** /**
* Build reliability features for hardening CI pipelines. * Build reliability features for hardening CI pipelines.
* Provides git integrity checks, stale lock cleanup, submodule validation,
* reserved filename removal, build archival, and git environment configuration.
* All features are opt-in and fail gracefully (warnings only). * All features are opt-in and fail gracefully (warnings only).
*/ */
export class BuildReliabilityService { export class BuildReliabilityService {
@@ -34,45 +36,65 @@ export class BuildReliabilityService {
'lpt9', 'lpt9',
]); ]);
// Common git lock files left by crashed processes // Lock files to look for in the .git directory
private static readonly LOCK_FILE_PATTERNS = [ private static readonly LOCK_FILE_NAMES = new Set(['index.lock', 'shallow.lock', 'config.lock', 'HEAD.lock']);
'index.lock',
'shallow.lock', // Maximum age in milliseconds before a lock file is considered stale (10 minutes)
'config.lock', private static readonly LOCK_FILE_MAX_AGE_MS = 10 * 60 * 1000;
'HEAD.lock',
'refs/heads/*.lock',
'refs/remotes/**/*.lock',
];
/** /**
* Run git fsck to check repository integrity. * Run git fsck to check repository integrity.
* Returns true if the repo is healthy, false if corruption detected. * Returns true if the repo is healthy, false if corruption detected.
*/ */
static async checkGitIntegrity(repoPath: string): Promise<boolean> { static checkGitIntegrity(repoPath: string = '.'): boolean {
OrchestratorLogger.log(`[Reliability] Checking git integrity in ${repoPath}`); core.info(`[Reliability] Checking git integrity in ${repoPath}`);
try { try {
await OrchestratorSystem.Run(`git -C "${repoPath}" fsck --no-dangling --no-progress`, true); const output = execSync(`git -C "${repoPath}" fsck --no-dangling`, {
OrchestratorLogger.log(`[Reliability] Git integrity check passed`); encoding: 'utf8',
timeout: 120_000,
stdio: ['pipe', 'pipe', 'pipe'],
});
// Parse output for corruption indicators
const corruptionPatterns = [
/broken link/i,
/missing (blob|tree|commit|tag)/i,
/dangling/i,
/corrupt/i,
/error in /i,
];
for (const pattern of corruptionPatterns) {
if (pattern.test(output)) {
core.warning(`[Reliability] Git integrity check found issues: ${output.trim()}`);
return false;
}
}
core.info('[Reliability] Git integrity check passed');
return true; return true;
} catch (error: any) { } catch (error: any) {
OrchestratorLogger.logWarning(`[Reliability] Git integrity check failed: ${error.message}`); // execSync throws on non-zero exit code
const stderr = error.stderr?.toString() ?? error.message;
core.warning(`[Reliability] Git integrity check failed: ${stderr}`);
return false; return false;
} }
} }
/** /**
* Remove stale lock files from the .git directory. * Remove stale .lock files from the .git directory.
* Only removes lock files older than 10 minutes to avoid interfering with active operations.
* Returns the number of lock files removed. * Returns the number of lock files removed.
*/ */
static async cleanStaleLockFiles(repoPath: string): Promise<number> { static cleanStaleLockFiles(repoPath: string = '.'): number {
const gitDirectory = path.join(repoPath, '.git'); const gitDir = path.join(repoPath, '.git');
if (!fs.existsSync(gitDirectory)) { if (!fs.existsSync(gitDir) || !fs.statSync(gitDir).isDirectory()) {
return 0; return 0;
} }
core.info(`[Reliability] Scanning for stale lock files in ${gitDir}`);
const now = Date.now();
let removed = 0; let removed = 0;
const cleanDirectory = (directory: string): void => { const cleanDirectory = (directory: string): void => {
@@ -85,12 +107,30 @@ export class BuildReliabilityService {
if (entry.isDirectory()) { if (entry.isDirectory()) {
cleanDirectory(fullPath); cleanDirectory(fullPath);
} else if (entry.name.endsWith('.lock')) { } else if (entry.name.endsWith('.lock')) {
// Check if it is a known lock file location OR under refs/
const relativePath = path.relative(gitDir, fullPath);
const isKnownLock = BuildReliabilityService.LOCK_FILE_NAMES.has(entry.name);
const isRefsLock = relativePath.startsWith('refs' + path.sep);
if (isKnownLock || isRefsLock) {
try { try {
const stat = fs.statSync(fullPath);
const ageMs = now - stat.mtimeMs;
if (ageMs > BuildReliabilityService.LOCK_FILE_MAX_AGE_MS) {
fs.unlinkSync(fullPath); fs.unlinkSync(fullPath);
removed++; removed++;
OrchestratorLogger.log(`[Reliability] Removed stale lock file: ${fullPath}`); core.info(
`[Reliability] Removed stale lock file (age: ${Math.round(ageMs / 1000)}s): ${relativePath}`,
);
} else {
core.info(
`[Reliability] Lock file is recent (age: ${Math.round(ageMs / 1000)}s), skipping: ${relativePath}`,
);
}
} catch { } catch {
OrchestratorLogger.logWarning(`[Reliability] Could not remove lock file: ${fullPath}`); core.warning(`[Reliability] Could not remove lock file: ${fullPath}`);
}
} }
} }
} }
@@ -99,27 +139,32 @@ export class BuildReliabilityService {
} }
}; };
cleanDirectory(gitDirectory); cleanDirectory(gitDir);
if (removed > 0) { if (removed > 0) {
OrchestratorLogger.log(`[Reliability] Cleaned ${removed} stale lock file(s)`); core.info(`[Reliability] Cleaned ${removed} stale lock file(s)`);
} else {
core.info('[Reliability] No stale lock files found');
} }
return removed; return removed;
} }
/** /**
* Validate that submodule .git files point to existing backing stores. * Validate that submodule .git files point to existing backing stores
* Returns list of submodules with broken backing stores. * under .git/modules/. Returns list of submodule paths with broken backing stores.
*/ */
static async validateSubmoduleBackingStores(repoPath: string): Promise<string[]> { static validateSubmoduleBackingStores(repoPath: string = '.'): string[] {
const broken: string[] = []; const broken: string[] = [];
const gitmodulesPath = path.join(repoPath, '.gitmodules'); const gitmodulesPath = path.join(repoPath, '.gitmodules');
if (!fs.existsSync(gitmodulesPath)) { if (!fs.existsSync(gitmodulesPath)) {
core.info('[Reliability] No .gitmodules found, skipping submodule validation');
return broken; return broken;
} }
core.info(`[Reliability] Validating submodule backing stores in ${repoPath}`);
try { try {
const content = fs.readFileSync(gitmodulesPath, 'utf8'); const content = fs.readFileSync(gitmodulesPath, 'utf8');
const pathMatches = content.matchAll(/path\s*=\s*(.+)/g); const pathMatches = content.matchAll(/path\s*=\s*(.+)/g);
@@ -128,7 +173,10 @@ export class BuildReliabilityService {
const submodulePath = match[1].trim(); const submodulePath = match[1].trim();
const gitFile = path.join(repoPath, submodulePath, '.git'); const gitFile = path.join(repoPath, submodulePath, '.git');
if (!fs.existsSync(gitFile)) continue; if (!fs.existsSync(gitFile)) {
// Submodule not initialized -- not necessarily broken
continue;
}
try { try {
const stat = fs.statSync(gitFile); const stat = fs.statSync(gitFile);
@@ -141,64 +189,85 @@ export class BuildReliabilityService {
const backingStore = path.resolve(path.join(repoPath, submodulePath), gitdirMatch[1]); const backingStore = path.resolve(path.join(repoPath, submodulePath), gitdirMatch[1]);
if (!fs.existsSync(backingStore)) { if (!fs.existsSync(backingStore)) {
broken.push(submodulePath); broken.push(submodulePath);
OrchestratorLogger.logWarning( core.warning(`[Reliability] Submodule ${submodulePath} has broken backing store: ${backingStore}`);
`[Reliability] Submodule ${submodulePath} has broken backing store: ${backingStore}`, } else {
); core.info(`[Reliability] Submodule ${submodulePath} backing store OK`);
} }
} else {
broken.push(submodulePath);
core.warning(`[Reliability] Submodule ${submodulePath} .git file has invalid format`);
} }
} }
} catch { } catch {
// Can't read .git file // Can't read .git file
core.warning(`[Reliability] Could not read .git file for submodule: ${submodulePath}`);
} }
} }
} catch { } catch (error: any) {
// Can't read .gitmodules core.warning(`[Reliability] Could not read .gitmodules: ${error.message}`);
} }
if (broken.length > 0) { if (broken.length > 0) {
OrchestratorLogger.logWarning(`[Reliability] ${broken.length} submodule(s) have broken backing stores`); core.warning(`[Reliability] ${broken.length} submodule(s) have broken backing stores`);
} else {
core.info('[Reliability] All submodule backing stores are valid');
} }
return broken; return broken;
} }
/** /**
* Attempt to recover a corrupted repository by removing .git and re-cloning. * Orchestrate recovery of a corrupted repository.
* This is a last resort -- only called when git fsck fails and autoRecover is enabled. * Sequence: fsck -> clean locks -> re-fetch -> retry fsck.
* Returns true if recovery succeeded.
*/ */
static async recoverCorruptedRepo(repoPath: string): Promise<void> { static recoverCorruptedRepo(repoPath: string = '.'): boolean {
OrchestratorLogger.logWarning(`[Reliability] Attempting to recover corrupted repository at ${repoPath}`); core.warning(`[Reliability] Attempting automatic recovery for ${repoPath}`);
const gitDirectory = path.join(repoPath, '.git'); // Step 1: Clean stale lock files that may be preventing operations
if (fs.existsSync(gitDirectory)) { const locksRemoved = BuildReliabilityService.cleanStaleLockFiles(repoPath);
try { if (locksRemoved > 0) {
fs.rmSync(gitDirectory, { recursive: true, force: true }); core.info(`[Reliability] Recovery: cleaned ${locksRemoved} lock file(s)`);
OrchestratorLogger.log(`[Reliability] Removed corrupted .git directory`);
} catch (error: any) {
OrchestratorLogger.logWarning(`[Reliability] Failed to remove .git: ${error.message}`);
}
} }
// Re-initialize -- the checkout action will handle the full clone // Step 2: Re-fetch to restore missing objects
try { try {
await OrchestratorSystem.Run(`git -C "${repoPath}" init`, true); core.info('[Reliability] Recovery: re-fetching from remote');
OrchestratorLogger.log(`[Reliability] Repository re-initialized, checkout action will complete the clone`); execSync(`git -C "${repoPath}" fetch --all`, {
encoding: 'utf8',
timeout: 300_000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Recovery: fetch completed');
} catch (error: any) { } catch (error: any) {
OrchestratorLogger.logWarning(`[Reliability] Re-init failed: ${error.message}`); core.warning(`[Reliability] Recovery: fetch failed: ${error.stderr?.toString() ?? error.message}`);
} }
// Step 3: Retry fsck
const healthy = BuildReliabilityService.checkGitIntegrity(repoPath);
if (healthy) {
core.info('[Reliability] Recovery succeeded -- repository is healthy');
} else {
core.warning('[Reliability] Recovery failed -- repository still has integrity issues');
}
return healthy;
} }
/** /**
* Scan a directory tree for files/directories with Windows reserved names. * Scan a directory tree for files/directories with Windows reserved names.
* Returns list of paths that were cleaned up. * These names (con, prn, aux, nul, com1-9, lpt1-9) with any extension
* cause Unity asset importer infinite loops on Windows.
* Returns list of paths that were removed.
*/ */
static async cleanReservedFilenames(projectPath: string): Promise<string[]> { static cleanReservedFilenames(projectPath: string): string[] {
const assetsPath = path.join(projectPath, 'Assets'); const assetsPath = path.join(projectPath, 'Assets');
if (!fs.existsSync(assetsPath)) { if (!fs.existsSync(assetsPath)) {
core.info(`[Reliability] No Assets directory found at ${assetsPath}, skipping reserved filename scan`);
return []; return [];
} }
OrchestratorLogger.log(`[Reliability] Scanning for reserved filenames in ${assetsPath}`); core.info(`[Reliability] Scanning for reserved filenames in ${assetsPath}`);
const cleaned: string[] = []; const cleaned: string[] = [];
const scanDirectory = (directory: string): void => { const scanDirectory = (directory: string): void => {
@@ -216,9 +285,9 @@ export class BuildReliabilityService {
fs.unlinkSync(fullPath); fs.unlinkSync(fullPath);
} }
cleaned.push(fullPath); cleaned.push(fullPath);
OrchestratorLogger.logWarning(`[Reliability] Removed reserved filename: ${fullPath}`); core.warning(`[Reliability] Removed reserved filename: ${fullPath}`);
} catch { } catch {
OrchestratorLogger.logWarning(`[Reliability] Could not remove: ${fullPath}`); core.warning(`[Reliability] Could not remove reserved filename: ${fullPath}`);
} }
} else if (entry.isDirectory()) { } else if (entry.isDirectory()) {
scanDirectory(fullPath); scanDirectory(fullPath);
@@ -232,105 +301,125 @@ export class BuildReliabilityService {
scanDirectory(assetsPath); scanDirectory(assetsPath);
if (cleaned.length > 0) { if (cleaned.length > 0) {
OrchestratorLogger.logWarning(`[Reliability] Cleaned ${cleaned.length} reserved filename(s)`); core.warning(`[Reliability] Cleaned ${cleaned.length} reserved filename(s)`);
} else { } else {
OrchestratorLogger.log(`[Reliability] No reserved filenames found`); core.info('[Reliability] No reserved filenames found');
} }
return cleaned; return cleaned;
} }
/** /**
* Archive build output to a designated location with retention policy. * Create a tar.gz archive of build output.
*/ */
static async archiveBuildOutput( static archiveBuildOutput(sourcePath: string, archivePath: string): void {
outputPath: string, if (!fs.existsSync(sourcePath)) {
archivePath: string, core.info(`[Reliability] No build output to archive at ${sourcePath}`);
retention: number,
platform: string,
): Promise<void> {
if (!fs.existsSync(outputPath)) {
OrchestratorLogger.log(`[Reliability] No build output to archive at ${outputPath}`);
return; return;
} }
const platformArchive = path.join(archivePath, platform); fs.mkdirSync(archivePath, { recursive: true });
fs.mkdirSync(platformArchive, { recursive: true });
const timestamp = new Date().toISOString().replace(/[.:]/g, '-'); const timestamp = new Date().toISOString().replace(/[.:]/g, '-');
const archiveDirectory = path.join(platformArchive, `build-${timestamp}`); const archiveFile = path.join(archivePath, `build-${timestamp}.tar.gz`);
try { try {
fs.renameSync(outputPath, archiveDirectory); execSync(`tar -czf "${archiveFile}" -C "${path.dirname(sourcePath)}" "${path.basename(sourcePath)}"`, {
OrchestratorLogger.log(`[Reliability] Build output archived to ${archiveDirectory}`); encoding: 'utf8',
} catch { timeout: 600_000,
// Cross-device move -- fall back to copy stdio: ['pipe', 'pipe', 'pipe'],
try { });
await OrchestratorSystem.Run(`cp -r "${outputPath}" "${archiveDirectory}"`, true); core.info(`[Reliability] Build output archived to ${archiveFile}`);
fs.rmSync(outputPath, { recursive: true, force: true });
OrchestratorLogger.log(`[Reliability] Build output copied and archived to ${archiveDirectory}`);
} catch (error: any) { } catch (error: any) {
OrchestratorLogger.logWarning(`[Reliability] Failed to archive build output: ${error.message}`); core.warning(`[Reliability] Failed to archive build output: ${error.stderr?.toString() ?? error.message}`);
return;
} }
} }
// Enforce retention
await BuildReliabilityService.enforceRetention(platformArchive, retention);
}
/** /**
* Enforce retention policy -- keep only the N most recent builds. * Enforce retention policy -- delete archives older than the retention period.
* Returns the number of old builds removed. * Returns the number of old archives removed.
*/ */
static async enforceRetention(archivePath: string, retention: number): Promise<number> { static enforceRetention(archivePath: string, retentionDays: number): number {
if (!fs.existsSync(archivePath)) return 0; if (!fs.existsSync(archivePath)) {
return 0;
try { }
const entries = fs
.readdirSync(archivePath, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => ({
name: entry.name,
path: path.join(archivePath, entry.name),
mtime: fs.statSync(path.join(archivePath, entry.name)).mtimeMs,
}))
.sort((a, b) => b.mtime - a.mtime);
const now = Date.now();
const retentionMs = retentionDays * 24 * 60 * 60 * 1000;
let removed = 0; let removed = 0;
if (entries.length > retention) {
const toRemove = entries.slice(retention);
for (const entry of toRemove) {
try { try {
fs.rmSync(entry.path, { recursive: true, force: true }); const entries = fs.readdirSync(archivePath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(archivePath, entry.name);
try {
const stat = fs.statSync(fullPath);
const ageMs = now - stat.mtimeMs;
if (ageMs > retentionMs) {
if (entry.isDirectory()) {
fs.rmSync(fullPath, { recursive: true, force: true });
} else {
fs.unlinkSync(fullPath);
}
removed++; removed++;
OrchestratorLogger.log(`[Reliability] Removed old build archive: ${entry.name}`); core.info(
`[Reliability] Removed old archive: ${entry.name} (age: ${Math.round(
ageMs / (24 * 60 * 60 * 1000),
)} days)`,
);
}
} catch { } catch {
OrchestratorLogger.logWarning(`[Reliability] Could not remove: ${entry.path}`); core.warning(`[Reliability] Could not process archive entry: ${fullPath}`);
} }
} }
} catch {
core.warning(`[Reliability] Could not read archive directory: ${archivePath}`);
return 0;
} }
if (removed > 0) { if (removed > 0) {
OrchestratorLogger.log( core.info(
`[Reliability] Retention enforced: removed ${removed} old archive(s), keeping ${retention}`, `[Reliability] Retention enforced: removed ${removed} old archive(s), retention: ${retentionDays} days`,
); );
} }
return removed; return removed;
} catch {
return 0;
}
} }
/** /**
* Configure environment for corrupted system git config bypass. * Configure git environment variables for CI reliability.
* Sets GIT_TERMINAL_PROMPT=0, increases http.postBuffer, enables core.longpaths.
*/ */
static configureGitEnvironment(): Record<string, string> { static configureGitEnvironment(): void {
return { core.info('[Reliability] Configuring git environment for CI');
GIT_CONFIG_NOSYSTEM: '1',
}; // Prevent git from prompting for credentials (hangs in CI)
process.env.GIT_TERMINAL_PROMPT = '0';
core.info('[Reliability] Set GIT_TERMINAL_PROMPT=0');
try {
// Increase http.postBuffer to 500MB for large pushes
execSync('git config --global http.postBuffer 524288000', {
encoding: 'utf8',
timeout: 10_000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Set http.postBuffer=524288000 (500MB)');
} catch (error: any) {
core.warning(`[Reliability] Could not set http.postBuffer: ${error.message}`);
}
try {
// Enable long paths on Windows
execSync('git config --global core.longpaths true', {
encoding: 'utf8',
timeout: 10_000,
stdio: ['pipe', 'pipe', 'pipe'],
});
core.info('[Reliability] Set core.longpaths=true');
} catch (error: any) {
core.warning(`[Reliability] Could not set core.longpaths: ${error.message}`);
}
} }
} }