PR feedback

This commit is contained in:
Frostebite
2025-12-06 00:53:27 +00:00
parent f9ef711978
commit 459b9298b2
5 changed files with 79 additions and 21 deletions
Generated Vendored
+23 -4
View File
@@ -4003,6 +4003,7 @@ class KubernetesPods {
// Check if only PreStopHook failed but container succeeded
const hasPreStopHookFailure = events.some((e) => e.reason === 'FailedPreStopHook');
const wasKilled = events.some((e) => e.reason === 'Killing');
const hasExceededGracePeriod = events.some((e) => e.reason === 'ExceededGracePeriod');
// If container succeeded (exit code 0), PreStopHook failure is non-critical
// Also check if pod was killed but container might have succeeded
if (containerSucceeded && containerExitCode === 0) {
@@ -4017,10 +4018,10 @@ class KubernetesPods {
// Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure
}
// If pod was killed and we have PreStopHook failure but no container status yet, wait a bit
// If pod was killed and we have PreStopHook failure, wait for container status
// The container might have succeeded but status hasn't been updated yet
if (wasKilled && hasPreStopHookFailure && containerExitCode === undefined) {
cloud_runner_logger_1.default.log(`Pod ${podName} was killed with PreStopHook failure, but container status not yet available. Waiting for container status...`);
if (wasKilled && hasPreStopHookFailure && (containerExitCode === undefined || !containerSucceeded)) {
cloud_runner_logger_1.default.log(`Pod ${podName} was killed with PreStopHook failure. Waiting for container status to determine if container succeeded...`);
// Wait a bit for container status to become available (up to 30 seconds)
for (let i = 0; i < 6; i++) {
await new Promise((resolve) => setTimeout(resolve, 5000));
@@ -4037,6 +4038,8 @@ class KubernetesPods {
else {
cloud_runner_logger_1.default.log(`Pod ${podName} container failed with exit code ${updatedExitCode} after waiting.`);
errorDetails.push(`Container terminated after wait: exit code ${updatedExitCode}`);
containerExitCode = updatedExitCode;
containerSucceeded = false;
break;
}
}
@@ -4046,7 +4049,18 @@ class KubernetesPods {
cloud_runner_logger_1.default.log(`Error while waiting for container status: ${waitError}`);
}
}
cloud_runner_logger_1.default.log(`Container status still not available after waiting. Assuming failure due to PreStopHook issues.`);
// If we still don't have container status after waiting, but only PreStopHook failed,
// be lenient - the container might have succeeded but status wasn't updated
if (containerExitCode === undefined && hasPreStopHookFailure && !hasExceededGracePeriod) {
cloud_runner_logger_1.default.logWarning(`Pod ${podName} container status not available after waiting, but only PreStopHook failed (no ExceededGracePeriod). Assuming container may have succeeded.`);
return false; // Be lenient - PreStopHook failure alone is not fatal
}
cloud_runner_logger_1.default.log(`Container status check completed. Exit code: ${containerExitCode}, PreStopHook failure: ${hasPreStopHookFailure}`);
}
// If we only have PreStopHook failure and no actual container failure, be lenient
if (hasPreStopHookFailure && !hasExceededGracePeriod && containerExitCode === undefined) {
cloud_runner_logger_1.default.logWarning(`Pod ${podName} has PreStopHook failure but no container failure detected. Treating as non-fatal.`);
return false; // PreStopHook failure alone is not fatal if container status is unclear
}
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
cloud_runner_logger_1.default.log(errorMessage);
@@ -5331,6 +5345,11 @@ class Caching {
await cloud_runner_system_1.CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
(0, node_console_1.assert)(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
(0, node_console_1.assert)(await fileExists(node_path_1.default.basename(sourceFolder)), 'source folder exists');
// Ensure the cache folder directory exists before moving the file
// (it might have been deleted by cleanup if it was empty)
if (!(await fileExists(cacheFolder))) {
await cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
}
await cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
(0, node_console_1.assert)(await fileExists(`${node_path_1.default.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`), 'cache archive exists inside cache folder');
Generated Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long