mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-14 20:16:48 -07:00
Cloud runner develop - Stabilizes kubernetes provider (#531)
* fixes * fixes * fixes * fixes * fixes * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * Update cloud-runner-ci-pipeline.yml * Update cloud-runner-ci-pipeline.yml * no storage class specified * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * updates * log file path * latest develop * log file path * log file path * Update package.json * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * update pipeline to use k3s * version: 'latest' * fixes * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * no wait, just repeat logs * no wait, just repeat logs * remove typo - double await * test fix - kubernetes - name typo in github yaml * test fix - kubernetes - name typo in github yaml * check missing log file * check missing log file * Push to steam test * Push to steam test * Fix path * k8s reliable log hashing * k8s reliable log hashing * k8s reliable log hashing * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Fix exit flow for k8s job * hash comparison logging for log complete in k8s flow * Interrupt k8s logs when logs found * cleanup async parameter * cleanup async parameter * cleanup async parameter * fixes * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix
This commit is contained in:
Frostebite
@@ -7,7 +7,6 @@ import KubernetesPods from './kubernetes-pods';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
|
||||
class KubernetesTaskRunner {
|
||||
static lastReceivedTimestamp: number = 0;
|
||||
static readonly maxRetry: number = 3;
|
||||
static lastReceivedMessage: string = ``;
|
||||
|
||||
@@ -22,38 +21,33 @@ class KubernetesTaskRunner {
|
||||
let output = '';
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
let sinceTime = ``;
|
||||
let retriesAfterFinish = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const lastReceivedMessage =
|
||||
KubernetesTaskRunner.lastReceivedTimestamp > 0
|
||||
? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
|
||||
: ``;
|
||||
CloudRunnerLogger.log(
|
||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
|
||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
|
||||
);
|
||||
if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
|
||||
const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
|
||||
const dateTimeIsoString = currentDate.toISOString();
|
||||
sinceTime = ` --since-time="${dateTimeIsoString}"`;
|
||||
}
|
||||
let extraFlags = ``;
|
||||
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
|
||||
? ` -f -c ${containerName}`
|
||||
: ` --previous`;
|
||||
let lastMessageSeenIncludedInChunk = false;
|
||||
let lastMessageSeen = false;
|
||||
|
||||
let logs;
|
||||
const callback = (outputChunk: string) => {
|
||||
output += outputChunk;
|
||||
|
||||
// split output chunk and handle per line
|
||||
for (const chunk of outputChunk.split(`\n`)) {
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
chunk,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
};
|
||||
try {
|
||||
logs = await CloudRunnerSystem.Run(
|
||||
`kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
|
||||
} catch (error: any) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
@@ -68,34 +62,6 @@ class KubernetesTaskRunner {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const splitLogs = logs.split(`\n`);
|
||||
for (const chunk of splitLogs) {
|
||||
if (
|
||||
chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
|
||||
KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
|
||||
) {
|
||||
CloudRunnerLogger.log(`Previous log message found ${chunk}`);
|
||||
lastMessageSeenIncludedInChunk = true;
|
||||
}
|
||||
}
|
||||
for (const chunk of splitLogs) {
|
||||
const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
|
||||
if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
|
||||
lastMessageSeen = true;
|
||||
}
|
||||
if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
|
||||
continue;
|
||||
}
|
||||
const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
|
||||
KubernetesTaskRunner.lastReceivedMessage = chunk;
|
||||
KubernetesTaskRunner.lastReceivedTimestamp = newDate;
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
message,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
CloudRunnerLogger.log('end of log stream');
|
||||
break;
|
||||
@@ -106,14 +72,14 @@ class KubernetesTaskRunner {
|
||||
}
|
||||
|
||||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||
let success: boolean = false;
|
||||
let waitComplete: boolean = false;
|
||||
let message = ``;
|
||||
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const phase = status?.body.status?.phase;
|
||||
success = phase === 'Running';
|
||||
waitComplete = phase !== 'Pending';
|
||||
message = `Phase:${status.body.status?.phase} \n Reason:${
|
||||
status.body.status?.conditions?.[0].reason || ''
|
||||
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
|
||||
@@ -133,7 +99,7 @@ class KubernetesTaskRunner {
|
||||
// 4,
|
||||
// ),
|
||||
// );
|
||||
if (success || phase !== 'Pending') return true;
|
||||
if (waitComplete || phase !== 'Pending') return true;
|
||||
|
||||
return false;
|
||||
},
|
||||
@@ -142,11 +108,11 @@ class KubernetesTaskRunner {
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
if (!success) {
|
||||
if (!waitComplete) {
|
||||
CloudRunnerLogger.log(message);
|
||||
}
|
||||
|
||||
return success;
|
||||
return waitComplete;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user