mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-14 20:16:48 -07:00
Cloud runner develop - latest fixes (#524)
Cloud runner develop - latest fixes (#524)
This commit is contained in:
Frostebite
@@ -1,12 +1,15 @@
|
||||
import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
|
||||
import { Writable } from 'stream';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import * as core from '@actions/core';
|
||||
import { CloudRunnerStatics } from '../../cloud-runner-statics';
|
||||
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import waitUntil from 'async-wait-until';
|
||||
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import KubernetesPods from './kubernetes-pods';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
|
||||
class KubernetesTaskRunner {
|
||||
static lastReceivedTimestamp: number = 0;
|
||||
static readonly maxRetry: number = 3;
|
||||
static lastReceivedMessage: string = ``;
|
||||
static async runTask(
|
||||
kubeConfig: KubeConfig,
|
||||
kubeClient: CoreV1Api,
|
||||
@@ -15,84 +18,120 @@ class KubernetesTaskRunner {
|
||||
containerName: string,
|
||||
namespace: string,
|
||||
) {
|
||||
CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
|
||||
const stream = new Writable();
|
||||
let output = '';
|
||||
let didStreamAnyLogs: boolean = false;
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
stream._write = (chunk, encoding, next) => {
|
||||
didStreamAnyLogs = true;
|
||||
let message = chunk.toString().trimRight(`\n`);
|
||||
message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
message,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
next();
|
||||
};
|
||||
const logOptions = {
|
||||
follow: true,
|
||||
pretty: false,
|
||||
previous: false,
|
||||
};
|
||||
try {
|
||||
const resultError = await new Promise((resolve) =>
|
||||
new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
|
||||
let sinceTime = ``;
|
||||
let retriesAfterFinish = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const lastReceivedMessage =
|
||||
KubernetesTaskRunner.lastReceivedTimestamp > 0
|
||||
? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
|
||||
: ``;
|
||||
CloudRunnerLogger.log(
|
||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
|
||||
);
|
||||
stream.destroy();
|
||||
if (resultError) {
|
||||
throw resultError;
|
||||
if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
|
||||
const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
|
||||
const dateTimeIsoString = currentDate.toISOString();
|
||||
sinceTime = ` --since-time="${dateTimeIsoString}"`;
|
||||
}
|
||||
if (!didStreamAnyLogs) {
|
||||
core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
|
||||
core.error(
|
||||
JSON.stringify(
|
||||
{
|
||||
events: (await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
.filter((x) => {
|
||||
return x.involvedObject.name === podName || x.involvedObject.name === jobName;
|
||||
})
|
||||
.map((x) => {
|
||||
return {
|
||||
type: x.involvedObject.kind,
|
||||
name: x.involvedObject.name,
|
||||
message: x.message,
|
||||
};
|
||||
}),
|
||||
},
|
||||
undefined,
|
||||
4,
|
||||
),
|
||||
let extraFlags = ``;
|
||||
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
|
||||
? ` -f -c ${containerName}`
|
||||
: ` --previous`;
|
||||
let lastMessageSeenIncludedInChunk = false;
|
||||
let lastMessageSeen = false;
|
||||
|
||||
let logs;
|
||||
|
||||
try {
|
||||
logs = await CloudRunnerSystem.Run(
|
||||
`kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
throw new Error(`No logs streamed from k8s`);
|
||||
} catch (error: any) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
|
||||
if (continueStreaming) {
|
||||
continue;
|
||||
}
|
||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||
retriesAfterFinish++;
|
||||
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
if (stream) {
|
||||
stream.destroy();
|
||||
const splitLogs = logs.split(`\n`);
|
||||
for (const chunk of splitLogs) {
|
||||
if (
|
||||
chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
|
||||
KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
|
||||
) {
|
||||
CloudRunnerLogger.log(`Previous log message found ${chunk}`);
|
||||
lastMessageSeenIncludedInChunk = true;
|
||||
}
|
||||
}
|
||||
for (const chunk of splitLogs) {
|
||||
const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
|
||||
if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
|
||||
lastMessageSeen = true;
|
||||
}
|
||||
if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
|
||||
continue;
|
||||
}
|
||||
const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
|
||||
KubernetesTaskRunner.lastReceivedMessage = chunk;
|
||||
KubernetesTaskRunner.lastReceivedTimestamp = newDate;
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
message,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
CloudRunnerLogger.log('end of log stream');
|
||||
break;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
CloudRunnerLogger.log('end of log stream');
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||
let success: boolean = false;
|
||||
let message = ``;
|
||||
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const phase = status?.body.status?.phase;
|
||||
success = phase === 'Running';
|
||||
CloudRunnerLogger.log(
|
||||
`${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
|
||||
status.body.status?.conditions?.[0].message || ''
|
||||
}`,
|
||||
);
|
||||
message = `Phase:${status.body.status?.phase} \n Reason:${
|
||||
status.body.status?.conditions?.[0].reason || ''
|
||||
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
|
||||
|
||||
// CloudRunnerLogger.log(
|
||||
// JSON.stringify(
|
||||
// (await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
// .map((x) => {
|
||||
// return {
|
||||
// message: x.message || ``,
|
||||
// name: x.metadata.name || ``,
|
||||
// reason: x.reason || ``,
|
||||
// };
|
||||
// })
|
||||
// .filter((x) => x.name.includes(podName)),
|
||||
// undefined,
|
||||
// 4,
|
||||
// ),
|
||||
// );
|
||||
if (success || phase !== 'Pending') return true;
|
||||
|
||||
return false;
|
||||
@@ -102,6 +141,9 @@ class KubernetesTaskRunner {
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
if (!success) {
|
||||
CloudRunnerLogger.log(message);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user