Cloud runner develop - latest fixes (#524)

2026-06-13 01:13:54 -07:00 · 2023-03-27 12:14:23 +01:00
parent 309d668d63
commit 7abb3a409d
91 changed files with 4943 additions and 3599 deletions
@@ -2,19 +2,18 @@ import * as k8s from '@kubernetes/client-node';
 import { BuildParameters } from '../../..';
 import * as core from '@actions/core';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import KubernetesStorage from './kubernetes-storage';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import KubernetesTaskRunner from './kubernetes-task-runner';
 import KubernetesSecret from './kubernetes-secret';
 import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
 import KubernetesServiceAccount from './kubernetes-service-account';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
-import KubernetesPods from './kubernetes-pods';

 class Kubernetes implements ProviderInterface {
  public static Instance: Kubernetes;
@@ -94,16 +93,8 @@ class Kubernetes implements ProviderInterface {
  ) {
    try {
      this.buildParameters = buildParameters;
-      const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
-      this.pvcName = `unity-builder-pvc-${id}`;
-      this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
+      this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
      this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
-      await KubernetesStorage.createPersistentVolumeClaim(
-        buildParameters,
-        this.pvcName,
-        this.kubeClient,
-        this.namespace,
-      );

      await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
    } catch (error) {
@@ -124,74 +115,99 @@ class Kubernetes implements ProviderInterface {
      CloudRunnerLogger.log('Cloud Runner K8s workflow!');

      // Setup
+      const id = BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
+        ? CloudRunner.lockedWorkspace
+        : this.buildParameters.buildGuid;
+      this.pvcName = `unity-builder-pvc-${id}`;
+      await KubernetesStorage.createPersistentVolumeClaim(
+        this.buildParameters,
+        this.pvcName,
+        this.kubeClient,
+        this.namespace,
+      );
      this.buildGuid = buildGuid;
      this.secretName = `build-credentials-${this.buildGuid}`;
      this.jobName = `unity-builder-job-${this.buildGuid}`;
      this.containerName = `main`;
      await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
-      await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
-      this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
-      CloudRunnerLogger.log('Watching pod until running');
-      await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
      let output = '';
-      // eslint-disable-next-line no-constant-condition
-      while (true) {
-        try {
-          CloudRunnerLogger.log('Pod running, streaming logs');
-          output = await KubernetesTaskRunner.runTask(
-            this.kubeConfig,
-            this.kubeClient,
-            this.jobName,
-            this.podName,
-            'main',
-            this.namespace,
-          );
-          const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
+      try {
+        CloudRunnerLogger.log('Job does not exist');
+        await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
+        CloudRunnerLogger.log('Watching pod until running');
+        await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);

-          if (!running) {
-            CloudRunnerLogger.log(`Pod not found, assumed ended!`);
-            break;
-          } else {
-            CloudRunnerLogger.log('Pod still running, recovering stream...');
-          }
-          await this.cleanupTaskResources();
-        } catch (error: any) {
-          let errorParsed;
-          try {
-            errorParsed = JSON.parse(error);
-          } catch {
-            errorParsed = error;
-          }
-
-          const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
-          const errorMessage = errorParsed.message || reason;
-
-          const continueStreaming =
-            errorMessage.includes(`dial timeout, backstop`) ||
-            errorMessage.includes(`HttpError: HTTP request failed`) ||
-            errorMessage.includes(`an error occurred when try to find container`) ||
-            errorMessage.includes(`not found`) ||
-            errorMessage.includes(`Not Found`);
-          if (continueStreaming) {
-            CloudRunnerLogger.log('Log Stream Container Not Found');
-            await new Promise((resolve) => resolve(5000));
-            continue;
-          } else {
-            CloudRunnerLogger.log(`error running k8s workflow ${error}`);
-            throw error;
-          }
-        }
+        CloudRunnerLogger.log('Pod running, streaming logs');
+        CloudRunnerLogger.log(
+          `Starting logs follow for pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace} pvc: ${this.pvcName} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
+        );
+        output += await KubernetesTaskRunner.runTask(
+          this.kubeConfig,
+          this.kubeClient,
+          this.jobName,
+          this.podName,
+          this.containerName,
+          this.namespace,
+        );
+      } catch (error: any) {
+        CloudRunnerLogger.log(`error running k8s workflow ${error}`);
+        await new Promise((resolve) => setTimeout(resolve, 3000));
+        CloudRunnerLogger.log(
+          JSON.stringify(
+            (await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
+              .map((x) => {
+                return {
+                  message: x.message || ``,
+                  name: x.metadata.name || ``,
+                  reason: x.reason || ``,
+                };
+              })
+              .filter((x) => x.name.includes(this.podName)),
+            undefined,
+            4,
+          ),
+        );
+        await this.cleanupTaskResources();
+        throw error;
      }

+      await this.cleanupTaskResources();
+
      return output;
    } catch (error) {
      CloudRunnerLogger.log('Running job failed');
      core.error(JSON.stringify(error, undefined, 4));
-      await this.cleanupTaskResources();
+
+      // await this.cleanupTaskResources();
      throw error;
    }
  }

+  private async createJob(
+    commands: string,
+    image: string,
+    mountdir: string,
+    workingdir: string,
+    environment: CloudRunnerEnvironmentVariable[],
+    secrets: CloudRunnerSecret[],
+  ) {
+    await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
+    const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
+    this.setPodNameAndContainerName(find);
+  }
+
+  private async doesJobExist(name: string) {
+    const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
+
+    return jobs.body.items.some((x) => x.metadata?.name === name);
+  }
+
+  private async doesFailedJobExist() {
+    const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
+
+    return podStatus.body.status?.phase === `Failed`;
+  }
+
  private async createNamespacedJob(
    commands: string,
    image: string,
@@ -215,14 +231,15 @@ class Kubernetes implements ProviderInterface {
          this.pvcName,
          this.jobName,
          k8s,
+          this.containerName,
        );
        await new Promise((promise) => setTimeout(promise, 15000));
-        await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
+        const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
        CloudRunnerLogger.log(`Build job created`);
        await new Promise((promise) => setTimeout(promise, 5000));
        CloudRunnerLogger.log('Job created');

-        return;
+        return result.body.metadata?.name;
      } catch (error) {
        CloudRunnerLogger.log(`Error occured creating job: ${error}`);
        throw error;
@@ -232,7 +249,7 @@ class Kubernetes implements ProviderInterface {

  setPodNameAndContainerName(pod: k8s.V1Pod) {
    this.podName = pod.metadata?.name || '';
-    this.containerName = pod.status?.containerStatuses?.[0].name || '';
+    this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
  }

  async cleanupTaskResources() {
@@ -265,7 +282,7 @@ class Kubernetes implements ProviderInterface {
    // eslint-disable-next-line no-unused-vars
    defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
  ) {
-    if (buildParameters.retainWorkspace) {
+    if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
      return;
    }
    CloudRunnerLogger.log(`deleting PVC`);
@@ -1,8 +1,8 @@
 import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import CloudRunner from '../../cloud-runner';

 class KubernetesJobSpecFactory {
@@ -19,63 +19,8 @@ class KubernetesJobSpecFactory {
    pvcName: string,
    jobName: string,
    k8s: any,
+    containerName: string,
  ) {
-    environment.push(
-      ...[
-        {
-          name: 'GITHUB_SHA',
-          value: buildGuid,
-        },
-        {
-          name: 'GITHUB_WORKSPACE',
-          value: '/data/repo',
-        },
-        {
-          name: 'PROJECT_PATH',
-          value: buildParameters.projectPath,
-        },
-        {
-          name: 'BUILD_PATH',
-          value: buildParameters.buildPath,
-        },
-        {
-          name: 'BUILD_FILE',
-          value: buildParameters.buildFile,
-        },
-        {
-          name: 'BUILD_NAME',
-          value: buildParameters.buildName,
-        },
-        {
-          name: 'BUILD_METHOD',
-          value: buildParameters.buildMethod,
-        },
-        {
-          name: 'CUSTOM_PARAMETERS',
-          value: buildParameters.customParameters,
-        },
-        {
-          name: 'CHOWN_FILES_TO',
-          value: buildParameters.chownFilesTo,
-        },
-        {
-          name: 'BUILD_TARGET',
-          value: buildParameters.targetPlatform,
-        },
-        {
-          name: 'ANDROID_VERSION_CODE',
-          value: buildParameters.androidVersionCode.toString(),
-        },
-        {
-          name: 'ANDROID_KEYSTORE_NAME',
-          value: buildParameters.androidKeystoreName,
-        },
-        {
-          name: 'ANDROID_KEYALIAS_NAME',
-          value: buildParameters.androidKeyaliasName,
-        },
-      ],
-    );
    const job = new k8s.V1Job();
    job.apiVersion = 'batch/v1';
    job.kind = 'Job';
@@ -87,6 +32,7 @@ class KubernetesJobSpecFactory {
      },
    };
    job.spec = {
+      ttlSecondsAfterFinished: 9999,
      backoffLimit: 0,
      template: {
        spec: {
@@ -100,16 +46,20 @@ class KubernetesJobSpecFactory {
          ],
          containers: [
            {
-              name: 'main',
+              ttlSecondsAfterFinished: 9999,
+              name: containerName,
              image,
              command: ['/bin/sh'],
-              args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
+              args: [
+                '-c',
+                `${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
+              ],

              workingDir: `${workingDirectory}`,
              resources: {
                requests: {
-                  memory: buildParameters.cloudRunnerMemory || '750M',
-                  cpu: buildParameters.cloudRunnerCpu || '1',
+                  memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
+                  cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
                },
              },
              env: [
@@ -135,7 +85,7 @@ class KubernetesJobSpecFactory {
              volumeMounts: [
                {
                  name: 'build-mount',
-                  mountPath: `/${mountdir}`,
+                  mountPath: `${mountdir}`,
                },
              ],
              lifecycle: {
@@ -158,7 +108,7 @@ class KubernetesJobSpecFactory {
      },
    };

-    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
+    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';

    return job;
  }
@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 class KubernetesPods {
  public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
@@ -12,6 +12,12 @@ class KubernetesPods {

    return running;
  }
+  public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
+    const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
+    const phase = pods?.status?.phase || 'undefined status';
+
+    return phase;
+  }
 }

 export default KubernetesPods;
@@ -1,7 +1,7 @@
 import { CoreV1Api } from '@kubernetes/client-node';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import * as k8s from '@kubernetes/client-node';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as base64 from 'base-64';

 class KubernetesSecret {
@@ -2,7 +2,7 @@ import waitUntil from 'async-wait-until';
 import * as core from '@actions/core';
 import * as k8s from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { IncomingMessage } from 'node:http';
 import GitHub from '../../../github';

@@ -1,12 +1,15 @@
-import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
-import { Writable } from 'stream';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
-import * as core from '@actions/core';
-import { CloudRunnerStatics } from '../../cloud-runner-statics';
+import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import waitUntil from 'async-wait-until';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
+import CloudRunner from '../../cloud-runner';
+import KubernetesPods from './kubernetes-pods';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';

 class KubernetesTaskRunner {
+  static lastReceivedTimestamp: number = 0;
+  static readonly maxRetry: number = 3;
+  static lastReceivedMessage: string = ``;
  static async runTask(
    kubeConfig: KubeConfig,
    kubeClient: CoreV1Api,
@@ -15,84 +18,120 @@ class KubernetesTaskRunner {
    containerName: string,
    namespace: string,
  ) {
-    CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
-    const stream = new Writable();
    let output = '';
-    let didStreamAnyLogs: boolean = false;
    let shouldReadLogs = true;
    let shouldCleanup = true;
-    stream._write = (chunk, encoding, next) => {
-      didStreamAnyLogs = true;
-      let message = chunk.toString().trimRight(`\n`);
-      message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
-      ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
-        message,
-        shouldReadLogs,
-        shouldCleanup,
-        output,
-      ));
-      next();
-    };
-    const logOptions = {
-      follow: true,
-      pretty: false,
-      previous: false,
-    };
-    try {
-      const resultError = await new Promise((resolve) =>
-        new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
+    let sinceTime = ``;
+    let retriesAfterFinish = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+      const lastReceivedMessage =
+        KubernetesTaskRunner.lastReceivedTimestamp > 0
+          ? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
+          : ``;
+      CloudRunnerLogger.log(
+        `Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
      );
-      stream.destroy();
-      if (resultError) {
-        throw resultError;
+      if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
+        const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
+        const dateTimeIsoString = currentDate.toISOString();
+        sinceTime = ` --since-time="${dateTimeIsoString}"`;
      }
-      if (!didStreamAnyLogs) {
-        core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
-        core.error(
-          JSON.stringify(
-            {
-              events: (await kubeClient.listNamespacedEvent(namespace)).body.items
-                .filter((x) => {
-                  return x.involvedObject.name === podName || x.involvedObject.name === jobName;
-                })
-                .map((x) => {
-                  return {
-                    type: x.involvedObject.kind,
-                    name: x.involvedObject.name,
-                    message: x.message,
-                  };
-                }),
-            },
-            undefined,
-            4,
-          ),
+      let extraFlags = ``;
+      extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
+        ? ` -f -c ${containerName}`
+        : ` --previous`;
+      let lastMessageSeenIncludedInChunk = false;
+      let lastMessageSeen = false;
+
+      let logs;
+
+      try {
+        logs = await CloudRunnerSystem.Run(
+          `kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
+          false,
+          true,
        );
-        throw new Error(`No logs streamed from k8s`);
+      } catch (error: any) {
+        await new Promise((resolve) => setTimeout(resolve, 3000));
+        const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
+        CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
+        if (continueStreaming) {
+          continue;
+        }
+        if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
+          retriesAfterFinish++;
+
+          continue;
+        }
+        throw error;
      }
-    } catch (error) {
-      if (stream) {
-        stream.destroy();
+      const splitLogs = logs.split(`\n`);
+      for (const chunk of splitLogs) {
+        if (
+          chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
+          KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
+        ) {
+          CloudRunnerLogger.log(`Previous log message found ${chunk}`);
+          lastMessageSeenIncludedInChunk = true;
+        }
+      }
+      for (const chunk of splitLogs) {
+        const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
+        if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
+          lastMessageSeen = true;
+        }
+        if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
+          continue;
+        }
+        const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
+        KubernetesTaskRunner.lastReceivedMessage = chunk;
+        KubernetesTaskRunner.lastReceivedTimestamp = newDate;
+        ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+          message,
+          shouldReadLogs,
+          shouldCleanup,
+          output,
+        ));
+      }
+      if (FollowLogStreamService.DidReceiveEndOfTransmission) {
+        CloudRunnerLogger.log('end of log stream');
+        break;
      }
-      throw error;
    }
-    CloudRunnerLogger.log('end of log stream');

    return output;
  }

  static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
    let success: boolean = false;
+    let message = ``;
    CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
    await waitUntil(
      async () => {
        const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
        const phase = status?.body.status?.phase;
        success = phase === 'Running';
-        CloudRunnerLogger.log(
-          `${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
-            status.body.status?.conditions?.[0].message || ''
-          }`,
-        );
+        message = `Phase:${status.body.status?.phase} \n Reason:${
+          status.body.status?.conditions?.[0].reason || ''
+        } \n Message:${status.body.status?.conditions?.[0].message || ''}`;
+
+        // CloudRunnerLogger.log(
+        //   JSON.stringify(
+        //     (await kubeClient.listNamespacedEvent(namespace)).body.items
+        //       .map((x) => {
+        //         return {
+        //           message: x.message || ``,
+        //           name: x.metadata.name || ``,
+        //           reason: x.reason || ``,
+        //         };
+        //       })
+        //       .filter((x) => x.name.includes(podName)),
+        //     undefined,
+        //     4,
+        //   ),
+        // );
        if (success || phase !== 'Pending') return true;

        return false;
@@ -102,6 +141,9 @@ class KubernetesTaskRunner {
        intervalBetweenAttempts: 15000,
      },
    );
+    if (!success) {
+      CloudRunnerLogger.log(message);
+    }

    return success;
  }