Cloud runner develop - Stabilizes kubernetes provider (#531)

* fixes * fixes * fixes * fixes * fixes * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * check for startup message in workflows * Update cloud-runner-ci-pipeline.yml * Update cloud-runner-ci-pipeline.yml * no storage class specified * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * updates * log file path * latest develop * log file path * log file path * Update package.json * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * log file path * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * stream logs through standard input and new remote client cli command * update pipeline to use k3s * version: 'latest' * fixes * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * disable aws pipe for now * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * push k8s logs to LOG SERVICE IP * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * tests * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * podname logs for log service * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * hashed logs * no wait, just repeat logs * no wait, just repeat logs * remove typo - double await * test fix - kubernetes - name typo in github yaml * test fix - kubernetes - name typo in github yaml * check missing log file * check missing log file * Push to steam test * Push to steam test * Fix path * k8s reliable log hashing * k8s reliable log hashing * k8s reliable log hashing * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * hashed logging k8s * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line * Fix exit flow for k8s job * hash comparison logging for log complete in k8s flow * Interrupt k8s logs when logs found * cleanup async parameter * cleanup async parameter * cleanup async parameter * fixes * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix
2026-06-12 08:53:55 -07:00 · 2024-02-06 23:46:31 +00:00
parent 2800d14403
commit e73b48fb38
47 changed files with 1456 additions and 856 deletions
@@ -12,16 +12,83 @@ import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import YAML from 'yaml';
 import GitHub from '../../github';
 import BuildParameters from '../../build-parameters';
+import { Cli } from '../../cli/cli';
+import CloudRunnerOptions from '../options/cloud-runner-options';

 export class RemoteClient {
  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-  static async runRemoteClientJob() {
+  static async setupRemoteClient() {
    CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
    if (!(await RemoteClient.handleRetainedWorkspace())) {
      await RemoteClient.bootstrapRepository();
    }
+    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
    await RemoteClient.runCustomHookFiles(`before-build`);
  }
+
+  @CliFunction('remote-cli-log-stream', `log stream from standard input`)
+  public static async remoteClientLogStream() {
+    const logFile = Cli.options!['logFile'];
+    process.stdin.resume();
+    process.stdin.setEncoding('utf8');
+
+    let lingeringLine = '';
+
+    process.stdin.on('data', (chunk) => {
+      const lines = chunk.toString().split('\n');
+
+      lines[0] = lingeringLine + lines[0];
+      lingeringLine = lines.pop() || '';
+
+      for (const element of lines) {
+        if (CloudRunnerOptions.providerStrategy !== 'k8s') {
+          CloudRunnerLogger.log(element);
+        } else {
+          fs.appendFileSync(logFile, element);
+          CloudRunnerLogger.log(element);
+        }
+      }
+    });
+
+    process.stdin.on('end', () => {
+      if (CloudRunnerOptions.providerStrategy !== 'k8s') {
+        CloudRunnerLogger.log(lingeringLine);
+      } else {
+        fs.appendFileSync(logFile, lingeringLine);
+        CloudRunnerLogger.log(lingeringLine);
+      }
+    });
+  }
+
+  @CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
+  public static async remoteClientPostBuild(): Promise<string> {
+    RemoteClientLogger.log(`Running POST build tasks`);
+
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
+      `lib-${CloudRunner.buildParameters.buildGuid}`,
+    );
+
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
+      `build-${CloudRunner.buildParameters.buildGuid}`,
+    );
+
+    if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
+      await CloudRunnerSystem.Run(
+        `rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
+      );
+    }
+
+    await RemoteClient.runCustomHookFiles(`after-build`);
+
+    // WIP - need to give the pod permissions to create config map
+    await RemoteClientLogger.handleLogManagementPostJob();
+
+    return new Promise((result) => result(``));
+  }
  static async runCustomHookFiles(hookLifecycle: string) {
    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
@@ -47,7 +114,6 @@ export class RemoteClient {
      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
    );
    await RemoteClient.cloneRepoWithoutLFSFiles();
-    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
    await RemoteClient.sizeOfFolder(
      'repo before lfs cache pull',
      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),