unity-builder/src/model/orchestrator/providers/k8s/kubernetes-job-spec-factory.ts

import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorSecret from '../../options/orchestrator-secret';
import Orchestrator from '../../orchestrator';
import OrchestratorLogger from '../../services/core/orchestrator-logger';

class KubernetesJobSpecFactory {
  static getJobSpec(
    command: string,
    image: string,
    mountdir: string,
    workingDirectory: string,
    environment: OrchestratorEnvironmentVariable[],
    secrets: OrchestratorSecret[],
    buildGuid: string,
    buildParameters: BuildParameters,
    secretName: string,
    pvcName: string,
    jobName: string,
    k8s: any,
    containerName: string,
    ip: string = '',
  ) {
    const endpointEnvironmentNames = new Set([
      'AWS_S3_ENDPOINT',
      'AWS_ENDPOINT',
      'AWS_CLOUD_FORMATION_ENDPOINT',
      'AWS_ECS_ENDPOINT',
      'AWS_KINESIS_ENDPOINT',
      'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
      'INPUT_AWSS3ENDPOINT',
      'INPUT_AWSENDPOINT',
    ]);

    // Determine the LocalStack hostname to use for K8s pods
    // Priority: K8S_LOCALSTACK_HOST env var > localstack-main (container name on shared network)
    // Note: Using K8S_LOCALSTACK_HOST instead of LOCALSTACK_HOST to avoid conflict with awslocal CLI
    const localstackHost = process.env['K8S_LOCALSTACK_HOST'] || 'localstack-main';
    OrchestratorLogger.log(`K8s pods will use LocalStack host: ${localstackHost}`);

    const adjustedEnvironment = environment.map((x) => {
      let value = x.value;
      if (
        typeof value === 'string' &&
        endpointEnvironmentNames.has(x.name) &&
        (value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
      ) {
        // Replace localhost with the LocalStack container hostname
        // When k3d and LocalStack are on the same Docker network, pods can reach LocalStack by container name
        value = value
          .replace('http://localhost', `http://${localstackHost}`)
          .replace('http://127.0.0.1', `http://${localstackHost}`);
        OrchestratorLogger.log(`Replaced localhost with ${localstackHost} for ${x.name}: ${value}`);
      }

      return { name: x.name, value } as OrchestratorEnvironmentVariable;
    });

    const job = new k8s.V1Job();
    job.apiVersion = 'batch/v1';
    job.kind = 'Job';
    job.metadata = {
      name: jobName,
      labels: {
        app: 'unity-builder',
        buildGuid,
      },
    };

    // Reduce TTL for tests to free up resources faster (default 9999s = ~2.8 hours)
    // For CI/test environments, use shorter TTL (300s = 5 minutes) to prevent disk pressure
    const jobTTL = process.env['orchestratorTests'] === 'true' ? 300 : 9999;
    job.spec = {
      ttlSecondsAfterFinished: jobTTL,
      backoffLimit: 0,
      template: {
        spec: {
          terminationGracePeriodSeconds: 90, // Give PreStopHook (60s sleep) time to complete
          volumes: [
            {
              name: 'build-mount',
              persistentVolumeClaim: {
                claimName: pvcName,
              },
            },
          ],
          containers: [
            {
              ttlSecondsAfterFinished: 9999,
              name: containerName,
              image,
              imagePullPolicy: process.env['orchestratorTests'] === 'true' ? 'IfNotPresent' : 'Always',
              command: ['/bin/sh'],
              args: [
                '-c',
                `${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, Orchestrator.buildParameters)}`,
              ],

              workingDir: `${workingDirectory}`,
              resources: {
                requests: (() => {
                  // Use smaller resource requests for lightweight hook containers
                  // Hook containers typically use utility images like aws-cli, rclone, etc.
                  const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu'];
                  const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage));

                  if (isLightweightContainer && process.env['orchestratorTests'] === 'true') {
                    // For test environments, use minimal resources for hook containers
                    return {
                      memory: '128Mi',
                      cpu: '100m', // 0.1 CPU
                    };
                  }

                  // For main build containers, use the configured resources
                  const memoryMB = Number.parseInt(buildParameters.containerMemory);
                  const cpuMB = Number.parseInt(buildParameters.containerCpu);

                  return {
                    memory: !Number.isNaN(memoryMB) && memoryMB > 0 ? `${memoryMB / 1024}G` : '750M',
                    cpu: !Number.isNaN(cpuMB) && cpuMB > 0 ? `${cpuMB / 1024}` : '1',
                  };
                })(),
              },
              env: [
                ...adjustedEnvironment.map((x) => {
                  const environmentVariable = new V1EnvVar();
                  environmentVariable.name = x.name;
                  environmentVariable.value = x.value;

                  return environmentVariable;
                }),
                ...secrets.map((x) => {
                  const secret = new V1EnvVarSource();
                  secret.secretKeyRef = new V1SecretKeySelector();
                  secret.secretKeyRef.key = x.ParameterKey;
                  secret.secretKeyRef.name = secretName;
                  const environmentVariable = new V1EnvVar();
                  environmentVariable.name = x.EnvironmentVariable;
                  environmentVariable.valueFrom = secret;

                  return environmentVariable;
                }),
                { name: 'LOG_SERVICE_IP', value: ip },
              ],
              volumeMounts: [
                {
                  name: 'build-mount',
                  mountPath: `${mountdir}`,
                },
              ],
              lifecycle: {
                preStop: {
                  exec: {
                    command: [
                      '/bin/sh',
                      '-c',
                      'sleep 60; cd /data/builder/action/steps && chmod +x /steps/return_license.sh 2>/dev/null || true; /steps/return_license.sh 2>/dev/null || true',
                    ],
                  },
                },
              },
            },
          ],
          restartPolicy: 'Never',

          // Add tolerations for CI/test environments to allow scheduling even with disk pressure
          // This is acceptable for CI where we aggressively clean up disk space
          tolerations: [
            {
              key: 'node.kubernetes.io/disk-pressure',
              operator: 'Exists',
              effect: 'NoSchedule',
            },
          ],
        },
      },
    };

    if (process.env['ORCHESTRATOR_MINIKUBE']) {
      job.spec.template.spec.volumes[0] = {
        name: 'build-mount',
        hostPath: {
          path: `/data`,
          type: `Directory`,
        },
      };
    }

    // Set ephemeral-storage request to a reasonable value to prevent evictions
    // For tests, don't set a request (or use minimal 128Mi) since k3d nodes have very limited disk space
    // Kubernetes will use whatever is available without a request, which is better for constrained environments
    // For production, use 2Gi to allow for larger builds
    // The node needs some free space headroom, so requesting too much causes evictions
    // With node at 96% usage and only ~2.7GB free, we can't request much without triggering evictions
    if (process.env['orchestratorTests'] !== 'true') {
      // Only set ephemeral-storage request for production builds
      job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '2Gi';
    }

    // For tests, don't set ephemeral-storage request - let Kubernetes use available space

    return job;
  }
}
export default KubernetesJobSpecFactory;