diff --git a/action.yml b/action.yml index 64b6ad32..34af2ba2 100644 --- a/action.yml +++ b/action.yml @@ -279,6 +279,95 @@ inputs: description: '[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or fixes.' + gcpProject: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Google Cloud project ID for Cloud Run Jobs provider. + Falls back to GOOGLE_CLOUD_PROJECT env var.' + gcpRegion: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Google Cloud region for Cloud Run Jobs (e.g. us-central1). + Defaults to the region input if empty.' + gcpBucket: + required: false + default: '' + description: + '[Orchestrator] [Experimental] GCS bucket name for build artifact storage. + Mounted via GCS FUSE for large filesystem support.' + gcpMachineType: + required: false + default: 'e2-standard-4' + description: + '[Orchestrator] [Experimental] Machine type for Cloud Run Jobs (e.g. e2-standard-4, e2-highmem-8).' + gcpDiskSizeGb: + required: false + default: '100' + description: + '[Orchestrator] [Experimental] Disk size in GB for Cloud Run Jobs. Supports up to 32GB + in-memory or unlimited via GCS FUSE bucket mount.' + gcpServiceAccount: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Google Cloud service account email for Cloud Run Jobs execution.' + gcpVpcConnector: + required: false + default: '' + description: + '[Orchestrator] [Experimental] VPC connector name for Cloud Run Jobs private networking.' + azureResourceGroup: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Azure resource group for Container Instances provider. + Falls back to AZURE_RESOURCE_GROUP env var.' + azureLocation: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Azure region for Container Instances (e.g. eastus, westeurope). + Defaults to the region input if empty.' + azureStorageAccount: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Azure Storage Account name for file share mounting. + Uses Premium FileStorage for high-throughput large artifact I/O.' + azureFileShareName: + required: false + default: 'unity-builds' + description: + '[Orchestrator] [Experimental] Azure File Share name within the storage account. + Supports up to 100 TiB per share.' + azureSubscriptionId: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Azure subscription ID. Falls back to AZURE_SUBSCRIPTION_ID env var.' + azureCpu: + required: false + default: '4' + description: + '[Orchestrator] [Experimental] CPU cores for Azure Container Instances (1-16).' + azureMemoryGb: + required: false + default: '16' + description: + '[Orchestrator] [Experimental] Memory in GB for Azure Container Instances (1-16).' + azureDiskSizeGb: + required: false + default: '100' + description: + '[Orchestrator] [Experimental] File share quota in GB for Azure Container Instances. + Premium shares support up to 102400 GB (100 TiB).' + azureSubnetId: + required: false + default: '' + description: + '[Orchestrator] [Experimental] Azure subnet resource ID for VNet-integrated Container Instances.' outputs: volume: diff --git a/src/model/build-parameters.ts b/src/model/build-parameters.ts index a338433b..22ecbc62 100644 --- a/src/model/build-parameters.ts +++ b/src/model/build-parameters.ts @@ -107,6 +107,26 @@ class BuildParameters { public unityHubVersionOnMac!: string; public dockerWorkspacePath!: string; + // GCP Cloud Run (Experimental) + public gcpProject!: string; + public gcpRegion!: string; + public gcpBucket!: string; + public gcpMachineType!: string; + public gcpDiskSizeGb!: string; + public gcpServiceAccount!: string; + public gcpVpcConnector!: string; + + // Azure Container Instances (Experimental) + public azureResourceGroup!: string; + public azureLocation!: string; + public azureStorageAccount!: string; + public azureFileShareName!: string; + public azureSubscriptionId!: string; + public azureCpu!: string; + public azureMemoryGb!: string; + public azureDiskSizeGb!: string; + public azureSubnetId!: string; + public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) { return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``; } @@ -228,6 +248,22 @@ class BuildParameters { inputPullCommand: OrchestratorOptions.inputPullCommand, pullInputList: OrchestratorOptions.pullInputList, kubeStorageClass: OrchestratorOptions.kubeStorageClass, + gcpProject: Input.gcpProject, + gcpRegion: Input.gcpRegion, + gcpBucket: Input.gcpBucket, + gcpMachineType: Input.gcpMachineType, + gcpDiskSizeGb: Input.gcpDiskSizeGb, + gcpServiceAccount: Input.gcpServiceAccount, + gcpVpcConnector: Input.gcpVpcConnector, + azureResourceGroup: Input.azureResourceGroup, + azureLocation: Input.azureLocation, + azureStorageAccount: Input.azureStorageAccount, + azureFileShareName: Input.azureFileShareName, + azureSubscriptionId: Input.azureSubscriptionId, + azureCpu: Input.azureCpu, + azureMemoryGb: Input.azureMemoryGb, + azureDiskSizeGb: Input.azureDiskSizeGb, + azureSubnetId: Input.azureSubnetId, cacheKey: OrchestratorOptions.cacheKey, maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces), useLargePackages: OrchestratorOptions.useLargePackages, diff --git a/src/model/input.ts b/src/model/input.ts index 51fda259..4459318d 100644 --- a/src/model/input.ts +++ b/src/model/input.ts @@ -282,6 +282,72 @@ class Input { return Input.getInput('skipActivation')?.toLowerCase() ?? 'false'; } + // GCP Cloud Run (Experimental) + static get gcpProject(): string { + return Input.getInput('gcpProject') ?? ''; + } + + static get gcpRegion(): string { + return Input.getInput('gcpRegion') ?? ''; + } + + static get gcpBucket(): string { + return Input.getInput('gcpBucket') ?? ''; + } + + static get gcpMachineType(): string { + return Input.getInput('gcpMachineType') ?? 'e2-standard-4'; + } + + static get gcpDiskSizeGb(): string { + return Input.getInput('gcpDiskSizeGb') ?? '100'; + } + + static get gcpServiceAccount(): string { + return Input.getInput('gcpServiceAccount') ?? ''; + } + + static get gcpVpcConnector(): string { + return Input.getInput('gcpVpcConnector') ?? ''; + } + + // Azure Container Instances (Experimental) + static get azureResourceGroup(): string { + return Input.getInput('azureResourceGroup') ?? ''; + } + + static get azureLocation(): string { + return Input.getInput('azureLocation') ?? ''; + } + + static get azureStorageAccount(): string { + return Input.getInput('azureStorageAccount') ?? ''; + } + + static get azureFileShareName(): string { + return Input.getInput('azureFileShareName') ?? 'unity-builds'; + } + + static get azureSubscriptionId(): string { + return Input.getInput('azureSubscriptionId') ?? ''; + } + + static get azureCpu(): string { + return Input.getInput('azureCpu') ?? '4'; + } + + static get azureMemoryGb(): string { + return Input.getInput('azureMemoryGb') ?? '16'; + } + + static get azureDiskSizeGb(): string { + return Input.getInput('azureDiskSizeGb') ?? '100'; + } + + static get azureSubnetId(): string { + return Input.getInput('azureSubnetId') ?? ''; + } + public static ToEnvVarFormat(input: string) { if (input.toUpperCase() === input) { return input; diff --git a/src/model/orchestrator/orchestrator.ts b/src/model/orchestrator/orchestrator.ts index ea1daa37..cec521bc 100644 --- a/src/model/orchestrator/orchestrator.ts +++ b/src/model/orchestrator/orchestrator.ts @@ -13,6 +13,8 @@ import OrchestratorEnvironmentVariable from './options/orchestrator-environment- import TestOrchestrator from './providers/test'; import LocalOrchestrator from './providers/local'; import LocalDockerOrchestrator from './providers/docker'; +import GcpCloudRunProvider from './providers/gcp-cloud-run'; +import AzureAciProvider from './providers/azure-aci'; import loadProvider from './providers/provider-loader'; import GitHub from '../github'; import SharedWorkspaceLocking from './services/core/shared-workspace-locking'; @@ -158,6 +160,14 @@ class Orchestrator { case 'local': Orchestrator.Provider = new LocalOrchestrator(); break; + case 'gcp-cloud-run': + OrchestratorLogger.log('⚠ EXPERIMENTAL: GCP Cloud Run Jobs provider'); + Orchestrator.Provider = new GcpCloudRunProvider(Orchestrator.buildParameters); + break; + case 'azure-aci': + OrchestratorLogger.log('⚠ EXPERIMENTAL: Azure Container Instances provider'); + Orchestrator.Provider = new AzureAciProvider(Orchestrator.buildParameters); + break; default: // Try to load provider using the dynamic loader for unknown providers try { diff --git a/src/model/orchestrator/providers/azure-aci/index.ts b/src/model/orchestrator/providers/azure-aci/index.ts new file mode 100644 index 00000000..9adbfdb9 --- /dev/null +++ b/src/model/orchestrator/providers/azure-aci/index.ts @@ -0,0 +1,405 @@ +/** + * Azure Container Instances (ACI) Provider (Experimental) + * + * Executes Unity builds as Azure Container Instances with Azure File Shares for large storage. + * + * Prerequisites: + * - Azure CLI authenticated (az login or service principal) + * - A resource group for build resources + * - An Azure Storage Account with a File Share for build artifacts + * - Contributor role on the resource group + * + * Architecture: + * - Uses Azure Container Instances for serverless container execution + * - Azure File Shares mounted as volumes for large artifact I/O (up to 100 TiB per share) + * - Container logs streamed via Azure Monitor / az container logs + * - Supports up to 16 CPU cores and 16 GB memory per container group + * - Premium file shares support up to 10 GiB/s throughput + * + * @experimental This provider is experimental. APIs and behavior may change. + */ + +import { ProviderInterface } from '../provider-interface'; +import BuildParameters from '../../../build-parameters'; +import OrchestratorLogger from '../../services/core/orchestrator-logger'; +import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable'; +import OrchestratorSecret from '../../options/orchestrator-secret'; +import { ProviderResource } from '../provider-resource'; +import { ProviderWorkflow } from '../provider-workflow'; +import { OrchestratorSystem } from '../../services/core/orchestrator-system'; +import { Input } from '../../..'; +import ResourceTracking from '../../services/core/resource-tracking'; + +class AzureAciProvider implements ProviderInterface { + private readonly resourceGroup: string; + private readonly location: string; + private readonly storageAccount: string; + private readonly fileShareName: string; + private readonly subscriptionId: string; + private readonly cpu: number; + private readonly memoryGb: number; + private readonly diskSizeGb: number; + private readonly subnetId: string; + private buildParameters: BuildParameters; + + constructor(buildParameters: BuildParameters) { + this.buildParameters = buildParameters; + this.resourceGroup = buildParameters.azureResourceGroup || process.env.AZURE_RESOURCE_GROUP || ''; + this.location = buildParameters.azureLocation || Input.region || 'eastus'; + this.storageAccount = buildParameters.azureStorageAccount || process.env.AZURE_STORAGE_ACCOUNT || ''; + this.fileShareName = buildParameters.azureFileShareName || 'unity-builds'; + this.subscriptionId = buildParameters.azureSubscriptionId || process.env.AZURE_SUBSCRIPTION_ID || ''; + this.cpu = Number.parseInt(buildParameters.azureCpu || '4', 10); + this.memoryGb = Number.parseInt(buildParameters.azureMemoryGb || '16', 10); + this.diskSizeGb = Number.parseInt(buildParameters.azureDiskSizeGb || '100', 10); + this.subnetId = buildParameters.azureSubnetId || ''; + + OrchestratorLogger.log('[Azure ACI] Provider initialized (EXPERIMENTAL)'); + OrchestratorLogger.log(`[Azure ACI] Resource Group: ${this.resourceGroup || '(not set)'}`); + OrchestratorLogger.log(`[Azure ACI] Location: ${this.location}`); + OrchestratorLogger.log(`[Azure ACI] Storage Account: ${this.storageAccount || '(not set)'}`); + OrchestratorLogger.log(`[Azure ACI] File Share: ${this.fileShareName}`); + OrchestratorLogger.log(`[Azure ACI] Resources: ${this.cpu} CPU, ${this.memoryGb}GB RAM`); + + if (!this.resourceGroup) { + OrchestratorLogger.logWarning( + '[Azure ACI] No resource group specified. Set azureResourceGroup input or AZURE_RESOURCE_GROUP env var.', + ); + } + } + + async setupWorkflow( + buildGuid: string, + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[], + ) { + OrchestratorLogger.log(`[Azure ACI] Setting up workflow for build ${buildGuid}`); + ResourceTracking.logAllocationSummary('azure-aci setup'); + + // Verify Azure CLI is available + try { + await OrchestratorSystem.Run('az version --output json', false, true); + OrchestratorLogger.log(`[Azure ACI] Azure CLI detected`); + } catch { + throw new Error( + '[Azure ACI] Azure CLI not found. Install Azure CLI: https://learn.microsoft.com/en-us/cli/azure/install-azure-cli', + ); + } + + // Set subscription if specified + if (this.subscriptionId) { + await OrchestratorSystem.Run(`az account set --subscription="${this.subscriptionId}"`); + } + + // Verify resource group exists + if (this.resourceGroup) { + try { + await OrchestratorSystem.Run( + `az group show --name "${this.resourceGroup}" --output json`, + false, + true, + ); + OrchestratorLogger.log(`[Azure ACI] Resource group ${this.resourceGroup} exists`); + } catch { + OrchestratorLogger.log(`[Azure ACI] Creating resource group ${this.resourceGroup}`); + await OrchestratorSystem.Run( + `az group create --name "${this.resourceGroup}" --location "${this.location}"`, + ); + } + } + + // Setup storage account and file share if specified + if (this.storageAccount) { + try { + await OrchestratorSystem.Run( + `az storage account show --name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + OrchestratorLogger.log(`[Azure ACI] Storage account ${this.storageAccount} exists`); + } catch { + OrchestratorLogger.log(`[Azure ACI] Creating storage account ${this.storageAccount}`); + await OrchestratorSystem.Run( + `az storage account create --name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --location "${this.location}" --sku Premium_LRS --kind FileStorage`, + ); + } + + // Get storage account key + const keyJson = await OrchestratorSystem.Run( + `az storage account keys list --account-name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + const keys = JSON.parse(keyJson); + const storageKey = keys[0]?.value || ''; + + // Create file share if it doesn't exist + try { + await OrchestratorSystem.Run( + `az storage share-rm show --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + } catch { + OrchestratorLogger.log(`[Azure ACI] Creating file share ${this.fileShareName}`); + await OrchestratorSystem.Run( + `az storage share-rm create --storage-account "${this.storageAccount}" --name "${this.fileShareName}" --resource-group "${this.resourceGroup}" --quota ${this.diskSizeGb}`, + ); + } + } + } + + async runTaskInWorkflow( + buildGuid: string, + image: string, + commands: string, + mountdir: string, + workingdir: string, + environment: OrchestratorEnvironmentVariable[], + secrets: OrchestratorSecret[], + ): Promise { + OrchestratorLogger.log(`[Azure ACI] Running task for build ${buildGuid}`); + ResourceTracking.logAllocationSummary('azure-aci task'); + + const containerName = `unity-build-${buildGuid}`.toLowerCase().replace(/[^a-z0-9-]/g, '-').slice(0, 63); + + // Build environment variable flags + const allEnvVars = [ + ...environment.map((env) => `${env.name}=${env.value}`), + ...secrets.map((s) => `${s.EnvironmentVariable}=${s.ParameterValue}`), + ]; + const envFlag = allEnvVars.length > 0 ? `--environment-variables ${allEnvVars.map((e) => `"${e}"`).join(' ')}` : ''; + + // Get storage account key for volume mount + let volumeFlags = ''; + if (this.storageAccount && this.resourceGroup) { + try { + const keyJson = await OrchestratorSystem.Run( + `az storage account keys list --account-name "${this.storageAccount}" --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + const keys = JSON.parse(keyJson); + const storageKey = keys[0]?.value || ''; + + if (storageKey) { + volumeFlags = [ + `--azure-file-volume-account-name "${this.storageAccount}"`, + `--azure-file-volume-account-key "${storageKey}"`, + `--azure-file-volume-share-name "${this.fileShareName}"`, + `--azure-file-volume-mount-path "${mountdir}"`, + ].join(' '); + } + } catch (error: any) { + OrchestratorLogger.logWarning(`[Azure ACI] Could not get storage key: ${error.message}`); + } + } + + // Subnet flag for VNet integration + const subnetFlag = this.subnetId ? `--subnet "${this.subnetId}"` : ''; + + // Build the command override + const commandFlag = commands ? `--command-line "/bin/sh -c '${commands.replace(/'/g, "'\\''")}'\"` : ''; + + // Create and run the container instance + const createCmd = [ + 'az container create', + `--resource-group "${this.resourceGroup}"`, + `--name "${containerName}"`, + `--image "${image}"`, + `--location "${this.location}"`, + `--cpu ${this.cpu}`, + `--memory ${this.memoryGb}`, + `--restart-policy Never`, + `--os-type Linux`, + volumeFlags, + envFlag, + subnetFlag, + commandFlag, + '--output json', + ] + .filter(Boolean) + .join(' '); + + try { + await OrchestratorSystem.Run(createCmd); + OrchestratorLogger.log(`[Azure ACI] Container ${containerName} created, waiting for completion...`); + } catch (error: any) { + throw new Error(`[Azure ACI] Failed to create container: ${error.message}`); + } + + // Poll for completion + const output = await this.waitForContainerCompletion(containerName); + + return output; + } + + private async waitForContainerCompletion(containerName: string): Promise { + const maxWaitMs = 24 * 60 * 60 * 1000; // 24 hours + const pollIntervalMs = 15_000; + const startTime = Date.now(); + let lastLogLength = 0; + + while (Date.now() - startTime < maxWaitMs) { + try { + // Check container state + const stateJson = await OrchestratorSystem.Run( + `az container show --resource-group "${this.resourceGroup}" --name "${containerName}" --output json`, + false, + true, + ); + + const state = JSON.parse(stateJson); + const containerState = + state.containers?.[0]?.instanceView?.currentState?.state || + state.instanceView?.state || + 'Unknown'; + const provisioningState = state.provisioningState || 'Unknown'; + + // Stream logs + try { + const logs = await OrchestratorSystem.Run( + `az container logs --resource-group "${this.resourceGroup}" --name "${containerName}"`, + false, + true, + ); + + if (logs && logs.length > lastLogLength) { + const newLogs = logs.slice(lastLogLength); + for (const line of newLogs.split('\n')) { + if (line.trim()) { + OrchestratorLogger.log(`[Build] ${line}`); + } + } + lastLogLength = logs.length; + } + } catch { + // Logs may not be available yet + } + + // Check if completed + if (containerState === 'Terminated' || provisioningState === 'Succeeded') { + const exitCode = state.containers?.[0]?.instanceView?.currentState?.exitCode; + if (exitCode !== undefined && exitCode !== 0) { + throw new Error(`[Azure ACI] Container exited with code ${exitCode}`); + } + OrchestratorLogger.log(`[Azure ACI] Container completed successfully`); + + // Get final logs + try { + return await OrchestratorSystem.Run( + `az container logs --resource-group "${this.resourceGroup}" --name "${containerName}"`, + false, + true, + ); + } catch { + return ''; + } + } + + if (provisioningState === 'Failed') { + const detail = + state.containers?.[0]?.instanceView?.currentState?.detailStatus || + state.containers?.[0]?.instanceView?.events?.map((e: any) => e.message).join('; ') || + 'Unknown error'; + throw new Error(`[Azure ACI] Container provisioning failed: ${detail}`); + } + } catch (error: any) { + if (error.message?.includes('Container provisioning failed') || error.message?.includes('exited with code')) { + throw error; + } + OrchestratorLogger.logWarning(`[Azure ACI] Polling error: ${error.message}`); + } + + // Wait before next poll + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } + + throw new Error('[Azure ACI] Container execution timed out after 24 hours'); + } + + async cleanupWorkflow( + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[], + ) { + OrchestratorLogger.log(`[Azure ACI] Cleaning up workflow`); + // ACI containers with restart-policy=Never auto-stop; cleanup is done during garbage collection + } + + async garbageCollect( + filter: string, + previewOnly: boolean, + olderThan: Number, + fullCache: boolean, + baseDependencies: boolean, + ): Promise { + OrchestratorLogger.log(`[Azure ACI] Garbage collecting old container groups`); + + try { + const containersJson = await OrchestratorSystem.Run( + `az container list --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + + const containers = JSON.parse(containersJson || '[]'); + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - Number(olderThan)); + + let deletedCount = 0; + for (const container of containers) { + const name = container.name || ''; + if (!name.startsWith('unity-build-')) continue; + + const createdAt = new Date(container.tags?.createdAt || container.properties?.provisioningState || 0); + const state = container.containers?.[0]?.instanceView?.currentState?.state || ''; + + // Delete terminated containers older than the threshold + if (state === 'Terminated' || createdAt < cutoffDate) { + if (previewOnly) { + OrchestratorLogger.log(`[Azure ACI] Would delete: ${name}`); + } else { + await OrchestratorSystem.Run( + `az container delete --resource-group "${this.resourceGroup}" --name "${name}" --yes`, + ); + deletedCount++; + } + } + } + + return `Garbage collected ${deletedCount} Azure container instances`; + } catch (error: any) { + OrchestratorLogger.logWarning(`[Azure ACI] Garbage collection failed: ${error.message}`); + return ''; + } + } + + async listResources(): Promise { + try { + const containersJson = await OrchestratorSystem.Run( + `az container list --resource-group "${this.resourceGroup}" --output json`, + false, + true, + ); + + const containers = JSON.parse(containersJson || '[]'); + return containers + .filter((c: any) => (c.name || '').startsWith('unity-build-')) + .map((c: any) => ({ Name: c.name || '' })); + } catch { + return []; + } + } + + listWorkflow(): Promise { + throw new Error('[Azure ACI] listWorkflow not implemented for this experimental provider'); + } + + async watchWorkflow(): Promise { + throw new Error('[Azure ACI] watchWorkflow not implemented for this experimental provider'); + } +} + +export default AzureAciProvider; diff --git a/src/model/orchestrator/providers/gcp-cloud-run/index.ts b/src/model/orchestrator/providers/gcp-cloud-run/index.ts new file mode 100644 index 00000000..e66d6909 --- /dev/null +++ b/src/model/orchestrator/providers/gcp-cloud-run/index.ts @@ -0,0 +1,331 @@ +/** + * Google Cloud Run Jobs Provider (Experimental) + * + * Executes Unity builds as Cloud Run Jobs with Cloud Storage (GCS) for large artifact storage. + * + * Prerequisites: + * - Google Cloud SDK authenticated (GOOGLE_APPLICATION_CREDENTIALS or gcloud auth) + * - Cloud Run Jobs API enabled + * - A GCS bucket for build artifacts + * - Service account with roles: Cloud Run Admin, Storage Admin, Logs Viewer + * + * Architecture: + * - Uses Cloud Run Jobs (not Services) for one-off build execution + * - GCS FUSE sidecar mounts a bucket as a local filesystem for large artifact I/O + * - Cloud Logging streams build output in real-time + * - Supports volumes up to 32 GiB in-memory or unlimited via GCS FUSE + * + * @experimental This provider is experimental. APIs and behavior may change. + */ + +import { ProviderInterface } from '../provider-interface'; +import BuildParameters from '../../../build-parameters'; +import OrchestratorLogger from '../../services/core/orchestrator-logger'; +import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable'; +import OrchestratorSecret from '../../options/orchestrator-secret'; +import { ProviderResource } from '../provider-resource'; +import { ProviderWorkflow } from '../provider-workflow'; +import { OrchestratorSystem } from '../../services/core/orchestrator-system'; +import { Input } from '../../..'; +import ResourceTracking from '../../services/core/resource-tracking'; + +class GcpCloudRunProvider implements ProviderInterface { + private readonly project: string; + private readonly region: string; + private readonly bucket: string; + private readonly machineType: string; + private readonly diskSizeGb: number; + private readonly serviceAccount: string; + private readonly vpcConnector: string; + private buildParameters: BuildParameters; + + constructor(buildParameters: BuildParameters) { + this.buildParameters = buildParameters; + this.project = buildParameters.gcpProject || process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT || ''; + this.region = buildParameters.gcpRegion || Input.region || 'us-central1'; + this.bucket = buildParameters.gcpBucket || ''; + this.machineType = buildParameters.gcpMachineType || 'e2-standard-4'; + this.diskSizeGb = Number.parseInt(buildParameters.gcpDiskSizeGb || '100', 10); + this.serviceAccount = buildParameters.gcpServiceAccount || ''; + this.vpcConnector = buildParameters.gcpVpcConnector || ''; + + OrchestratorLogger.log('[GCP Cloud Run] Provider initialized (EXPERIMENTAL)'); + OrchestratorLogger.log(`[GCP Cloud Run] Project: ${this.project || '(auto-detect)'}`); + OrchestratorLogger.log(`[GCP Cloud Run] Region: ${this.region}`); + OrchestratorLogger.log(`[GCP Cloud Run] Bucket: ${this.bucket || '(none)'}`); + OrchestratorLogger.log(`[GCP Cloud Run] Disk size: ${this.diskSizeGb}GB`); + + if (!this.project) { + OrchestratorLogger.logWarning( + '[GCP Cloud Run] No project specified. Set gcpProject input or GOOGLE_CLOUD_PROJECT env var.', + ); + } + } + + async setupWorkflow( + buildGuid: string, + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[], + ) { + OrchestratorLogger.log(`[GCP Cloud Run] Setting up workflow for build ${buildGuid}`); + ResourceTracking.logAllocationSummary('gcp-cloud-run setup'); + + // Verify gcloud CLI is available + try { + const version = await OrchestratorSystem.Run('gcloud --version', false, true); + OrchestratorLogger.log(`[GCP Cloud Run] gcloud CLI detected`); + } catch { + throw new Error( + '[GCP Cloud Run] gcloud CLI not found. Install Google Cloud SDK: https://cloud.google.com/sdk/docs/install', + ); + } + + // Verify Cloud Run Jobs API is enabled + try { + const projectFlag = this.project ? `--project=${this.project}` : ''; + await OrchestratorSystem.Run( + `gcloud services list --enabled --filter="name:run.googleapis.com" ${projectFlag} --format="value(name)"`, + false, + true, + ); + } catch (error) { + OrchestratorLogger.logWarning( + `[GCP Cloud Run] Could not verify Cloud Run API status. Ensure run.googleapis.com is enabled.`, + ); + } + + // Create GCS bucket for artifacts if specified and doesn't exist + if (this.bucket) { + try { + await OrchestratorSystem.Run(`gcloud storage buckets describe gs://${this.bucket} --format="value(name)"`, false, true); + OrchestratorLogger.log(`[GCP Cloud Run] Bucket gs://${this.bucket} exists`); + } catch { + OrchestratorLogger.log(`[GCP Cloud Run] Creating bucket gs://${this.bucket}`); + const projectFlag = this.project ? `--project=${this.project}` : ''; + await OrchestratorSystem.Run( + `gcloud storage buckets create gs://${this.bucket} --location=${this.region} ${projectFlag}`, + ); + } + } + } + + async runTaskInWorkflow( + buildGuid: string, + image: string, + commands: string, + mountdir: string, + workingdir: string, + environment: OrchestratorEnvironmentVariable[], + secrets: OrchestratorSecret[], + ): Promise { + OrchestratorLogger.log(`[GCP Cloud Run] Running task for build ${buildGuid}`); + ResourceTracking.logAllocationSummary('gcp-cloud-run task'); + + const jobName = `unity-build-${buildGuid}`.toLowerCase().replace(/[^a-z0-9-]/g, '-').slice(0, 63); + const projectFlag = this.project ? `--project=${this.project}` : ''; + + // Build environment variable flags + const envFlags = environment + .map((env) => `${env.name}=${env.value}`) + .concat(secrets.map((s) => `${s.EnvironmentVariable}=${s.ParameterValue}`)); + + const envString = envFlags.length > 0 ? `--set-env-vars="${envFlags.join(',')}"` : ''; + + // Build volume and mount flags for GCS FUSE + let volumeFlags = ''; + let mountFlags = ''; + if (this.bucket) { + volumeFlags = `--add-volume=name=gcs-fuse,type=cloud-storage,bucket=${this.bucket}`; + mountFlags = `--add-volume-mount=volume=gcs-fuse,mount-path=${mountdir}`; + } + + // Service account flag + const saFlag = this.serviceAccount ? `--service-account=${this.serviceAccount}` : ''; + + // VPC connector for private networking + const vpcFlag = this.vpcConnector ? `--vpc-connector=${this.vpcConnector}` : ''; + + // Create the Cloud Run Job + const createCmd = [ + 'gcloud run jobs create', + jobName, + `--image=${image}`, + `--region=${this.region}`, + `--task-timeout=86400s`, + `--max-retries=0`, + `--cpu=4`, + `--memory=16Gi`, + volumeFlags, + mountFlags, + envString, + saFlag, + vpcFlag, + projectFlag, + '--format=json', + '--quiet', + ] + .filter(Boolean) + .join(' '); + + try { + await OrchestratorSystem.Run(createCmd); + OrchestratorLogger.log(`[GCP Cloud Run] Job ${jobName} created`); + } catch (error: any) { + // Job might already exist from a retry + if (error.message?.includes('already exists')) { + OrchestratorLogger.log(`[GCP Cloud Run] Job ${jobName} already exists, updating...`); + const updateCmd = createCmd.replace('jobs create', 'jobs update'); + await OrchestratorSystem.Run(updateCmd); + } else { + throw error; + } + } + + // Override the command if provided + if (commands) { + const updateCmd = [ + 'gcloud run jobs update', + jobName, + `--region=${this.region}`, + `--command="/bin/sh"`, + `--args="-c,${commands}"`, + projectFlag, + '--quiet', + ] + .filter(Boolean) + .join(' '); + + await OrchestratorSystem.Run(updateCmd); + } + + // Execute the job + OrchestratorLogger.log(`[GCP Cloud Run] Executing job ${jobName}...`); + const executeCmd = [ + 'gcloud run jobs execute', + jobName, + `--region=${this.region}`, + projectFlag, + '--wait', + '--format=json', + '--quiet', + ] + .filter(Boolean) + .join(' '); + + let output = ''; + try { + output = await OrchestratorSystem.Run(executeCmd); + OrchestratorLogger.log(`[GCP Cloud Run] Job execution completed`); + } catch (error: any) { + // Try to get logs even on failure + await this.streamJobLogs(jobName); + throw new Error(`[GCP Cloud Run] Job execution failed: ${error.message}`); + } + + // Stream logs + await this.streamJobLogs(jobName); + + return output; + } + + private async streamJobLogs(jobName: string): Promise { + const projectFlag = this.project ? `--project=${this.project}` : ''; + try { + const logs = await OrchestratorSystem.Run( + `gcloud logging read "resource.type=cloud_run_job AND resource.labels.job_name=${jobName}" ${projectFlag} --limit=1000 --format="value(textPayload)" --order=asc`, + false, + true, + ); + if (logs) { + for (const line of logs.split('\n')) { + if (line.trim()) { + OrchestratorLogger.log(`[Build] ${line}`); + } + } + } + } catch { + OrchestratorLogger.logWarning(`[GCP Cloud Run] Could not retrieve job logs`); + } + } + + async cleanupWorkflow( + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[], + ) { + OrchestratorLogger.log(`[GCP Cloud Run] Cleaning up workflow`); + // Cloud Run Jobs auto-cleanup after execution; explicit delete is optional + } + + async garbageCollect( + filter: string, + previewOnly: boolean, + olderThan: Number, + fullCache: boolean, + baseDependencies: boolean, + ): Promise { + OrchestratorLogger.log(`[GCP Cloud Run] Garbage collecting old jobs`); + const projectFlag = this.project ? `--project=${this.project}` : ''; + + try { + // List old jobs matching the unity-build prefix + const jobsJson = await OrchestratorSystem.Run( + `gcloud run jobs list --region=${this.region} ${projectFlag} --filter="metadata.name~unity-build-" --format="json(metadata.name,metadata.creationTimestamp)"`, + false, + true, + ); + + const jobs = JSON.parse(jobsJson || '[]'); + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - Number(olderThan)); + + let deletedCount = 0; + for (const job of jobs) { + const createdAt = new Date(job.metadata?.creationTimestamp || 0); + if (createdAt < cutoffDate) { + const name = job.metadata?.name; + if (previewOnly) { + OrchestratorLogger.log(`[GCP Cloud Run] Would delete: ${name}`); + } else { + await OrchestratorSystem.Run( + `gcloud run jobs delete ${name} --region=${this.region} ${projectFlag} --quiet`, + ); + deletedCount++; + } + } + } + + return `Garbage collected ${deletedCount} Cloud Run jobs`; + } catch (error: any) { + OrchestratorLogger.logWarning(`[GCP Cloud Run] Garbage collection failed: ${error.message}`); + return ''; + } + } + + async listResources(): Promise { + const projectFlag = this.project ? `--project=${this.project}` : ''; + try { + const jobsJson = await OrchestratorSystem.Run( + `gcloud run jobs list --region=${this.region} ${projectFlag} --filter="metadata.name~unity-build-" --format="json(metadata.name)"`, + false, + true, + ); + + const jobs = JSON.parse(jobsJson || '[]'); + return jobs.map((job: any) => ({ Name: job.metadata?.name || '' })); + } catch { + return []; + } + } + + listWorkflow(): Promise { + throw new Error('[GCP Cloud Run] listWorkflow not implemented for this experimental provider'); + } + + async watchWorkflow(): Promise { + throw new Error('[GCP Cloud Run] watchWorkflow not implemented for this experimental provider'); + } +} + +export default GcpCloudRunProvider; diff --git a/src/model/orchestrator/providers/provider-loader.ts b/src/model/orchestrator/providers/provider-loader.ts index e940522e..c2b44f25 100644 --- a/src/model/orchestrator/providers/provider-loader.ts +++ b/src/model/orchestrator/providers/provider-loader.ts @@ -62,6 +62,8 @@ export default async function loadProvider( 'local-docker': './docker', 'local-system': './local', local: './local', + 'gcp-cloud-run': './gcp-cloud-run', + 'azure-aci': './azure-aci', }; modulePath = providerModuleMap[providerSource] || providerSource; @@ -136,7 +138,7 @@ export class ProviderLoader { * @returns string[] - Array of available provider names */ static getAvailableProviders(): string[] { - return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local']; + return ['aws', 'k8s', 'test', 'local-docker', 'local-system', 'local', 'gcp-cloud-run', 'azure-aci']; } /**