Rename Cloud Runner to Orchestrator (#775)

* Rename "Cloud Runner" to "Orchestrator" across entire codebase

Breaking change: All CloudRunner classes, options, environment variables,
and action.yml inputs have been renamed to Orchestrator equivalents.

- Renamed src/model/cloud-runner/ directory to src/model/orchestrator/
- Renamed all cloud-runner-* files to orchestrator-*
- Renamed all CloudRunner* classes to Orchestrator* (15+ classes)
- Renamed all cloudRunner* properties to orchestrator* equivalents
- Renamed CLOUD_RUNNER_* env vars to ORCHESTRATOR_*
- Updated action.yml [CloudRunner] markers to [Orchestrator]
- Updated workflow files and package.json test scripts
- Updated all runtime strings (cache paths, log messages, branch refs)
- Rebuilt dist/index.js

No backward compatibility layer is provided.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Remove tracked log/temp files and add to .gitignore

Remove $LOG_FILE and temp/job-log.txt debug artifacts that should
not be in the repository.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frostebite
2026-03-04 21:53:47 +00:00
committed by GitHub
parent f3849ee1c9
commit 9d475434d3
117 changed files with 4756 additions and 4745 deletions

View File

@@ -32,7 +32,7 @@ jobs:
- run: yarn build || { echo "build command should always succeed" ; exit 61; } - run: yarn build || { echo "build command should always succeed" ; exit 61; }
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; } # - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
cloud-runner: orchestrator:
name: Cloud Runner Integrity name: Orchestrator Integrity
uses: ./.github/workflows/cloud-runner-integrity.yml uses: ./.github/workflows/orchestrator-integrity.yml
secrets: inherit secrets: inherit

View File

@@ -18,16 +18,16 @@ env:
GKE_CLUSTER: 'game-ci-github-pipelines' GKE_CLUSTER: 'game-ci-github-pipelines'
GCP_LOGGING: true GCP_LOGGING: true
GCP_PROJECT: unitykubernetesbuilder GCP_PROJECT: unitykubernetesbuilder
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt GCP_LOG_FILE: ${{ github.workspace }}/orchestrator-logs.txt
# Commented out: Using LocalStack tests instead of real AWS # Commented out: Using LocalStack tests instead of real AWS
# AWS_REGION: eu-west-2 # AWS_REGION: eu-west-2
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# AWS_DEFAULT_REGION: eu-west-2 # AWS_DEFAULT_REGION: eu-west-2
# AWS_STACK_NAME: game-ci-github-pipelines # AWS_STACK_NAME: game-ci-github-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }} ORCHESTRATOR_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true ORCHESTRATOR_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true ORCHESTRATOR_DEBUG_TREE: true
DEBUG: true DEBUG: true
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
@@ -47,14 +47,14 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
CLOUD_RUNNER_CLUSTER: local-docker ORCHESTRATOR_CLUSTER: local-docker
# Commented out: Using LocalStack tests instead of real AWS # Commented out: Using LocalStack tests instead of real AWS
# AWS_STACK_NAME: game-ci-github-pipelines # AWS_STACK_NAME: game-ci-github-pipelines
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }} CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
run: | run: |
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder git clone -b orchestrator-develop https://github.com/game-ci/unity-builder
cd unity-builder cd unity-builder
yarn yarn
ls ls

View File

@@ -1,4 +1,4 @@
name: cloud-runner-integrity name: orchestrator-integrity
on: on:
workflow_call: workflow_call:
@@ -19,16 +19,16 @@ env:
# AWS_REGION: eu-west-2 # AWS_REGION: eu-west-2
# AWS_DEFAULT_REGION: eu-west-2 # AWS_DEFAULT_REGION: eu-west-2
AWS_STACK_NAME: game-ci-team-pipelines # Still needed for LocalStack S3 bucket creation AWS_STACK_NAME: game-ci-team-pipelines # Still needed for LocalStack S3 bucket creation
CLOUD_RUNNER_BRANCH: ${{ github.ref }} ORCHESTRATOR_BRANCH: ${{ github.ref }}
DEBUG: true DEBUG: true
PROJECT_PATH: test-project PROJECT_PATH: test-project
USE_IL2CPP: false USE_IL2CPP: false
# Increase CloudFormation stack wait time (GitHub Actions runners can be slow) # Increase CloudFormation stack wait time (GitHub Actions runners can be slow)
CLOUD_RUNNER_AWS_STACK_WAIT_TIME: 900 ORCHESTRATOR_AWS_STACK_WAIT_TIME: 900
jobs: jobs:
cloud-runner-tests: orchestrator-tests:
name: Cloud Runner Integrity Tests name: Orchestrator Integrity Tests
runs-on: ubuntu-latest runs-on: ubuntu-latest
env: env:
K3D_NODE_CONTAINERS: 'k3d-unity-builder-agent-0' K3D_NODE_CONTAINERS: 'k3d-unity-builder-agent-0'
@@ -71,8 +71,8 @@ jobs:
docker image prune -af || true docker image prune -af || true
docker volume prune -f || true docker volume prune -f || true
# Create a shared network for k3d and LocalStack # Create a shared network for k3d and LocalStack
docker network rm cloud-runner-net 2>/dev/null || true docker network rm orchestrator-net 2>/dev/null || true
docker network create cloud-runner-net || true docker network create orchestrator-net || true
echo "Disk usage after cleanup:" echo "Disk usage after cleanup:"
df -h df -h
- name: Start LocalStack (S3) as managed Docker container - name: Start LocalStack (S3) as managed Docker container
@@ -85,7 +85,7 @@ jobs:
# Use host networking alias so k3d pods can reach it # Use host networking alias so k3d pods can reach it
docker run -d \ docker run -d \
--name localstack-main \ --name localstack-main \
--network cloud-runner-net \ --network orchestrator-net \
--add-host=host.docker.internal:host-gateway \ --add-host=host.docker.internal:host-gateway \
-p 4566:4566 \ -p 4566:4566 \
-e SERVICES=s3,cloudformation,ecs,kinesis,cloudwatch,logs,efs,ec2,iam,elasticfilesystem,secretsmanager,lambda,events,sts \ -e SERVICES=s3,cloudformation,ecs,kinesis,cloudwatch,logs,efs,ec2,iam,elasticfilesystem,secretsmanager,lambda,events,sts \
@@ -201,7 +201,7 @@ jobs:
- name: Clean up disk space before K8s tests - name: Clean up disk space before K8s tests
run: | run: |
echo "Cleaning up disk space before K8s tests..." echo "Cleaning up disk space before K8s tests..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
sudo apt-get clean || true sudo apt-get clean || true
docker system prune -f || true docker system prune -f || true
df -h df -h
@@ -215,7 +215,7 @@ jobs:
# This allows pods to access LocalStack directly by container name or IP # This allows pods to access LocalStack directly by container name or IP
k3d cluster create unity-builder \ k3d cluster create unity-builder \
--agents 1 \ --agents 1 \
--network cloud-runner-net \ --network orchestrator-net \
--wait --wait
kubectl config current-context | cat kubectl config current-context | cat
# Store LocalStack IP for later use in tests # Store LocalStack IP for later use in tests
@@ -243,7 +243,7 @@ jobs:
echo "From host via localhost (should work):" echo "From host via localhost (should work):"
curl -s --max-time 5 http://localhost:4566/_localstack/health | head -5 || echo "Host connectivity failed" curl -s --max-time 5 http://localhost:4566/_localstack/health | head -5 || echo "Host connectivity failed"
echo "From host via container name (should work on shared network):" echo "From host via container name (should work on shared network):"
docker run --rm --network cloud-runner-net curlimages/curl \ docker run --rm --network orchestrator-net curlimages/curl \
curl -s --max-time 5 http://localstack-main:4566/_localstack/health 2>&1 | head -5 || echo "Container network test failed" curl -s --max-time 5 http://localstack-main:4566/_localstack/health 2>&1 | head -5 || echo "Container network test failed"
echo "From k3d cluster via LocalStack container IP ($LOCALSTACK_IP):" echo "From k3d cluster via LocalStack container IP ($LOCALSTACK_IP):"
kubectl run test-localstack --image=curlimages/curl --rm -i --restart=Never --timeout=30s -- \ kubectl run test-localstack --image=curlimages/curl --rm -i --restart=Never --timeout=30s -- \
@@ -278,16 +278,16 @@ jobs:
done || true done || true
sleep 3 sleep 3
docker system prune -f || true docker system prune -f || true
- name: Run cloud-runner-image test (K8s) - name: Run orchestrator-image test (K8s)
timeout-minutes: 10 timeout-minutes: 10
run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s PROVIDER_STRATEGY: k8s
@@ -296,10 +296,10 @@ jobs:
containerMemory: '512' containerMemory: '512'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up after cloud-runner-image test - name: Clean up after orchestrator-image test
if: always() if: always()
run: | run: |
echo "Cleaning up after cloud-runner-image test..." echo "Cleaning up after orchestrator-image test..."
kubectl delete jobs --all --ignore-not-found=true -n default || true kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true kubectl delete "$pod" --ignore-not-found=true || true
@@ -320,18 +320,18 @@ jobs:
# Clean up unused layers # Clean up unused layers
docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
- name: Run cloud-runner-kubernetes test - name: Run orchestrator-kubernetes test
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-kubernetes" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-kubernetes" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s PROVIDER_STRATEGY: k8s
@@ -348,10 +348,10 @@ jobs:
AWS_EC2_METADATA_DISABLED: 'true' AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up after cloud-runner-kubernetes test - name: Clean up after orchestrator-kubernetes test
if: always() if: always()
run: | run: |
echo "Cleaning up after cloud-runner-kubernetes test..." echo "Cleaning up after orchestrator-kubernetes test..."
kubectl delete jobs --all --ignore-not-found=true -n default || true kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true kubectl delete "$pod" --ignore-not-found=true || true
@@ -372,18 +372,18 @@ jobs:
# Clean up unused layers # Clean up unused layers
docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
- name: Run cloud-runner-s3-steps test (K8s) - name: Run orchestrator-s3-steps test (K8s)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-s3-steps" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s PROVIDER_STRATEGY: k8s
@@ -400,10 +400,10 @@ jobs:
AWS_EC2_METADATA_DISABLED: 'true' AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up after cloud-runner-s3-steps test - name: Clean up after orchestrator-s3-steps test
if: always() if: always()
run: | run: |
echo "Cleaning up after cloud-runner-s3-steps test..." echo "Cleaning up after orchestrator-s3-steps test..."
kubectl delete jobs --all --ignore-not-found=true -n default || true kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true kubectl delete "$pod" --ignore-not-found=true || true
@@ -424,18 +424,18 @@ jobs:
# Clean up unused layers # Clean up unused layers
docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
- name: Run cloud-runner-end2end-caching test (K8s) - name: Run orchestrator-end2end-caching test (K8s)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s PROVIDER_STRATEGY: k8s
@@ -452,10 +452,10 @@ jobs:
AWS_EC2_METADATA_DISABLED: 'true' AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up after cloud-runner-end2end-caching test - name: Clean up after orchestrator-end2end-caching test
if: always() if: always()
run: | run: |
echo "Cleaning up after cloud-runner-end2end-caching test..." echo "Cleaning up after orchestrator-end2end-caching test..."
kubectl delete jobs --all --ignore-not-found=true -n default || true kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true kubectl delete "$pod" --ignore-not-found=true || true
@@ -476,7 +476,7 @@ jobs:
# Clean up unused layers # Clean up unused layers
docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
- name: Clean up disk space before end2end-retaining test - name: Clean up disk space before end2end-retaining test
run: | run: |
@@ -495,20 +495,20 @@ jobs:
# Clean up unused layers # Clean up unused layers
docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
echo "Disk usage before end2end-retaining test:" echo "Disk usage before end2end-retaining test:"
df -h df -h
- name: Run cloud-runner-end2end-retaining test (K8s) - name: Run orchestrator-end2end-retaining test (K8s)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s PROVIDER_STRATEGY: k8s
@@ -551,7 +551,7 @@ jobs:
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true kubectl delete "$secret" --ignore-not-found=true || true
done || true done || true
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -af --volumes || true docker system prune -af --volumes || true
# Aggressive cleanup in k3d nodes to free ephemeral storage, but preserve Unity images # Aggressive cleanup in k3d nodes to free ephemeral storage, but preserve Unity images
K3D_NODE_CONTAINERS="${K3D_NODE_CONTAINERS:-k3d-unity-builder-agent-0 k3d-unity-builder-server-0}" K3D_NODE_CONTAINERS="${K3D_NODE_CONTAINERS:-k3d-unity-builder-agent-0 k3d-unity-builder-server-0}"
@@ -582,21 +582,21 @@ jobs:
- name: Clean up disk space before AWS/LocalStack provider tests - name: Clean up disk space before AWS/LocalStack provider tests
run: | run: |
echo "Cleaning up disk space before AWS/LocalStack provider tests..." echo "Cleaning up disk space before AWS/LocalStack provider tests..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
sudo apt-get clean || true sudo apt-get clean || true
docker system prune -af --volumes || true docker system prune -af --volumes || true
echo "Disk usage:" echo "Disk usage:"
df -h df -h
- name: Run cloud-runner-image test (AWS provider) - name: Run orchestrator-image test (AWS provider)
timeout-minutes: 10 timeout-minutes: 10
run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -608,19 +608,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-environment test (AWS provider) - name: Run orchestrator-environment test (AWS provider)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-environment" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-environment" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -632,19 +632,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-s3-steps test (AWS provider) - name: Run orchestrator-s3-steps test (AWS provider)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-s3-steps" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -656,19 +656,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-hooks test (AWS provider) - name: Run orchestrator-hooks test (AWS provider)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-hooks" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -680,19 +680,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-end2end-caching test (AWS provider) - name: Run orchestrator-end2end-caching test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -704,19 +704,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-end2end-retaining test (AWS provider) - name: Run orchestrator-end2end-retaining test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -728,19 +728,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-caching test (AWS provider) - name: Run orchestrator-caching test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-caching" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-caching" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -752,19 +752,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-locking-core test (AWS provider) - name: Run orchestrator-locking-core test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-locking-core" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-locking-core" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -776,19 +776,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-locking-get-locked test (AWS provider) - name: Run orchestrator-locking-get-locked test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-locking-get-locked" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-locking-get-locked" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -800,19 +800,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-end2end-locking test (AWS provider) - name: Run orchestrator-end2end-locking test (AWS provider)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-locking" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-locking" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
KUBE_STORAGE_CLASS: local-path KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws PROVIDER_STRATEGY: aws
@@ -825,7 +825,7 @@ jobs:
- name: Clean up disk space after AWS/LocalStack provider tests - name: Clean up disk space after AWS/LocalStack provider tests
run: | run: |
echo "Cleaning up disk space after AWS/LocalStack provider tests..." echo "Cleaning up disk space after AWS/LocalStack provider tests..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -af --volumes || true docker system prune -af --volumes || true
echo "Disk usage:" echo "Disk usage:"
df -h df -h
@@ -857,16 +857,16 @@ jobs:
rclone lsd localstack-s3: || echo "No buckets yet (expected)" rclone lsd localstack-s3: || echo "No buckets yet (expected)"
rclone ls localstack-s3:game-ci-team-pipelines || echo "Bucket may be empty" rclone ls localstack-s3:game-ci-team-pipelines || echo "Bucket may be empty"
echo "Rclone configured successfully" echo "Rclone configured successfully"
- name: Run cloud-runner-rclone-steps test (rclone with LocalStack S3) - name: Run orchestrator-rclone-steps test (rclone with LocalStack S3)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-rclone-steps" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-rclone-steps" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
RCLONE_REMOTE: 'localstack-s3:game-ci-team-pipelines' RCLONE_REMOTE: 'localstack-s3:game-ci-team-pipelines'
@@ -876,7 +876,7 @@ jobs:
- name: Clean up disk space after rclone tests - name: Clean up disk space after rclone tests
run: | run: |
echo "Cleaning up disk space after rclone tests..." echo "Cleaning up disk space after rclone tests..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
echo "Disk usage:" echo "Disk usage:"
df -h df -h
@@ -887,78 +887,78 @@ jobs:
- name: Clean up disk space before local-docker tests - name: Clean up disk space before local-docker tests
run: | run: |
echo "Cleaning up disk space before local-docker tests..." echo "Cleaning up disk space before local-docker tests..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
sudo apt-get clean || true sudo apt-get clean || true
docker system prune -af --volumes || true docker system prune -af --volumes || true
echo "Disk usage:" echo "Disk usage:"
df -h df -h
- name: Run cloud-runner-image test (local-docker) - name: Run orchestrator-image test (local-docker)
timeout-minutes: 10 timeout-minutes: 10
run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-hooks test (local-docker) - name: Run orchestrator-hooks test (local-docker)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-hooks" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-local-persistence test - name: Run orchestrator-local-persistence test
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-local-persistence" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-local-persistence" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-locking-core test (local-docker with S3) - name: Run orchestrator-locking-core test (local-docker with S3)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-locking-core" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-locking-core" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
AWS_STACK_NAME: game-ci-team-pipelines AWS_STACK_NAME: game-ci-team-pipelines
@@ -975,19 +975,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-locking-get-locked test (local-docker with S3) - name: Run orchestrator-locking-get-locked test (local-docker with S3)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-locking-get-locked" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-locking-get-locked" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
AWS_STACK_NAME: game-ci-team-pipelines AWS_STACK_NAME: game-ci-team-pipelines
@@ -1004,57 +1004,57 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-caching test (local-docker) - name: Run orchestrator-caching test (local-docker)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-caching" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-caching" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-github-checks test (local-docker) - name: Run orchestrator-github-checks test (local-docker)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-github-checks" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-github-checks" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64 TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-s3-steps test (local-docker with S3) - name: Run orchestrator-s3-steps test (local-docker with S3)
timeout-minutes: 30 timeout-minutes: 30
run: yarn run test "cloud-runner-s3-steps" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
AWS_STACK_NAME: game-ci-team-pipelines AWS_STACK_NAME: game-ci-team-pipelines
@@ -1071,19 +1071,19 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space - name: Clean up disk space
run: | run: |
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker system prune -f || true docker system prune -f || true
df -h df -h
- name: Run cloud-runner-end2end-caching test (local-docker with S3) - name: Run orchestrator-end2end-caching test (local-docker with S3)
timeout-minutes: 60 timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneLinux64 TARGET_PLATFORM: StandaloneLinux64
cloudRunnerTests: true orchestratorTests: true
versioning: None versioning: None
PROVIDER_STRATEGY: local-docker PROVIDER_STRATEGY: local-docker
AWS_STACK_NAME: game-ci-team-pipelines AWS_STACK_NAME: game-ci-team-pipelines
@@ -1101,7 +1101,7 @@ jobs:
- name: Final disk space cleanup - name: Final disk space cleanup
run: | run: |
echo "Final disk space cleanup..." echo "Final disk space cleanup..."
rm -rf ./cloud-runner-cache/* || true rm -rf ./orchestrator-cache/* || true
docker stop localstack-main 2>/dev/null || true docker stop localstack-main 2>/dev/null || true
docker rm localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true
docker system prune -af --volumes || true docker system prune -af --volumes || true

2
.gitignore vendored
View File

@@ -5,3 +5,5 @@ lib/
.vsconfig .vsconfig
yarn-error.log yarn-error.log
.orig .orig
$LOG_FILE
temp/

View File

@@ -104,11 +104,11 @@ inputs:
gitPrivateToken: gitPrivateToken:
required: false required: false
default: '' default: ''
description: '[CloudRunner] Github private token to pull from github' description: '[Orchestrator] Github private token to pull from github'
githubOwner: githubOwner:
required: false required: false
default: '' default: ''
description: '[CloudRunner] GitHub owner name or organization/team name' description: '[Orchestrator] GitHub owner name or organization/team name'
runAsHostUser: runAsHostUser:
required: false required: false
default: 'false' default: 'false'
@@ -149,101 +149,101 @@ inputs:
allowDirtyBuild: allowDirtyBuild:
required: false required: false
default: '' default: ''
description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.' description: '[Orchestrator] Allows the branch of the build to be dirty, and still generate the build.'
postBuildSteps: postBuildSteps:
required: false required: false
default: '' default: ''
description: description:
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array), '[Orchestrator] run a post build job in yaml format with the keys image, secrets (name, value object array),
command string' command string'
preBuildSteps: preBuildSteps:
required: false required: false
default: '' default: ''
description: description:
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the '[Orchestrator] Run a pre build job after the repository setup but before the build job (in yaml format with the
keys image, secrets (name, value object array), command line string)' keys image, secrets (name, value object array), command line string)'
containerHookFiles: containerHookFiles:
required: false required: false
default: '' default: ''
description: description:
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must '[Orchestrator] Specify the names (by file name) of custom steps to run before or after orchestrator jobs, must
match a yaml step file inside your repo in the folder .game-ci/steps/' match a yaml step file inside your repo in the folder .game-ci/steps/'
customHookFiles: customHookFiles:
required: false required: false
default: '' default: ''
description: description:
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must '[Orchestrator] Specify the names (by file name) of custom hooks to run before or after orchestrator jobs, must
match a yaml step file inside your repo in the folder .game-ci/hooks/' match a yaml step file inside your repo in the folder .game-ci/hooks/'
customCommandHooks: customCommandHooks:
required: false required: false
default: '' default: ''
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)' description: '[Orchestrator] Specify custom commands and trigger hooks (injects commands into jobs)'
customJob: customJob:
required: false required: false
default: '' default: ''
description: description:
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the '[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
keys image, secrets (name, value object array), command line string)' keys image, secrets (name, value object array), command line string)'
awsStackName: awsStackName:
default: 'game-ci' default: 'game-ci'
required: false required: false
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.' description: '[Orchestrator] The Cloud Formation stack name that must be setup before using this option.'
providerStrategy: providerStrategy:
default: 'local' default: 'local'
required: false required: false
description: description:
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must '[Orchestrator] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
be configured.' be configured.'
resourceTracking: resourceTracking:
default: 'false' default: 'false'
required: false required: false
description: '[CloudRunner] Enable resource tracking logs for disk usage and allocation summaries.' description: '[Orchestrator] Enable resource tracking logs for disk usage and allocation summaries.'
containerCpu: containerCpu:
default: '' default: ''
required: false required: false
description: '[CloudRunner] Amount of CPU time to assign the remote build container' description: '[Orchestrator] Amount of CPU time to assign the remote build container'
containerMemory: containerMemory:
default: '' default: ''
required: false required: false
description: '[CloudRunner] Amount of memory to assign the remote build container' description: '[Orchestrator] Amount of memory to assign the remote build container'
readInputFromOverrideList: readInputFromOverrideList:
default: '' default: ''
required: false required: false
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"' description: '[Orchestrator] Comma separated list of input value names to read from "input override command"'
readInputOverrideCommand: readInputOverrideCommand:
default: '' default: ''
required: false required: false
description: description:
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud '[Orchestrator] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
provider secret managers' provider secret managers'
kubeConfig: kubeConfig:
default: '' default: ''
required: false required: false
description: description:
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until '[Orchestrator] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
completion.' completion.'
kubeVolume: kubeVolume:
default: '' default: ''
required: false required: false
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.' description: '[Orchestrator] Supply a Persistent Volume Claim name to use for the Unity build.'
kubeStorageClass: kubeStorageClass:
default: '' default: ''
required: false required: false
description: description:
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.' '[Orchestrator] Kubernetes storage class to use for orchestrator jobs, leave empty to install rook cluster.'
kubeVolumeSize: kubeVolumeSize:
default: '5Gi' default: '5Gi'
required: false required: false
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume' description: '[Orchestrator] Amount of disc space to assign the Kubernetes Persistent Volume'
cacheKey: cacheKey:
default: '' default: ''
required: false required: false
description: '[CloudRunner] Cache key to indicate bucket for cache' description: '[Orchestrator] Cache key to indicate bucket for cache'
watchToEnd: watchToEnd:
default: 'true' default: 'true'
required: false required: false
description: description:
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g '[Orchestrator] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
imports or self-hosted ephemeral runners.' imports or self-hosted ephemeral runners.'
cacheUnityInstallationOnMac: cacheUnityInstallationOnMac:
default: 'false' default: 'false'
@@ -272,12 +272,12 @@ inputs:
cloneDepth: cloneDepth:
default: '50' default: '50'
required: false required: false
description: '[CloudRunner] Specifies the depth of the git clone for the repository. Use 0 for full clone.' description: '[Orchestrator] Specifies the depth of the git clone for the repository. Use 0 for full clone.'
cloudRunnerRepoName: orchestratorRepoName:
default: 'game-ci/unity-builder' default: 'game-ci/unity-builder'
required: false required: false
description: description:
'[CloudRunner] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or '[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
fixes.' fixes.'
outputs: outputs:

5237
dist/index.js generated vendored

File diff suppressed because it is too large Load Diff

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@@ -12,18 +12,18 @@
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts", "lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
"format": "prettier --write \"src/**/*.{js,ts}\"", "format": "prettier --write \"src/**/*.{js,ts}\"",
"cli": "yarn ts-node src/index.ts -m cli", "cli": "yarn ts-node src/index.ts -m cli",
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"", "gcp-secrets-tests": "cross-env providerStrategy=aws orchestratorTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"orchestrator\"",
"gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", "gcp-secrets-cli": "cross-env orchestratorTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", "aws-secrets-cli": "cross-env orchestratorTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli", "cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli", "cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project", "test-cli": "cross-env orchestratorTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
"test": "jest", "test": "jest",
"test:ci": "jest --config=jest.ci.config.js --runInBand", "test:ci": "jest --config=jest.ci.config.js --runInBand",
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"", "test-i": "cross-env orchestratorTests=true yarn test -i -t \"orchestrator\"",
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s", "test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"", "test-i-aws": "cross-env orchestratorTests=true providerStrategy=aws yarn test -i -t \"orchestrator\"",
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\"" "test-i-k8s": "cross-env orchestratorTests=true providerStrategy=k8s yarn test -i -t \"orchestrator\""
}, },
"engines": { "engines": {
"node": ">=18.x" "node": ">=18.x"

View File

@@ -1,5 +1,5 @@
import * as core from '@actions/core'; import * as core from '@actions/core';
import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model'; import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
import { Cli } from './model/cli/cli'; import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder'; import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup'; import PlatformSetup from './model/platform-setup';
@@ -33,7 +33,7 @@ async function runMain() {
...buildParameters, ...buildParameters,
}); });
} else { } else {
await CloudRunner.run(buildParameters, baseImage.toString()); await Orchestrator.run(buildParameters, baseImage.toString());
exitCode = 0; exitCode = 0;
} }

View File

@@ -1,13 +1,13 @@
// Integration test for exercising real GitHub check creation and updates. // Integration test for exercising real GitHub check creation and updates.
import CloudRunner from '../model/cloud-runner/cloud-runner'; import Orchestrator from '../model/orchestrator/orchestrator';
import UnityVersioning from '../model/unity-versioning'; import UnityVersioning from '../model/unity-versioning';
import GitHub from '../model/github'; import GitHub from '../model/github';
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/cloud-runner-test-helpers'; import { TIMEOUT_INFINITE, createParameters } from '../test-utils/orchestrator-test-helpers';
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true'; const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
const describeOrSkip = runIntegration ? describe : describe.skip; const describeOrSkip = runIntegration ? describe : describe.skip;
describeOrSkip('Cloud Runner Github Checks Integration', () => { describeOrSkip('Orchestrator Github Checks Integration', () => {
it( it(
'creates and updates a real GitHub check', 'creates and updates a real GitHub check',
async () => { async () => {
@@ -15,10 +15,10 @@ describeOrSkip('Cloud Runner Github Checks Integration', () => {
versioning: 'None', versioning: 'None',
projectPath: 'test-project', projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'), unityVersion: UnityVersioning.read('test-project'),
asyncCloudRunner: `true`, asyncOrchestrator: `true`,
githubChecks: `true`, githubChecks: `true`,
}); });
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
const checkId = await GitHub.createGitHubCheck(`integration create`); const checkId = await GitHub.createGitHubCheck(`integration create`);
expect(checkId).not.toEqual(''); expect(checkId).not.toEqual('');
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`); await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);

View File

@@ -1,7 +1,7 @@
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import AndroidVersioning from './android-versioning'; import AndroidVersioning from './android-versioning';
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants'; import OrchestratorConstants from './orchestrator/options/orchestrator-constants';
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid'; import OrchestratorBuildGuid from './orchestrator/options/orchestrator-guid';
import Input from './input'; import Input from './input';
import Platform from './platform'; import Platform from './platform';
import UnityVersioning from './unity-versioning'; import UnityVersioning from './unity-versioning';
@@ -10,8 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
import { GithubCliReader } from './input-readers/github-cli'; import { GithubCliReader } from './input-readers/github-cli';
import { Cli } from './cli/cli'; import { Cli } from './cli/cli';
import GitHub from './github'; import GitHub from './github';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from './orchestrator/options/orchestrator-options';
import CloudRunner from './cloud-runner/cloud-runner'; import Orchestrator from './orchestrator/orchestrator';
import * as core from '@actions/core'; import * as core from '@actions/core';
class BuildParameters { class BuildParameters {
@@ -84,13 +84,13 @@ class BuildParameters {
public runNumber!: string; public runNumber!: string;
public branch!: string; public branch!: string;
public githubRepo!: string; public githubRepo!: string;
public cloudRunnerRepoName!: string; public orchestratorRepoName!: string;
public cloneDepth!: number; public cloneDepth!: number;
public gitSha!: string; public gitSha!: string;
public logId!: string; public logId!: string;
public buildGuid!: string; public buildGuid!: string;
public cloudRunnerBranch!: string; public orchestratorBranch!: string;
public cloudRunnerDebug!: boolean | undefined; public orchestratorDebug!: boolean | undefined;
public buildPlatform!: string | undefined; public buildPlatform!: string | undefined;
public isCliMode!: boolean; public isCliMode!: boolean;
public maxRetainedWorkspaces!: number; public maxRetainedWorkspaces!: number;
@@ -108,7 +108,7 @@ class BuildParameters {
public dockerWorkspacePath!: string; public dockerWorkspacePath!: string;
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) { public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``; return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
} }
static async create(): Promise<BuildParameters> { static async create(): Promise<BuildParameters> {
@@ -193,52 +193,52 @@ class BuildParameters {
dockerIsolationMode: Input.dockerIsolationMode, dockerIsolationMode: Input.dockerIsolationMode,
containerRegistryRepository: Input.containerRegistryRepository, containerRegistryRepository: Input.containerRegistryRepository,
containerRegistryImageVersion: Input.containerRegistryImageVersion, containerRegistryImageVersion: Input.containerRegistryImageVersion,
providerStrategy: CloudRunnerOptions.providerStrategy, providerStrategy: OrchestratorOptions.providerStrategy,
buildPlatform: CloudRunnerOptions.buildPlatform, buildPlatform: OrchestratorOptions.buildPlatform,
kubeConfig: CloudRunnerOptions.kubeConfig, kubeConfig: OrchestratorOptions.kubeConfig,
containerMemory: CloudRunnerOptions.containerMemory, containerMemory: OrchestratorOptions.containerMemory,
containerCpu: CloudRunnerOptions.containerCpu, containerCpu: OrchestratorOptions.containerCpu,
containerNamespace: CloudRunnerOptions.containerNamespace, containerNamespace: OrchestratorOptions.containerNamespace,
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize, kubeVolumeSize: OrchestratorOptions.kubeVolumeSize,
kubeVolume: CloudRunnerOptions.kubeVolume, kubeVolume: OrchestratorOptions.kubeVolume,
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks, postBuildContainerHooks: OrchestratorOptions.postBuildContainerHooks,
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks, preBuildContainerHooks: OrchestratorOptions.preBuildContainerHooks,
customJob: CloudRunnerOptions.customJob, customJob: OrchestratorOptions.customJob,
runNumber: Input.runNumber, runNumber: Input.runNumber,
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()), branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0], orchestratorBranch: OrchestratorOptions.orchestratorBranch.split('/').reverse()[0],
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug, orchestratorDebug: OrchestratorOptions.orchestratorDebug,
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || CloudRunnerOptions.cloudRunnerRepoName, githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || OrchestratorOptions.orchestratorRepoName,
cloudRunnerRepoName: CloudRunnerOptions.cloudRunnerRepoName, orchestratorRepoName: OrchestratorOptions.orchestratorRepoName,
cloneDepth: Number.parseInt(CloudRunnerOptions.cloneDepth), cloneDepth: Number.parseInt(OrchestratorOptions.cloneDepth),
isCliMode: Cli.isCliMode, isCliMode: Cli.isCliMode,
awsStackName: CloudRunnerOptions.awsStackName, awsStackName: OrchestratorOptions.awsStackName,
awsEndpoint: CloudRunnerOptions.awsEndpoint, awsEndpoint: OrchestratorOptions.awsEndpoint,
awsCloudFormationEndpoint: CloudRunnerOptions.awsCloudFormationEndpoint, awsCloudFormationEndpoint: OrchestratorOptions.awsCloudFormationEndpoint,
awsEcsEndpoint: CloudRunnerOptions.awsEcsEndpoint, awsEcsEndpoint: OrchestratorOptions.awsEcsEndpoint,
awsKinesisEndpoint: CloudRunnerOptions.awsKinesisEndpoint, awsKinesisEndpoint: OrchestratorOptions.awsKinesisEndpoint,
awsCloudWatchLogsEndpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint, awsCloudWatchLogsEndpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
awsS3Endpoint: CloudRunnerOptions.awsS3Endpoint, awsS3Endpoint: OrchestratorOptions.awsS3Endpoint,
storageProvider: CloudRunnerOptions.storageProvider, storageProvider: OrchestratorOptions.storageProvider,
rcloneRemote: CloudRunnerOptions.rcloneRemote, rcloneRemote: OrchestratorOptions.rcloneRemote,
gitSha: Input.gitSha, gitSha: Input.gitSha,
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(), logId: customAlphabet(OrchestratorConstants.alphabet, 9)(),
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform), buildGuid: OrchestratorBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
commandHooks: CloudRunnerOptions.commandHooks, commandHooks: OrchestratorOptions.commandHooks,
inputPullCommand: CloudRunnerOptions.inputPullCommand, inputPullCommand: OrchestratorOptions.inputPullCommand,
pullInputList: CloudRunnerOptions.pullInputList, pullInputList: OrchestratorOptions.pullInputList,
kubeStorageClass: CloudRunnerOptions.kubeStorageClass, kubeStorageClass: OrchestratorOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey, cacheKey: OrchestratorOptions.cacheKey,
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces), maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces),
useLargePackages: CloudRunnerOptions.useLargePackages, useLargePackages: OrchestratorOptions.useLargePackages,
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy, useCompressionStrategy: OrchestratorOptions.useCompressionStrategy,
garbageMaxAge: CloudRunnerOptions.garbageMaxAge, garbageMaxAge: OrchestratorOptions.garbageMaxAge,
githubChecks: CloudRunnerOptions.githubChecks, githubChecks: OrchestratorOptions.githubChecks,
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner, asyncWorkflow: OrchestratorOptions.asyncOrchestrator,
githubCheckId: CloudRunnerOptions.githubCheckId, githubCheckId: OrchestratorOptions.githubCheckId,
finalHooks: CloudRunnerOptions.finalHooks, finalHooks: OrchestratorOptions.finalHooks,
skipLfs: CloudRunnerOptions.skipLfs, skipLfs: OrchestratorOptions.skipLfs,
skipCache: CloudRunnerOptions.skipCache, skipCache: OrchestratorOptions.skipCache,
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac, cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac, unityHubVersionOnMac: Input.unityHubVersionOnMac,
dockerWorkspacePath: Input.dockerWorkspacePath, dockerWorkspacePath: Input.dockerWorkspacePath,

View File

@@ -1,14 +1,14 @@
import { Command } from 'commander-ts'; import { Command } from 'commander-ts';
import { BuildParameters, CloudRunner, ImageTag, Input } from '..'; import { BuildParameters, Orchestrator, ImageTag, Input } from '..';
import * as core from '@actions/core'; import * as core from '@actions/core';
import { ActionYamlReader } from '../input-readers/action-yaml'; import { ActionYamlReader } from '../input-readers/action-yaml';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger'; import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override'; import OrchestratorQueryOverride from '../orchestrator/options/orchestrator-query-override';
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository'; import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
import { Caching } from '../cloud-runner/remote-client/caching'; import { Caching } from '../orchestrator/remote-client/caching';
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing'; import { LfsHashing } from '../orchestrator/services/utility/lfs-hashing';
import { RemoteClient } from '../cloud-runner/remote-client'; import { RemoteClient } from '../orchestrator/remote-client';
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader'; import OrchestratorOptionsReader from '../orchestrator/options/orchestrator-options-reader';
import GitHub from '../github'; import GitHub from '../github';
import { OptionValues } from 'commander'; import { OptionValues } from 'commander';
import { InputKey } from '../input'; import { InputKey } from '../input';
@@ -36,7 +36,7 @@ export class Cli {
const program = new Command(); const program = new Command();
program.version('0.0.1'); program.version('0.0.1');
const properties = CloudRunnerOptionsReader.GetProperties(); const properties = OrchestratorOptionsReader.GetProperties();
const actionYamlReader: ActionYamlReader = new ActionYamlReader(); const actionYamlReader: ActionYamlReader = new ActionYamlReader();
for (const element of properties) { for (const element of properties) {
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element)); program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
@@ -62,23 +62,23 @@ export class Cli {
static async RunCli(): Promise<void> { static async RunCli(): Promise<void> {
GitHub.githubInputEnabled = false; GitHub.githubInputEnabled = false;
if (Cli.options!['populateOverride'] === `true`) { if (Cli.options!['populateOverride'] === `true`) {
await CloudRunnerQueryOverride.PopulateQueryOverrideInput(); await OrchestratorQueryOverride.PopulateQueryOverrideInput();
} }
if (Cli.options!['logInput']) { if (Cli.options!['logInput']) {
Cli.logInput(); Cli.logInput();
} }
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode); const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
CloudRunnerLogger.log(`Entrypoint: ${results.key}`); OrchestratorLogger.log(`Entrypoint: ${results.key}`);
Cli.options!.versioning = 'None'; Cli.options!.versioning = 'None';
CloudRunner.buildParameters = await BuildParameters.create(); Orchestrator.buildParameters = await BuildParameters.create();
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``; Orchestrator.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
CloudRunnerLogger.log(`Build Params: OrchestratorLogger.log(`Build Params:
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)} ${JSON.stringify(Orchestrator.buildParameters, undefined, 4)}
`); `);
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``; Orchestrator.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`); OrchestratorLogger.log(`Locked Workspace: ${Orchestrator.lockedWorkspace}`);
await CloudRunner.setup(CloudRunner.buildParameters); await Orchestrator.setup(Orchestrator.buildParameters);
return await results.target[results.propertyKey](Cli.options); return await results.target[results.propertyKey](Cli.options);
} }
@@ -87,7 +87,7 @@ export class Cli {
private static logInput() { private static logInput() {
core.info(`\n`); core.info(`\n`);
core.info(`INPUT:`); core.info(`INPUT:`);
const properties = CloudRunnerOptionsReader.GetProperties(); const properties = OrchestratorOptionsReader.GetProperties();
for (const element of properties) { for (const element of properties) {
if ( if (
element in Input && element in Input &&
@@ -104,28 +104,28 @@ export class Cli {
core.info(`\n`); core.info(`\n`);
} }
@CliFunction(`cli-build`, `runs a cloud runner build`) @CliFunction(`cli-build`, `runs a orchestrator build`)
public static async CLIBuild(): Promise<string> { public static async CLIBuild(): Promise<string> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults; return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
} }
@CliFunction(`async-workflow`, `runs a cloud runner build`) @CliFunction(`async-workflow`, `runs a orchestrator build`)
public static async asyncronousWorkflow(): Promise<string> { public static async asyncronousWorkflow(): Promise<string> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults; return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
} }
@CliFunction(`checks-update`, `runs a cloud runner build`) @CliFunction(`checks-update`, `runs a orchestrator build`)
public static async checksUpdate() { public static async checksUpdate() {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
const input = JSON.parse(process.env.CHECKS_UPDATE || ``); const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`); core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
if (input.mode === `create`) { if (input.mode === `create`) {
@@ -139,18 +139,18 @@ export class Cli {
public static async GarbageCollect(): Promise<string> { public static async GarbageCollect(): Promise<string> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false); return await Orchestrator.Provider.garbageCollect(``, false, 0, false, false);
} }
@CliFunction(`list-resources`, `lists active resources`) @CliFunction(`list-resources`, `lists active resources`)
public static async ListResources(): Promise<string[]> { public static async ListResources(): Promise<string[]> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
const result = await CloudRunner.Provider.listResources(); const result = await Orchestrator.Provider.listResources();
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4)); OrchestratorLogger.log(JSON.stringify(result, undefined, 4));
return result.map((x) => x.Name); return result.map((x) => x.Name);
} }
@@ -159,17 +159,17 @@ export class Cli {
public static async ListWorfklow(): Promise<string[]> { public static async ListWorfklow(): Promise<string[]> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name); return (await Orchestrator.Provider.listWorkflow()).map((x) => x.Name);
} }
@CliFunction(`watch`, `follows logs of a running workflow`) @CliFunction(`watch`, `follows logs of a running workflow`)
public static async Watch(): Promise<string> { public static async Watch(): Promise<string> {
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
return await CloudRunner.Provider.watchWorkflow(); return await Orchestrator.Provider.watchWorkflow();
} }
} }

View File

@@ -1,15 +0,0 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import CloudRunner from '../cloud-runner';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import BuildParameters from '../../build-parameters';
export class CloudRunnerError {
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
core.setFailed('Cloud Runner failed');
if (CloudRunner.Provider !== undefined) {
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
}
}
}

View File

@@ -1,4 +0,0 @@
class CloudRunnerConstants {
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
}
export default CloudRunnerConstants;

View File

@@ -1,5 +0,0 @@
class CloudRunnerEnvironmentVariable {
public name!: string;
public value!: string;
}
export default CloudRunnerEnvironmentVariable;

View File

@@ -1,90 +0,0 @@
import path from 'node:path';
import CloudRunnerOptions from './cloud-runner-options';
import CloudRunner from '../cloud-runner';
import BuildParameters from '../../build-parameters';
export class CloudRunnerFolders {
public static readonly repositoryFolder = 'repo';
public static ToLinuxFolder(folder: string) {
return folder.replace(/\\/g, `/`);
}
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}
public static get cacheFolderForAllFull(): string {
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
}
public static get cacheFolderForCacheKeyFull(): string {
return path.join(
'/',
CloudRunnerFolders.buildVolumeFolder,
CloudRunnerFolders.cacheFolder,
CloudRunner.buildParameters.cacheKey,
);
}
public static get builderPathAbsolute(): string {
return path.join(
CloudRunnerOptions.useSharedBuilder
? `/${CloudRunnerFolders.buildVolumeFolder}`
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
`builder`,
);
}
public static get repoPathAbsolute(): string {
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
}
public static get projectPathAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
}
public static get libraryFolderAbsolute(): string {
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
}
public static get projectBuildFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
}
public static get lfsFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
}
public static get purgeRemoteCaching(): boolean {
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
}
public static get lfsCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
}
public static get libraryCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
}
public static get unityBuilderRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.cloudRunnerRepoName}.git`;
}
public static get targetBuildRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
}
public static get buildVolumeFolder() {
return 'data';
}
public static get cacheFolder() {
return 'cache';
}
}

View File

@@ -1,10 +0,0 @@
import Input from '../../input';
import CloudRunnerOptions from './cloud-runner-options';
class CloudRunnerOptionsReader {
static GetProperties() {
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
}
}
export default CloudRunnerOptionsReader;

View File

@@ -1,335 +0,0 @@
import { Cli } from '../../cli/cli';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
import GitHub from '../../github';
import * as core from '@actions/core';
class CloudRunnerOptions {
// ### ### ###
// Input Handling
// ### ### ###
public static getInput(query: string): string | undefined {
if (GitHub.githubInputEnabled) {
const coreInput = core.getInput(query);
if (coreInput && coreInput !== '') {
return coreInput;
}
}
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
// Query input sources
if (Cli.query(query, alternativeQuery)) {
return Cli.query(query, alternativeQuery);
}
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
return CloudRunnerQueryOverride.query(query, alternativeQuery);
}
if (process.env[query] !== undefined) {
return process.env[query];
}
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
return process.env[alternativeQuery];
}
}
public static ToEnvVarFormat(input: string): string {
if (input.toUpperCase() === input) {
return input;
}
return input
.replace(/([A-Z])/g, ' $1')
.trim()
.toUpperCase()
.replace(/ /g, '_');
}
// ### ### ###
// Provider parameters
// ### ### ###
static get region(): string {
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
}
// ### ### ###
// GitHub parameters
// ### ### ###
static get githubChecks(): boolean {
const value = CloudRunnerOptions.getInput('githubChecks');
return value === `true` || false;
}
static get githubCheckId(): string {
return CloudRunnerOptions.getInput('githubCheckId') || ``;
}
static get githubOwner(): string {
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
}
static get githubRepoName(): string {
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
}
static get cloudRunnerRepoName(): string {
return CloudRunnerOptions.getInput('cloudRunnerRepoName') || 'game-ci/unity-builder';
}
static get cloneDepth(): string {
return CloudRunnerOptions.getInput('cloneDepth') || '50';
}
static get finalHooks(): string[] {
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
}
// ### ### ###
// Git syncronization parameters
// ### ### ###
static get githubRepo(): string | undefined {
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
}
static get branch(): string {
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
return (
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
);
} else if (CloudRunnerOptions.getInput('branch')) {
return CloudRunnerOptions.getInput('branch') || ``;
} else {
return '';
}
}
// ### ### ###
// Cloud Runner parameters
// ### ### ###
static get buildPlatform(): string {
const input = CloudRunnerOptions.getInput('buildPlatform');
if (input && input !== '') {
return input;
}
if (CloudRunnerOptions.providerStrategy !== 'local') {
return 'linux';
}
return process.platform;
}
static get cloudRunnerBranch(): string {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
}
static get providerStrategy(): string {
const provider =
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
if (Cli.isCliMode) {
return provider || 'aws';
}
return provider || 'local';
}
static get containerCpu(): string {
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
}
static get containerMemory(): string {
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
}
static get containerNamespace(): string {
return CloudRunnerOptions.getInput('containerNamespace') || `default`;
}
static get customJob(): string {
return CloudRunnerOptions.getInput('customJob') || '';
}
// ### ### ###
// Custom commands from files parameters
// ### ### ###
static get containerHookFiles(): string[] {
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
}
static get commandHookFiles(): string[] {
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
}
// ### ### ###
// Custom commands from yaml parameters
// ### ### ###
static get commandHooks(): string {
return CloudRunnerOptions.getInput('commandHooks') || '';
}
static get postBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
}
static get preBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
}
// ### ### ###
// Input override handling
// ### ### ###
static get pullInputList(): string[] {
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
}
static get inputPullCommand(): string {
const value = CloudRunnerOptions.getInput('inputPullCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
} else if (value === 'aws-secret-manager') {
return 'aws secretsmanager get-secret-value --secret-id {0}';
}
return value || '';
}
// ### ### ###
// Aws
// ### ### ###
static get awsStackName() {
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
}
static get awsEndpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsEndpoint');
}
static get awsCloudFormationEndpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsCloudFormationEndpoint') || CloudRunnerOptions.awsEndpoint;
}
static get awsEcsEndpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsEcsEndpoint') || CloudRunnerOptions.awsEndpoint;
}
static get awsKinesisEndpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsKinesisEndpoint') || CloudRunnerOptions.awsEndpoint;
}
static get awsCloudWatchLogsEndpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsCloudWatchLogsEndpoint') || CloudRunnerOptions.awsEndpoint;
}
static get awsS3Endpoint(): string | undefined {
return CloudRunnerOptions.getInput('awsS3Endpoint') || CloudRunnerOptions.awsEndpoint;
}
// ### ### ###
// Storage
// ### ### ###
static get storageProvider(): string {
return CloudRunnerOptions.getInput('storageProvider') || 's3';
}
static get rcloneRemote(): string {
return CloudRunnerOptions.getInput('rcloneRemote') || '';
}
// ### ### ###
// K8s
// ### ### ###
static get kubeConfig(): string {
return CloudRunnerOptions.getInput('kubeConfig') || '';
}
static get kubeVolume(): string {
return CloudRunnerOptions.getInput('kubeVolume') || '';
}
static get kubeVolumeSize(): string {
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
}
static get kubeStorageClass(): string {
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
}
// ### ### ###
// Caching
// ### ### ###
static get cacheKey(): string {
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
}
// ### ### ###
// Utility Parameters
// ### ### ###
static get cloudRunnerDebug(): boolean {
return (
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false
);
}
static get skipLfs(): boolean {
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
}
static get skipCache(): boolean {
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
}
public static get asyncCloudRunner(): boolean {
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
}
public static get resourceTracking(): boolean {
return CloudRunnerOptions.getInput('resourceTracking') === 'true';
}
public static get useLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
}
public static get useSharedBuilder(): boolean {
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
}
public static get useCompressionStrategy(): boolean {
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
}
public static get useCleanupCron(): boolean {
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
}
// ### ### ###
// Retained Workspace
// ### ### ###
public static get maxRetainedWorkspaces(): string {
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get garbageMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
}
}
export default CloudRunnerOptions;

View File

@@ -1,67 +0,0 @@
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import CloudRunnerOptions from './cloud-runner-options';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
value = value.replace(`{${element.key}}`, element.value);
}
return value;
};
class CloudRunnerQueryOverride {
static queryOverrides: { [key: string]: string } | undefined;
// TODO accept premade secret sources or custom secret source definition yamls
public static query(key: string, alternativeKey: string) {
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
return CloudRunnerQueryOverride.queryOverrides[key];
}
if (
CloudRunnerQueryOverride.queryOverrides &&
alternativeKey &&
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
) {
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
}
return;
}
private static shouldUseOverride(query: string) {
if (CloudRunnerOptions.inputPullCommand !== '') {
if (CloudRunnerOptions.pullInputList.length > 0) {
const doesInclude =
CloudRunnerOptions.pullInputList.includes(query) ||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
return true;
}
}
}
private static async queryOverride(query: string) {
if (!this.shouldUseOverride(query)) {
throw new Error(`Should not be trying to run override query on ${query}`);
}
return await GenericInputReader.Run(
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = CloudRunnerOptions.pullInputList;
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
}
}
}
}
export default CloudRunnerQueryOverride;

View File

@@ -1,3 +0,0 @@
export class CloudRunnerStatics {
public static readonly logPrefix = `Cloud-Runner`;
}

View File

@@ -1,13 +0,0 @@
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import CloudRunnerSecret from './cloud-runner-secret';
export class CloudRunnerStepParameters {
public image: string;
public environment: CloudRunnerEnvironmentVariable[];
public secrets: CloudRunnerSecret[];
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
this.image = image;
this.environment = environmentVariables;
this.secrets = secrets;
}
}

View File

@@ -1,39 +0,0 @@
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
import { CustomWorkflow } from './custom-workflow';
import { WorkflowInterface } from './workflow-interface';
import { BuildAutomationWorkflow } from './build-automation-workflow';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../options/cloud-runner-options';
import { AsyncWorkflow } from './async-workflow';
export class WorkflowCompositionRoot implements WorkflowInterface {
async run(cloudRunnerStepState: CloudRunnerStepParameters) {
try {
if (
CloudRunnerOptions.asyncCloudRunner &&
!CloudRunner.isCloudRunnerAsyncEnvironment &&
!CloudRunner.isCloudRunnerEnvironment
) {
return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
}
if (CloudRunner.buildParameters.customJob !== '') {
return await CustomWorkflow.runContainerJobFromString(
CloudRunner.buildParameters.customJob,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
);
}
return await new BuildAutomationWorkflow().run(
new CloudRunnerStepParameters(
cloudRunnerStepState.image.toString(),
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
),
);
} catch (error) {
throw error;
}
}
}

View File

@@ -1,8 +0,0 @@
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
export interface WorkflowInterface {
run(
// eslint-disable-next-line no-unused-vars
cloudRunnerStepState: CloudRunnerStepParameters,
): Promise<string>;
}

View File

@@ -1,6 +1,6 @@
import CloudRunnerLogger from './cloud-runner/services/core/cloud-runner-logger'; import OrchestratorLogger from './orchestrator/services/core/orchestrator-logger';
import CloudRunner from './cloud-runner/cloud-runner'; import Orchestrator from './orchestrator/orchestrator';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from './orchestrator/options/orchestrator-options';
import * as core from '@actions/core'; import * as core from '@actions/core';
import { Octokit } from '@octokit/core'; import { Octokit } from '@octokit/core';
@@ -19,15 +19,15 @@ class GitHub {
} }
private static get octokitPAT() { private static get octokitPAT() {
return new Octokit({ return new Octokit({
auth: CloudRunner.buildParameters.gitPrivateToken, auth: Orchestrator.buildParameters.gitPrivateToken,
}); });
} }
private static get sha() { private static get sha() {
return CloudRunner.buildParameters.gitSha; return Orchestrator.buildParameters.gitSha;
} }
private static get checkName() { private static get checkName() {
return `Cloud Runner (${CloudRunner.buildParameters.buildGuid})`; return `Orchestrator (${Orchestrator.buildParameters.buildGuid})`;
} }
private static get nameReadable() { private static get nameReadable() {
@@ -35,24 +35,24 @@ class GitHub {
} }
private static get checkRunId() { private static get checkRunId() {
return CloudRunner.buildParameters.githubCheckId; return Orchestrator.buildParameters.githubCheckId;
} }
private static get owner() { private static get owner() {
return CloudRunnerOptions.githubOwner; return OrchestratorOptions.githubOwner;
} }
private static get repo() { private static get repo() {
return CloudRunnerOptions.githubRepoName; return OrchestratorOptions.githubRepoName;
} }
public static async createGitHubCheck(summary: string) { public static async createGitHubCheck(summary: string) {
if (!CloudRunner.buildParameters.githubChecks) { if (!Orchestrator.buildParameters.githubChecks) {
return ``; return ``;
} }
GitHub.startedDate = new Date().toISOString(); GitHub.startedDate = new Date().toISOString();
CloudRunnerLogger.log(`Creating github check`); OrchestratorLogger.log(`Creating github check`);
const data = { const data = {
owner: GitHub.owner, owner: GitHub.owner,
repo: GitHub.repo, repo: GitHub.repo,
@@ -61,7 +61,7 @@ class GitHub {
head_sha: GitHub.sha, head_sha: GitHub.sha,
status: 'queued', status: 'queued',
// eslint-disable-next-line camelcase // eslint-disable-next-line camelcase
external_id: CloudRunner.buildParameters.buildGuid, external_id: Orchestrator.buildParameters.buildGuid,
// eslint-disable-next-line camelcase // eslint-disable-next-line camelcase
started_at: GitHub.startedDate, started_at: GitHub.startedDate,
output: { output: {
@@ -79,7 +79,7 @@ class GitHub {
}; };
const result = await GitHub.createGitHubCheckRequest(data); const result = await GitHub.createGitHubCheckRequest(data);
CloudRunnerLogger.log(`Creating github check ${result.status}`); OrchestratorLogger.log(`Creating github check ${result.status}`);
return result.data.id.toString(); return result.data.id.toString();
} }
@@ -90,11 +90,11 @@ class GitHub {
result = `neutral`, result = `neutral`,
status = `in_progress`, status = `in_progress`,
) { ) {
if (`${CloudRunner.buildParameters.githubChecks}` !== `true`) { if (`${Orchestrator.buildParameters.githubChecks}` !== `true`) {
return; return;
} }
CloudRunnerLogger.log( OrchestratorLogger.log(
`githubChecks: ${CloudRunner.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${CloudRunner.isCloudRunnerAsyncEnvironment}`, `githubChecks: ${Orchestrator.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${Orchestrator.isOrchestratorAsyncEnvironment}`,
); );
GitHub.longDescriptionContent += `\n${longDescription}`; GitHub.longDescriptionContent += `\n${longDescription}`;
if (GitHub.result !== `success` && GitHub.result !== `failure`) { if (GitHub.result !== `success` && GitHub.result !== `failure`) {
@@ -130,7 +130,7 @@ class GitHub {
data.conclusion = result; data.conclusion = result;
} }
await (CloudRunner.isCloudRunnerAsyncEnvironment || GitHub.forceAsyncTest await (Orchestrator.isOrchestratorAsyncEnvironment || GitHub.forceAsyncTest
? GitHub.runUpdateAsyncChecksWorkflow(data, `update`) ? GitHub.runUpdateAsyncChecksWorkflow(data, `update`)
: GitHub.updateGitHubCheckRequest(data)); : GitHub.updateGitHubCheckRequest(data));
} }
@@ -152,7 +152,7 @@ class GitHub {
repo: GitHub.repo, repo: GitHub.repo,
}); });
const workflows = workflowsResult.data.workflows; const workflows = workflowsResult.data.workflows;
CloudRunnerLogger.log(`Got ${workflows.length} workflows`); OrchestratorLogger.log(`Got ${workflows.length} workflows`);
let selectedId = ``; let selectedId = ``;
for (let index = 0; index < workflowsResult.data.total_count; index++) { for (let index = 0; index < workflowsResult.data.total_count; index++) {
if (workflows[index].name === GitHub.asyncChecksApiWorkflowName) { if (workflows[index].name === GitHub.asyncChecksApiWorkflowName) {
@@ -168,7 +168,7 @@ class GitHub {
repo: GitHub.repo, repo: GitHub.repo,
// eslint-disable-next-line camelcase // eslint-disable-next-line camelcase
workflow_id: selectedId, workflow_id: selectedId,
ref: CloudRunnerOptions.branch, ref: OrchestratorOptions.branch,
inputs: { inputs: {
checksObject: JSON.stringify({ data, mode }), checksObject: JSON.stringify({ data, mode }),
}, },
@@ -176,7 +176,7 @@ class GitHub {
} }
static async triggerWorkflowOnComplete(triggerWorkflowOnComplete: string[]) { static async triggerWorkflowOnComplete(triggerWorkflowOnComplete: string[]) {
const isLocalAsync = CloudRunner.buildParameters.asyncWorkflow && !CloudRunner.isCloudRunnerAsyncEnvironment; const isLocalAsync = Orchestrator.buildParameters.asyncWorkflow && !Orchestrator.isOrchestratorAsyncEnvironment;
if (isLocalAsync || triggerWorkflowOnComplete === undefined || triggerWorkflowOnComplete.length === 0) { if (isLocalAsync || triggerWorkflowOnComplete === undefined || triggerWorkflowOnComplete.length === 0) {
return; return;
} }
@@ -186,7 +186,7 @@ class GitHub {
repo: GitHub.repo, repo: GitHub.repo,
}); });
const workflows = workflowsResult.data.workflows; const workflows = workflowsResult.data.workflows;
CloudRunnerLogger.log(`Got ${workflows.length} workflows`); OrchestratorLogger.log(`Got ${workflows.length} workflows`);
for (const element of triggerWorkflowOnComplete) { for (const element of triggerWorkflowOnComplete) {
let selectedId = ``; let selectedId = ``;
for (let index = 0; index < workflowsResult.data.total_count; index++) { for (let index = 0; index < workflowsResult.data.total_count; index++) {
@@ -203,9 +203,9 @@ class GitHub {
repo: GitHub.repo, repo: GitHub.repo,
// eslint-disable-next-line camelcase // eslint-disable-next-line camelcase
workflow_id: selectedId, workflow_id: selectedId,
ref: CloudRunnerOptions.branch, ref: OrchestratorOptions.branch,
inputs: { inputs: {
buildGuid: CloudRunner.buildParameters.buildGuid, buildGuid: Orchestrator.buildParameters.buildGuid,
}, },
}); });
} }

View File

@@ -84,7 +84,7 @@ class ImageEnvironmentFactory {
{ name: 'RUNNER_WORKSPACE', value: process.env.RUNNER_WORKSPACE }, { name: 'RUNNER_WORKSPACE', value: process.env.RUNNER_WORKSPACE },
]; ];
// Always merge additional variables (e.g., secrets/env from Cloud Runner) uniquely by name // Always merge additional variables (e.g., secrets/env from Orchestrator) uniquely by name
for (const element of additionalVariables) { for (const element of additionalVariables) {
if (!element || !element.name) continue; if (!element || !element.name) continue;
environmentVariables = environmentVariables.filter((x) => x?.name !== element.name); environmentVariables = environmentVariables.filter((x) => x?.name !== element.name);

View File

@@ -9,8 +9,8 @@ import Platform from './platform';
import Project from './project'; import Project from './project';
import Unity from './unity'; import Unity from './unity';
import Versioning from './versioning'; import Versioning from './versioning';
import CloudRunner from './cloud-runner/cloud-runner'; import Orchestrator from './orchestrator/orchestrator';
import loadProvider, { ProviderLoader } from './cloud-runner/providers/provider-loader'; import loadProvider, { ProviderLoader } from './orchestrator/providers/provider-loader';
export { export {
Action, Action,
@@ -24,7 +24,7 @@ export {
Project, Project,
Unity, Unity,
Versioning, Versioning,
CloudRunner as CloudRunner, Orchestrator as Orchestrator,
loadProvider, loadProvider,
ProviderLoader, ProviderLoader,
}; };

View File

@@ -1,12 +1,12 @@
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system'; import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
export class GenericInputReader { export class GenericInputReader {
public static async Run(command: string) { public static async Run(command: string) {
if (CloudRunnerOptions.providerStrategy === 'local') { if (OrchestratorOptions.providerStrategy === 'local') {
return ''; return '';
} }
return await CloudRunnerSystem.Run(command, false, true); return await OrchestratorSystem.Run(command, false, true);
} }
} }

View File

@@ -1,6 +1,6 @@
import { GitRepoReader } from './git-repo'; import { GitRepoReader } from './git-repo';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system'; import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
describe(`git repo tests`, () => { describe(`git repo tests`, () => {
it(`Branch value parsed from CLI to not contain illegal characters`, async () => { it(`Branch value parsed from CLI to not contain illegal characters`, async () => {
@@ -10,15 +10,15 @@ describe(`git repo tests`, () => {
it(`returns valid branch name when using https`, async () => { it(`returns valid branch name when using https`, async () => {
const mockValue = 'https://github.com/example/example.git'; const mockValue = 'https://github.com/example/example.git';
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue)); await jest.spyOn(OrchestratorSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local'); await jest.spyOn(OrchestratorOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`); expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
}); });
it(`returns valid branch name when using ssh`, async () => { it(`returns valid branch name when using ssh`, async () => {
const mockValue = 'git@github.com:example/example.git'; const mockValue = 'git@github.com:example/example.git';
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue)); await jest.spyOn(OrchestratorSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local'); await jest.spyOn(OrchestratorOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`); expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
}); });
}); });

View File

@@ -1,33 +1,33 @@
import { assert } from 'node:console'; import { assert } from 'node:console';
import fs from 'node:fs'; import fs from 'node:fs';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system'; import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger'; import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
import Input from '../input'; import Input from '../input';
export class GitRepoReader { export class GitRepoReader {
public static async GetRemote() { public static async GetRemote() {
if (CloudRunnerOptions.providerStrategy === 'local') { if (OrchestratorOptions.providerStrategy === 'local') {
return ''; return '';
} }
assert(fs.existsSync(`.git`)); assert(fs.existsSync(`.git`));
const value = (await CloudRunnerSystem.Run(`cd ${Input.projectPath} && git remote -v`, false, true)).replace( const value = (await OrchestratorSystem.Run(`cd ${Input.projectPath} && git remote -v`, false, true)).replace(
/ /g, / /g,
``, ``,
); );
CloudRunnerLogger.log(`value ${value}`); OrchestratorLogger.log(`value ${value}`);
assert(value.includes('github.com')); assert(value.includes('github.com'));
return value.split('github.com')[1].split('.git')[0].slice(1); return value.split('github.com')[1].split('.git')[0].slice(1);
} }
public static async GetBranch() { public static async GetBranch() {
if (CloudRunnerOptions.providerStrategy === 'local') { if (OrchestratorOptions.providerStrategy === 'local') {
return ''; return '';
} }
assert(fs.existsSync(`.git`)); assert(fs.existsSync(`.git`));
return (await CloudRunnerSystem.Run(`cd ${Input.projectPath} && git branch --show-current`, false, true)) return (await OrchestratorSystem.Run(`cd ${Input.projectPath} && git branch --show-current`, false, true))
.split('\n')[0] .split('\n')[0]
.replace(/ /g, ``) .replace(/ /g, ``)
.replace('/head', ''); .replace('/head', '');

View File

@@ -1,19 +1,19 @@
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system'; import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
import * as core from '@actions/core'; import * as core from '@actions/core';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
export class GithubCliReader { export class GithubCliReader {
static async GetGitHubAuthToken() { static async GetGitHubAuthToken() {
if (CloudRunnerOptions.providerStrategy === 'local') { if (OrchestratorOptions.providerStrategy === 'local') {
return ''; return '';
} }
try { try {
const authStatus = await CloudRunnerSystem.Run(`gh auth status`, true, true); const authStatus = await OrchestratorSystem.Run(`gh auth status`, true, true);
if (authStatus.includes('You are not logged') || authStatus === '') { if (authStatus.includes('You are not logged') || authStatus === '') {
return ''; return '';
} }
return (await CloudRunnerSystem.Run(`gh auth status -t`, false, true)) return (await OrchestratorSystem.Run(`gh auth status -t`, false, true))
.split(`Token: `)[1] .split(`Token: `)[1]
.replace(/ /g, '') .replace(/ /g, '')
.replace(/\n/g, ''); .replace(/\n/g, '');

View File

@@ -1,13 +1,13 @@
import path from 'node:path'; import path from 'node:path';
import fs from 'node:fs'; import fs from 'node:fs';
import YAML from 'yaml'; import YAML from 'yaml';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options'; import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
export function ReadLicense(): string { export function ReadLicense(): string {
if (CloudRunnerOptions.providerStrategy === 'local') { if (OrchestratorOptions.providerStrategy === 'local') {
return ''; return '';
} }
const pipelineFile = path.join(__dirname, `.github`, `workflows`, `cloud-runner-k8s-pipeline.yml`); const pipelineFile = path.join(__dirname, `.github`, `workflows`, `orchestrator-k8s-pipeline.yml`);
return fs.existsSync(pipelineFile) ? YAML.parse(fs.readFileSync(pipelineFile, 'utf8')).env.UNITY_LICENSE : ''; return fs.existsSync(pipelineFile) ? YAML.parse(fs.readFileSync(pipelineFile, 'utf8')).env.UNITY_LICENSE : '';
} }

View File

@@ -1,7 +1,7 @@
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import { Cli } from './cli/cli'; import { Cli } from './cli/cli';
import CloudRunnerQueryOverride from './cloud-runner/options/cloud-runner-query-override'; import OrchestratorQueryOverride from './orchestrator/options/orchestrator-query-override';
import Platform from './platform'; import Platform from './platform';
import GitHub from './github'; import GitHub from './github';
import os from 'node:os'; import os from 'node:os';
@@ -32,8 +32,8 @@ class Input {
return Cli.query(query, alternativeQuery); return Cli.query(query, alternativeQuery);
} }
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) { if (OrchestratorQueryOverride.query(query, alternativeQuery)) {
return CloudRunnerQueryOverride.query(query, alternativeQuery); return OrchestratorQueryOverride.query(query, alternativeQuery);
} }
if (process.env[query] !== undefined) { if (process.env[query] !== undefined) {

View File

@@ -0,0 +1,15 @@
import OrchestratorLogger from '../services/core/orchestrator-logger';
import * as core from '@actions/core';
import Orchestrator from '../orchestrator';
import OrchestratorSecret from '../options/orchestrator-secret';
import BuildParameters from '../../build-parameters';
export class OrchestratorError {
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: OrchestratorSecret[]) {
OrchestratorLogger.error(JSON.stringify(error, undefined, 4));
core.setFailed('Orchestrator failed');
if (Orchestrator.Provider !== undefined) {
await Orchestrator.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
}
}
}

View File

@@ -0,0 +1,4 @@
class OrchestratorConstants {
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
}
export default OrchestratorConstants;

View File

@@ -0,0 +1,5 @@
class OrchestratorEnvironmentVariable {
public name!: string;
public value!: string;
}
export default OrchestratorEnvironmentVariable;

View File

@@ -0,0 +1,90 @@
import path from 'node:path';
import OrchestratorOptions from './orchestrator-options';
import Orchestrator from '../orchestrator';
import BuildParameters from '../../build-parameters';
export class OrchestratorFolders {
public static readonly repositoryFolder = 'repo';
public static ToLinuxFolder(folder: string) {
return folder.replace(/\\/g, `/`);
}
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueOrchestratorJobFolderAbsolute(): string {
return Orchestrator.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)
? path.join(`/`, OrchestratorFolders.buildVolumeFolder, Orchestrator.lockedWorkspace)
: path.join(`/`, OrchestratorFolders.buildVolumeFolder, Orchestrator.buildParameters.buildGuid);
}
public static get cacheFolderForAllFull(): string {
return path.join('/', OrchestratorFolders.buildVolumeFolder, OrchestratorFolders.cacheFolder);
}
public static get cacheFolderForCacheKeyFull(): string {
return path.join(
'/',
OrchestratorFolders.buildVolumeFolder,
OrchestratorFolders.cacheFolder,
Orchestrator.buildParameters.cacheKey,
);
}
public static get builderPathAbsolute(): string {
return path.join(
OrchestratorOptions.useSharedBuilder
? `/${OrchestratorFolders.buildVolumeFolder}`
: OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute,
`builder`,
);
}
public static get repoPathAbsolute(): string {
return path.join(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute, OrchestratorFolders.repositoryFolder);
}
public static get projectPathAbsolute(): string {
return path.join(OrchestratorFolders.repoPathAbsolute, Orchestrator.buildParameters.projectPath);
}
public static get libraryFolderAbsolute(): string {
return path.join(OrchestratorFolders.projectPathAbsolute, `Library`);
}
public static get projectBuildFolderAbsolute(): string {
return path.join(OrchestratorFolders.repoPathAbsolute, Orchestrator.buildParameters.buildPath);
}
public static get lfsFolderAbsolute(): string {
return path.join(OrchestratorFolders.repoPathAbsolute, `.git`, `lfs`);
}
public static get purgeRemoteCaching(): boolean {
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
}
public static get lfsCacheFolderFull() {
return path.join(OrchestratorFolders.cacheFolderForCacheKeyFull, `lfs`);
}
public static get libraryCacheFolderFull() {
return path.join(OrchestratorFolders.cacheFolderForCacheKeyFull, `Library`);
}
public static get unityBuilderRepoUrl(): string {
return `https://${Orchestrator.buildParameters.gitPrivateToken}@github.com/${Orchestrator.buildParameters.orchestratorRepoName}.git`;
}
public static get targetBuildRepoUrl(): string {
return `https://${Orchestrator.buildParameters.gitPrivateToken}@github.com/${Orchestrator.buildParameters.githubRepo}.git`;
}
public static get buildVolumeFolder() {
return 'data';
}
public static get cacheFolder() {
return 'cache';
}
}

View File

@@ -1,11 +1,11 @@
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import CloudRunnerConstants from './cloud-runner-constants'; import OrchestratorConstants from './orchestrator-constants';
class CloudRunnerNamespace { class OrchestratorNamespace {
static generateGuid(runNumber: string | number, platform: string) { static generateGuid(runNumber: string | number, platform: string) {
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4); const nanoid = customAlphabet(OrchestratorConstants.alphabet, 4);
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`; return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
} }
} }
export default CloudRunnerNamespace; export default OrchestratorNamespace;

View File

@@ -0,0 +1,10 @@
import Input from '../../input';
import OrchestratorOptions from './orchestrator-options';
class OrchestratorOptionsReader {
static GetProperties() {
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(OrchestratorOptions)];
}
}
export default OrchestratorOptionsReader;

View File

@@ -0,0 +1,338 @@
import { Cli } from '../../cli/cli';
import OrchestratorQueryOverride from './orchestrator-query-override';
import GitHub from '../../github';
import * as core from '@actions/core';
class OrchestratorOptions {
// ### ### ###
// Input Handling
// ### ### ###
public static getInput(query: string): string | undefined {
if (GitHub.githubInputEnabled) {
const coreInput = core.getInput(query);
if (coreInput && coreInput !== '') {
return coreInput;
}
}
const alternativeQuery = OrchestratorOptions.ToEnvVarFormat(query);
// Query input sources
if (Cli.query(query, alternativeQuery)) {
return Cli.query(query, alternativeQuery);
}
if (OrchestratorQueryOverride.query(query, alternativeQuery)) {
return OrchestratorQueryOverride.query(query, alternativeQuery);
}
if (process.env[query] !== undefined) {
return process.env[query];
}
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
return process.env[alternativeQuery];
}
}
public static ToEnvVarFormat(input: string): string {
if (input.toUpperCase() === input) {
return input;
}
return input
.replace(/([A-Z])/g, ' $1')
.trim()
.toUpperCase()
.replace(/ /g, '_');
}
// ### ### ###
// Provider parameters
// ### ### ###
static get region(): string {
return OrchestratorOptions.getInput('region') || 'eu-west-2';
}
// ### ### ###
// GitHub parameters
// ### ### ###
static get githubChecks(): boolean {
const value = OrchestratorOptions.getInput('githubChecks');
return value === `true` || false;
}
static get githubCheckId(): string {
return OrchestratorOptions.getInput('githubCheckId') || ``;
}
static get githubOwner(): string {
return OrchestratorOptions.getInput('githubOwner') || OrchestratorOptions.githubRepo?.split(`/`)[0] || '';
}
static get githubRepoName(): string {
return OrchestratorOptions.getInput('githubRepoName') || OrchestratorOptions.githubRepo?.split(`/`)[1] || '';
}
static get orchestratorRepoName(): string {
return OrchestratorOptions.getInput('orchestratorRepoName') || 'game-ci/unity-builder';
}
static get cloneDepth(): string {
return OrchestratorOptions.getInput('cloneDepth') || '50';
}
static get finalHooks(): string[] {
return OrchestratorOptions.getInput('finalHooks')?.split(',') || [];
}
// ### ### ###
// Git syncronization parameters
// ### ### ###
static get githubRepo(): string | undefined {
return (
OrchestratorOptions.getInput('GITHUB_REPOSITORY') || OrchestratorOptions.getInput('GITHUB_REPO') || undefined
);
}
static get branch(): string {
if (OrchestratorOptions.getInput(`GITHUB_REF`)) {
return (
OrchestratorOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') ||
``
);
} else if (OrchestratorOptions.getInput('branch')) {
return OrchestratorOptions.getInput('branch') || ``;
} else {
return '';
}
}
// ### ### ###
// Orchestrator parameters
// ### ### ###
static get buildPlatform(): string {
const input = OrchestratorOptions.getInput('buildPlatform');
if (input && input !== '') {
return input;
}
if (OrchestratorOptions.providerStrategy !== 'local') {
return 'linux';
}
return process.platform;
}
static get orchestratorBranch(): string {
return OrchestratorOptions.getInput('orchestratorBranch') || 'main';
}
static get providerStrategy(): string {
const provider =
OrchestratorOptions.getInput('orchestratorCluster') || OrchestratorOptions.getInput('providerStrategy');
if (Cli.isCliMode) {
return provider || 'aws';
}
return provider || 'local';
}
static get containerCpu(): string {
return OrchestratorOptions.getInput('containerCpu') || `1024`;
}
static get containerMemory(): string {
return OrchestratorOptions.getInput('containerMemory') || `3072`;
}
static get containerNamespace(): string {
return OrchestratorOptions.getInput('containerNamespace') || `default`;
}
static get customJob(): string {
return OrchestratorOptions.getInput('customJob') || '';
}
// ### ### ###
// Custom commands from files parameters
// ### ### ###
static get containerHookFiles(): string[] {
return OrchestratorOptions.getInput('containerHookFiles')?.split(`,`) || [];
}
static get commandHookFiles(): string[] {
return OrchestratorOptions.getInput('commandHookFiles')?.split(`,`) || [];
}
// ### ### ###
// Custom commands from yaml parameters
// ### ### ###
static get commandHooks(): string {
return OrchestratorOptions.getInput('commandHooks') || '';
}
static get postBuildContainerHooks(): string {
return OrchestratorOptions.getInput('postBuildContainerHooks') || '';
}
static get preBuildContainerHooks(): string {
return OrchestratorOptions.getInput('preBuildContainerHooks') || '';
}
// ### ### ###
// Input override handling
// ### ### ###
static get pullInputList(): string[] {
return OrchestratorOptions.getInput('pullInputList')?.split(`,`) || [];
}
static get inputPullCommand(): string {
const value = OrchestratorOptions.getInput('inputPullCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
} else if (value === 'aws-secret-manager') {
return 'aws secretsmanager get-secret-value --secret-id {0}';
}
return value || '';
}
// ### ### ###
// Aws
// ### ### ###
static get awsStackName() {
return OrchestratorOptions.getInput('awsStackName') || 'game-ci';
}
static get awsEndpoint(): string | undefined {
return OrchestratorOptions.getInput('awsEndpoint');
}
static get awsCloudFormationEndpoint(): string | undefined {
return OrchestratorOptions.getInput('awsCloudFormationEndpoint') || OrchestratorOptions.awsEndpoint;
}
static get awsEcsEndpoint(): string | undefined {
return OrchestratorOptions.getInput('awsEcsEndpoint') || OrchestratorOptions.awsEndpoint;
}
static get awsKinesisEndpoint(): string | undefined {
return OrchestratorOptions.getInput('awsKinesisEndpoint') || OrchestratorOptions.awsEndpoint;
}
static get awsCloudWatchLogsEndpoint(): string | undefined {
return OrchestratorOptions.getInput('awsCloudWatchLogsEndpoint') || OrchestratorOptions.awsEndpoint;
}
static get awsS3Endpoint(): string | undefined {
return OrchestratorOptions.getInput('awsS3Endpoint') || OrchestratorOptions.awsEndpoint;
}
// ### ### ###
// Storage
// ### ### ###
static get storageProvider(): string {
return OrchestratorOptions.getInput('storageProvider') || 's3';
}
static get rcloneRemote(): string {
return OrchestratorOptions.getInput('rcloneRemote') || '';
}
// ### ### ###
// K8s
// ### ### ###
static get kubeConfig(): string {
return OrchestratorOptions.getInput('kubeConfig') || '';
}
static get kubeVolume(): string {
return OrchestratorOptions.getInput('kubeVolume') || '';
}
static get kubeVolumeSize(): string {
return OrchestratorOptions.getInput('kubeVolumeSize') || '25Gi';
}
static get kubeStorageClass(): string {
return OrchestratorOptions.getInput('kubeStorageClass') || '';
}
// ### ### ###
// Caching
// ### ### ###
static get cacheKey(): string {
return OrchestratorOptions.getInput('cacheKey') || OrchestratorOptions.branch;
}
// ### ### ###
// Utility Parameters
// ### ### ###
static get orchestratorDebug(): boolean {
return (
OrchestratorOptions.getInput(`orchestratorTests`) === `true` ||
OrchestratorOptions.getInput(`orchestratorDebug`) === `true` ||
OrchestratorOptions.getInput(`orchestratorDebugTree`) === `true` ||
OrchestratorOptions.getInput(`orchestratorDebugEnv`) === `true` ||
false
);
}
static get skipLfs(): boolean {
return OrchestratorOptions.getInput(`skipLfs`) === `true`;
}
static get skipCache(): boolean {
return OrchestratorOptions.getInput(`skipCache`) === `true`;
}
public static get asyncOrchestrator(): boolean {
return OrchestratorOptions.getInput('asyncOrchestrator') === 'true';
}
public static get resourceTracking(): boolean {
return OrchestratorOptions.getInput('resourceTracking') === 'true';
}
public static get useLargePackages(): boolean {
return OrchestratorOptions.getInput(`useLargePackages`) === `true`;
}
public static get useSharedBuilder(): boolean {
return OrchestratorOptions.getInput(`useSharedBuilder`) === `true`;
}
public static get useCompressionStrategy(): boolean {
return OrchestratorOptions.getInput(`useCompressionStrategy`) === `true`;
}
public static get useCleanupCron(): boolean {
return (OrchestratorOptions.getInput(`useCleanupCron`) || 'true') === 'true';
}
// ### ### ###
// Retained Workspace
// ### ### ###
public static get maxRetainedWorkspaces(): string {
return OrchestratorOptions.getInput(`maxRetainedWorkspaces`) || `0`;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get garbageMaxAge(): number {
return Number(OrchestratorOptions.getInput(`garbageMaxAge`)) || 24;
}
}
export default OrchestratorOptions;

View File

@@ -0,0 +1,67 @@
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import OrchestratorOptions from './orchestrator-options';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
value = value.replace(`{${element.key}}`, element.value);
}
return value;
};
class OrchestratorQueryOverride {
static queryOverrides: { [key: string]: string } | undefined;
// TODO accept premade secret sources or custom secret source definition yamls
public static query(key: string, alternativeKey: string) {
if (OrchestratorQueryOverride.queryOverrides && OrchestratorQueryOverride.queryOverrides[key] !== undefined) {
return OrchestratorQueryOverride.queryOverrides[key];
}
if (
OrchestratorQueryOverride.queryOverrides &&
alternativeKey &&
OrchestratorQueryOverride.queryOverrides[alternativeKey] !== undefined
) {
return OrchestratorQueryOverride.queryOverrides[alternativeKey];
}
return;
}
private static shouldUseOverride(query: string) {
if (OrchestratorOptions.inputPullCommand !== '') {
if (OrchestratorOptions.pullInputList.length > 0) {
const doesInclude =
OrchestratorOptions.pullInputList.includes(query) ||
OrchestratorOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
return true;
}
}
}
private static async queryOverride(query: string) {
if (!this.shouldUseOverride(query)) {
throw new Error(`Should not be trying to run override query on ${query}`);
}
return await GenericInputReader.Run(
formatFunction(OrchestratorOptions.inputPullCommand, [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = OrchestratorOptions.pullInputList;
OrchestratorQueryOverride.queryOverrides = {};
for (const element of queries) {
if (OrchestratorQueryOverride.shouldUseOverride(element)) {
OrchestratorQueryOverride.queryOverrides[element] = await OrchestratorQueryOverride.queryOverride(element);
}
}
}
}
export default OrchestratorQueryOverride;

View File

@@ -1,6 +1,6 @@
class CloudRunnerSecret { class OrchestratorSecret {
public ParameterKey!: string; public ParameterKey!: string;
public EnvironmentVariable!: string; public EnvironmentVariable!: string;
public ParameterValue!: string; public ParameterValue!: string;
} }
export default CloudRunnerSecret; export default OrchestratorSecret;

View File

@@ -0,0 +1,3 @@
export class OrchestratorStatics {
public static readonly logPrefix = `Orchestrator`;
}

View File

@@ -0,0 +1,13 @@
import OrchestratorEnvironmentVariable from './orchestrator-environment-variable';
import OrchestratorSecret from './orchestrator-secret';
export class OrchestratorStepParameters {
public image: string;
public environment: OrchestratorEnvironmentVariable[];
public secrets: OrchestratorSecret[];
constructor(image: string, environmentVariables: OrchestratorEnvironmentVariable[], secrets: OrchestratorSecret[]) {
this.image = image;
this.environment = environmentVariables;
this.secrets = secrets;
}
}

View File

@@ -1,70 +1,72 @@
import AwsBuildPlatform from './providers/aws'; import AwsBuildPlatform from './providers/aws';
import { BuildParameters, Input } from '..'; import { BuildParameters, Input } from '..';
import Kubernetes from './providers/k8s'; import Kubernetes from './providers/k8s';
import CloudRunnerLogger from './services/core/cloud-runner-logger'; import OrchestratorLogger from './services/core/orchestrator-logger';
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters'; import { OrchestratorStepParameters } from './options/orchestrator-step-parameters';
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root'; import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
import { CloudRunnerError } from './error/cloud-runner-error'; import { OrchestratorError } from './error/orchestrator-error';
import { TaskParameterSerializer } from './services/core/task-parameter-serializer'; import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
import * as core from '@actions/core'; import * as core from '@actions/core';
import CloudRunnerSecret from './options/cloud-runner-secret'; import OrchestratorSecret from './options/orchestrator-secret';
import { ProviderInterface } from './providers/provider-interface'; import { ProviderInterface } from './providers/provider-interface';
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from './options/orchestrator-environment-variable';
import TestCloudRunner from './providers/test'; import TestOrchestrator from './providers/test';
import LocalCloudRunner from './providers/local'; import LocalOrchestrator from './providers/local';
import LocalDockerCloudRunner from './providers/docker'; import LocalDockerOrchestrator from './providers/docker';
import loadProvider from './providers/provider-loader'; import loadProvider from './providers/provider-loader';
import GitHub from '../github'; import GitHub from '../github';
import SharedWorkspaceLocking from './services/core/shared-workspace-locking'; import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
import { FollowLogStreamService } from './services/core/follow-log-stream-service'; import { FollowLogStreamService } from './services/core/follow-log-stream-service';
import CloudRunnerResult from './services/core/cloud-runner-result'; import OrchestratorResult from './services/core/orchestrator-result';
import CloudRunnerOptions from './options/cloud-runner-options'; import OrchestratorOptions from './options/orchestrator-options';
import ResourceTracking from './services/core/resource-tracking'; import ResourceTracking from './services/core/resource-tracking';
class CloudRunner { class Orchestrator {
public static Provider: ProviderInterface; public static Provider: ProviderInterface;
public static buildParameters: BuildParameters; public static buildParameters: BuildParameters;
private static defaultSecrets: CloudRunnerSecret[]; private static defaultSecrets: OrchestratorSecret[];
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[]; private static orchestratorEnvironmentVariables: OrchestratorEnvironmentVariable[];
static lockedWorkspace: string = ``; static lockedWorkspace: string = ``;
public static readonly retainedWorkspacePrefix: string = `retained-workspace`; public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
// When true, validates AWS CloudFormation templates even when using local-docker execution // When true, validates AWS CloudFormation templates even when using local-docker execution
// This is set by AWS_FORCE_PROVIDER=aws-local mode // This is set by AWS_FORCE_PROVIDER=aws-local mode
public static validateAwsTemplates: boolean = false; public static validateAwsTemplates: boolean = false;
public static get isCloudRunnerEnvironment() { public static get isOrchestratorEnvironment() {
return process.env[`GITHUB_ACTIONS`] !== `true`; return process.env[`GITHUB_ACTIONS`] !== `true`;
} }
public static get isCloudRunnerAsyncEnvironment() { public static get isOrchestratorAsyncEnvironment() {
return process.env[`ASYNC_WORKFLOW`] === `true`; return process.env[`ASYNC_WORKFLOW`] === `true`;
} }
public static async setup(buildParameters: BuildParameters) { public static async setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup(); OrchestratorLogger.setup();
CloudRunnerLogger.log(`Setting up cloud runner`); OrchestratorLogger.log(`Setting up orchestrator`);
CloudRunner.buildParameters = buildParameters; Orchestrator.buildParameters = buildParameters;
ResourceTracking.logAllocationSummary('setup'); ResourceTracking.logAllocationSummary('setup');
await ResourceTracking.logDiskUsageSnapshot('setup'); await ResourceTracking.logDiskUsageSnapshot('setup');
if (CloudRunner.buildParameters.githubCheckId === ``) { if (Orchestrator.buildParameters.githubCheckId === ``) {
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid); Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(
Orchestrator.buildParameters.buildGuid,
);
} }
await CloudRunner.setupSelectedBuildPlatform(); await Orchestrator.setupSelectedBuildPlatform();
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets(); Orchestrator.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
CloudRunner.cloudRunnerEnvironmentVariables = Orchestrator.orchestratorEnvironmentVariables =
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters); TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameters);
if (GitHub.githubInputEnabled) { if (GitHub.githubInputEnabled) {
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters); const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) { for (const element of Orchestrator.orchestratorEnvironmentVariables) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`); // OrchestratorLogger.log(`Orchestrator output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
core.setOutput(Input.ToEnvVarFormat(element.name), element.value); core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
} }
for (const element of buildParameterPropertyNames) { for (const element of buildParameterPropertyNames) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`); // OrchestratorLogger.log(`Orchestrator output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]); core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
} }
core.setOutput( core.setOutput(
Input.ToEnvVarFormat(`buildArtifact`), Input.ToEnvVarFormat(`buildArtifact`),
`build-${CloudRunner.buildParameters.buildGuid}.tar${ `build-${Orchestrator.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`, }`,
); );
} }
@@ -72,7 +74,7 @@ class CloudRunner {
} }
private static async setupSelectedBuildPlatform() { private static async setupSelectedBuildPlatform() {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`); OrchestratorLogger.log(`Orchestrator platform selected ${Orchestrator.buildParameters.providerStrategy}`);
// Detect LocalStack endpoints and handle AWS provider appropriately // Detect LocalStack endpoints and handle AWS provider appropriately
// AWS_FORCE_PROVIDER options: // AWS_FORCE_PROVIDER options:
@@ -89,35 +91,35 @@ class CloudRunner {
process.env.AWS_ECS_ENDPOINT, process.env.AWS_ECS_ENDPOINT,
process.env.AWS_KINESIS_ENDPOINT, process.env.AWS_KINESIS_ENDPOINT,
process.env.AWS_CLOUD_WATCH_LOGS_ENDPOINT, process.env.AWS_CLOUD_WATCH_LOGS_ENDPOINT,
CloudRunnerOptions.awsEndpoint, OrchestratorOptions.awsEndpoint,
CloudRunnerOptions.awsS3Endpoint, OrchestratorOptions.awsS3Endpoint,
CloudRunnerOptions.awsCloudFormationEndpoint, OrchestratorOptions.awsCloudFormationEndpoint,
CloudRunnerOptions.awsEcsEndpoint, OrchestratorOptions.awsEcsEndpoint,
CloudRunnerOptions.awsKinesisEndpoint, OrchestratorOptions.awsKinesisEndpoint,
CloudRunnerOptions.awsCloudWatchLogsEndpoint, OrchestratorOptions.awsCloudWatchLogsEndpoint,
] ]
.filter((x) => typeof x === 'string') .filter((x) => typeof x === 'string')
.join(' '); .join(' ');
const isLocalStack = /localstack|localhost|127\.0\.0\.1/i.test(endpointsToCheck); const isLocalStack = /localstack|localhost|127\.0\.0\.1/i.test(endpointsToCheck);
let provider = CloudRunner.buildParameters.providerStrategy; let provider = Orchestrator.buildParameters.providerStrategy;
let validateAwsTemplates = false; let validateAwsTemplates = false;
if (provider === 'aws' && isLocalStack) { if (provider === 'aws' && isLocalStack) {
if (useAwsLocalMode) { if (useAwsLocalMode) {
// aws-local mode: Validate AWS templates but execute via local-docker // aws-local mode: Validate AWS templates but execute via local-docker
// This provides confidence in AWS CloudFormation without requiring LocalStack Pro // This provides confidence in AWS CloudFormation without requiring LocalStack Pro
CloudRunnerLogger.log('AWS_FORCE_PROVIDER=aws-local: Validating AWS templates, executing via local-docker'); OrchestratorLogger.log('AWS_FORCE_PROVIDER=aws-local: Validating AWS templates, executing via local-docker');
validateAwsTemplates = true; validateAwsTemplates = true;
provider = 'local-docker'; provider = 'local-docker';
} else if (forceAwsProvider) { } else if (forceAwsProvider) {
// Force full AWS provider (requires LocalStack Pro with ECS support) // Force full AWS provider (requires LocalStack Pro with ECS support)
CloudRunnerLogger.log( OrchestratorLogger.log(
'LocalStack endpoints detected but AWS_FORCE_PROVIDER=aws; using full AWS provider (requires ECS support)', 'LocalStack endpoints detected but AWS_FORCE_PROVIDER=aws; using full AWS provider (requires ECS support)',
); );
} else { } else {
// Auto-fallback to local-docker // Auto-fallback to local-docker
CloudRunnerLogger.log('LocalStack endpoints detected; routing provider to local-docker for this run'); OrchestratorLogger.log('LocalStack endpoints detected; routing provider to local-docker for this run');
CloudRunnerLogger.log( OrchestratorLogger.log(
'Note: Set AWS_FORCE_PROVIDER=aws-local to validate AWS templates with local-docker execution', 'Note: Set AWS_FORCE_PROVIDER=aws-local to validate AWS templates with local-docker execution',
); );
provider = 'local-docker'; provider = 'local-docker';
@@ -125,54 +127,54 @@ class CloudRunner {
} }
// Store whether we should validate AWS templates (used by aws-local mode) // Store whether we should validate AWS templates (used by aws-local mode)
CloudRunner.validateAwsTemplates = validateAwsTemplates; Orchestrator.validateAwsTemplates = validateAwsTemplates;
switch (provider) { switch (provider) {
case 'k8s': case 'k8s':
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters); Orchestrator.Provider = new Kubernetes(Orchestrator.buildParameters);
break; break;
case 'aws': case 'aws':
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters); Orchestrator.Provider = new AwsBuildPlatform(Orchestrator.buildParameters);
// Validate that AWS provider is actually being used when expected // Validate that AWS provider is actually being used when expected
if (isLocalStack && forceAwsProvider) { if (isLocalStack && forceAwsProvider) {
CloudRunnerLogger.log('✓ AWS provider initialized with LocalStack - AWS functionality will be validated'); OrchestratorLogger.log('✓ AWS provider initialized with LocalStack - AWS functionality will be validated');
} else if (isLocalStack && !forceAwsProvider) { } else if (isLocalStack && !forceAwsProvider) {
CloudRunnerLogger.log( OrchestratorLogger.log(
'⚠ WARNING: AWS provider was requested but LocalStack detected without AWS_FORCE_PROVIDER', '⚠ WARNING: AWS provider was requested but LocalStack detected without AWS_FORCE_PROVIDER',
); );
CloudRunnerLogger.log('⚠ This may cause AWS functionality tests to fail validation'); OrchestratorLogger.log('⚠ This may cause AWS functionality tests to fail validation');
} }
break; break;
case 'test': case 'test':
CloudRunner.Provider = new TestCloudRunner(); Orchestrator.Provider = new TestOrchestrator();
break; break;
case 'local-docker': case 'local-docker':
CloudRunner.Provider = new LocalDockerCloudRunner(); Orchestrator.Provider = new LocalDockerOrchestrator();
break; break;
case 'local-system': case 'local-system':
CloudRunner.Provider = new LocalCloudRunner(); Orchestrator.Provider = new LocalOrchestrator();
break; break;
case 'local': case 'local':
CloudRunner.Provider = new LocalCloudRunner(); Orchestrator.Provider = new LocalOrchestrator();
break; break;
default: default:
// Try to load provider using the dynamic loader for unknown providers // Try to load provider using the dynamic loader for unknown providers
try { try {
CloudRunner.Provider = await loadProvider(provider, CloudRunner.buildParameters); Orchestrator.Provider = await loadProvider(provider, Orchestrator.buildParameters);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to load provider '${provider}' using dynamic loader: ${error.message}`); OrchestratorLogger.log(`Failed to load provider '${provider}' using dynamic loader: ${error.message}`);
CloudRunnerLogger.log('Falling back to local provider...'); OrchestratorLogger.log('Falling back to local provider...');
CloudRunner.Provider = new LocalCloudRunner(); Orchestrator.Provider = new LocalOrchestrator();
} }
break; break;
} }
// Final validation: Ensure provider matches expectations // Final validation: Ensure provider matches expectations
const finalProviderName = CloudRunner.Provider.constructor.name; const finalProviderName = Orchestrator.Provider.constructor.name;
if (CloudRunner.buildParameters.providerStrategy === 'aws' && finalProviderName !== 'AWSBuildEnvironment') { if (Orchestrator.buildParameters.providerStrategy === 'aws' && finalProviderName !== 'AWSBuildEnvironment') {
CloudRunnerLogger.log(`⚠ WARNING: Expected AWS provider but got ${finalProviderName}`); OrchestratorLogger.log(`⚠ WARNING: Expected AWS provider but got ${finalProviderName}`);
CloudRunnerLogger.log('⚠ AWS functionality tests may not be validating AWS services correctly'); OrchestratorLogger.log('⚠ AWS functionality tests may not be validating AWS services correctly');
} }
} }
@@ -180,67 +182,67 @@ class CloudRunner {
if (baseImage.includes(`undefined`)) { if (baseImage.includes(`undefined`)) {
throw new Error(`baseImage is undefined`); throw new Error(`baseImage is undefined`);
} }
await CloudRunner.setup(buildParameters); await Orchestrator.setup(buildParameters);
// When aws-local mode is enabled, validate AWS CloudFormation templates // When aws-local mode is enabled, validate AWS CloudFormation templates
// This ensures AWS templates are correct even when executing via local-docker // This ensures AWS templates are correct even when executing via local-docker
if (CloudRunner.validateAwsTemplates) { if (Orchestrator.validateAwsTemplates) {
await CloudRunner.validateAwsCloudFormationTemplates(); await Orchestrator.validateAwsCloudFormationTemplates();
} }
await CloudRunner.Provider.setupWorkflow( await Orchestrator.Provider.setupWorkflow(
CloudRunner.buildParameters.buildGuid, Orchestrator.buildParameters.buildGuid,
CloudRunner.buildParameters, Orchestrator.buildParameters,
CloudRunner.buildParameters.branch, Orchestrator.buildParameters.branch,
CloudRunner.defaultSecrets, Orchestrator.defaultSecrets,
); );
try { try {
if (buildParameters.maxRetainedWorkspaces > 0) { if (buildParameters.maxRetainedWorkspaces > 0) {
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName(); Orchestrator.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
const result = await SharedWorkspaceLocking.GetLockedWorkspace( const result = await SharedWorkspaceLocking.GetLockedWorkspace(
CloudRunner.lockedWorkspace, Orchestrator.lockedWorkspace,
CloudRunner.buildParameters.buildGuid, Orchestrator.buildParameters.buildGuid,
CloudRunner.buildParameters, Orchestrator.buildParameters,
); );
if (result) { if (result) {
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`); OrchestratorLogger.logLine(`Using retained workspace ${Orchestrator.lockedWorkspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [ Orchestrator.orchestratorEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables, ...Orchestrator.orchestratorEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace }, { name: `LOCKED_WORKSPACE`, value: Orchestrator.lockedWorkspace },
]; ];
} else { } else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`); OrchestratorLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.maxRetainedWorkspaces = 0; buildParameters.maxRetainedWorkspaces = 0;
CloudRunner.lockedWorkspace = ``; Orchestrator.lockedWorkspace = ``;
} }
} }
await CloudRunner.updateStatusWithBuildParameters(); await Orchestrator.updateStatusWithBuildParameters();
const output = await new WorkflowCompositionRoot().run( const output = await new WorkflowCompositionRoot().run(
new CloudRunnerStepParameters( new OrchestratorStepParameters(
baseImage, baseImage,
CloudRunner.cloudRunnerEnvironmentVariables, Orchestrator.orchestratorEnvironmentVariables,
CloudRunner.defaultSecrets, Orchestrator.defaultSecrets,
), ),
); );
await CloudRunner.Provider.cleanupWorkflow( await Orchestrator.Provider.cleanupWorkflow(
CloudRunner.buildParameters, Orchestrator.buildParameters,
CloudRunner.buildParameters.branch, Orchestrator.buildParameters.branch,
CloudRunner.defaultSecrets, Orchestrator.defaultSecrets,
); );
if (!CloudRunner.buildParameters.isCliMode) core.endGroup(); if (!Orchestrator.buildParameters.isCliMode) core.endGroup();
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) { if (buildParameters.asyncWorkflow && this.isOrchestratorEnvironment && this.isOrchestratorAsyncEnvironment) {
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`); await GitHub.updateGitHubCheck(Orchestrator.buildParameters.buildGuid, `success`, `success`, `completed`);
} }
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) { if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``; const workspace = Orchestrator.lockedWorkspace || ``;
await SharedWorkspaceLocking.ReleaseWorkspace( await SharedWorkspaceLocking.ReleaseWorkspace(
workspace, workspace,
CloudRunner.buildParameters.buildGuid, Orchestrator.buildParameters.buildGuid,
CloudRunner.buildParameters, Orchestrator.buildParameters,
); );
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters); const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, Orchestrator.buildParameters);
if (isLocked) { if (isLocked) {
throw new Error( throw new Error(
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace( `still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
@@ -249,38 +251,38 @@ class CloudRunner {
)}`, )}`,
); );
} }
CloudRunner.lockedWorkspace = ``; Orchestrator.lockedWorkspace = ``;
} }
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks); await GitHub.triggerWorkflowOnComplete(Orchestrator.buildParameters.finalHooks);
if (buildParameters.constantGarbageCollection) { if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true); Orchestrator.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
} }
return new CloudRunnerResult(buildParameters, output, true, true, false); return new OrchestratorResult(buildParameters, output, true, true, false);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4)); OrchestratorLogger.log(JSON.stringify(error, undefined, 4));
await GitHub.updateGitHubCheck( await GitHub.updateGitHubCheck(
CloudRunner.buildParameters.buildGuid, Orchestrator.buildParameters.buildGuid,
`Failed - Error ${error?.message || error}`, `Failed - Error ${error?.message || error}`,
`failure`, `failure`,
`completed`, `completed`,
); );
if (!CloudRunner.buildParameters.isCliMode) core.endGroup(); if (!Orchestrator.buildParameters.isCliMode) core.endGroup();
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets); await OrchestratorError.handleException(error, Orchestrator.buildParameters, Orchestrator.defaultSecrets);
throw error; throw error;
} }
} }
private static async updateStatusWithBuildParameters() { private static async updateStatusWithBuildParameters() {
const content = { ...CloudRunner.buildParameters }; const content = { ...Orchestrator.buildParameters };
content.gitPrivateToken = ``; content.gitPrivateToken = ``;
content.unitySerial = ``; content.unitySerial = ``;
content.unityEmail = ``; content.unityEmail = ``;
content.unityPassword = ``; content.unityPassword = ``;
const jsonContent = JSON.stringify(content, undefined, 4); const jsonContent = JSON.stringify(content, undefined, 4);
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid); await GitHub.updateGitHubCheck(jsonContent, Orchestrator.buildParameters.buildGuid);
} }
/** /**
@@ -289,7 +291,7 @@ class CloudRunner {
* This provides confidence that AWS ECS deployments would work with the generated templates. * This provides confidence that AWS ECS deployments would work with the generated templates.
*/ */
private static async validateAwsCloudFormationTemplates() { private static async validateAwsCloudFormationTemplates() {
CloudRunnerLogger.log('=== AWS CloudFormation Template Validation (aws-local mode) ==='); OrchestratorLogger.log('=== AWS CloudFormation Template Validation (aws-local mode) ===');
try { try {
// Import AWS template formations // Import AWS template formations
@@ -298,13 +300,13 @@ class CloudRunner {
// Validate base stack template // Validate base stack template
const baseTemplate = BaseStackFormation.formation; const baseTemplate = BaseStackFormation.formation;
CloudRunnerLogger.log(`✓ Base stack template generated (${baseTemplate.length} chars)`); OrchestratorLogger.log(`✓ Base stack template generated (${baseTemplate.length} chars)`);
// Check for required resources in base stack // Check for required resources in base stack
const requiredBaseResources = ['AWS::EC2::VPC', 'AWS::ECS::Cluster', 'AWS::S3::Bucket', 'AWS::IAM::Role']; const requiredBaseResources = ['AWS::EC2::VPC', 'AWS::ECS::Cluster', 'AWS::S3::Bucket', 'AWS::IAM::Role'];
for (const resource of requiredBaseResources) { for (const resource of requiredBaseResources) {
if (baseTemplate.includes(resource)) { if (baseTemplate.includes(resource)) {
CloudRunnerLogger.log(` ✓ Contains ${resource}`); OrchestratorLogger.log(` ✓ Contains ${resource}`);
} else { } else {
throw new Error(`Base stack template missing required resource: ${resource}`); throw new Error(`Base stack template missing required resource: ${resource}`);
} }
@@ -312,13 +314,13 @@ class CloudRunner {
// Validate task definition template // Validate task definition template
const taskTemplate = TaskDefinitionFormation.formation; const taskTemplate = TaskDefinitionFormation.formation;
CloudRunnerLogger.log(`✓ Task definition template generated (${taskTemplate.length} chars)`); OrchestratorLogger.log(`✓ Task definition template generated (${taskTemplate.length} chars)`);
// Check for required resources in task definition // Check for required resources in task definition
const requiredTaskResources = ['AWS::ECS::TaskDefinition', 'AWS::Logs::LogGroup']; const requiredTaskResources = ['AWS::ECS::TaskDefinition', 'AWS::Logs::LogGroup'];
for (const resource of requiredTaskResources) { for (const resource of requiredTaskResources) {
if (taskTemplate.includes(resource)) { if (taskTemplate.includes(resource)) {
CloudRunnerLogger.log(` ✓ Contains ${resource}`); OrchestratorLogger.log(` ✓ Contains ${resource}`);
} else { } else {
throw new Error(`Task definition template missing required resource: ${resource}`); throw new Error(`Task definition template missing required resource: ${resource}`);
} }
@@ -332,12 +334,12 @@ class CloudRunner {
throw new Error('Task definition template missing AWSTemplateFormatVersion'); throw new Error('Task definition template missing AWSTemplateFormatVersion');
} }
CloudRunnerLogger.log('=== AWS CloudFormation templates validated successfully ==='); OrchestratorLogger.log('=== AWS CloudFormation templates validated successfully ===');
CloudRunnerLogger.log('Note: Actual execution will use local-docker provider'); OrchestratorLogger.log('Note: Actual execution will use local-docker provider');
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`AWS CloudFormation template validation failed: ${error.message}`); OrchestratorLogger.log(`AWS CloudFormation template validation failed: ${error.message}`);
throw error; throw error;
} }
} }
} }
export default CloudRunner; export default Orchestrator;

View File

@@ -2,9 +2,9 @@
## What is a Provider? ## What is a Provider?
A **provider** is a pluggable backend that Cloud Runner uses to run builds and workflows. Examples include **AWS**, **Kubernetes**, or local execution. Each provider implements the [ProviderInterface](https://github.com/game-ci/unity-builder/blob/main/src/model/cloud-runner/providers/provider-interface.ts), which defines the common lifecycle methods (setup, run, cleanup, garbage collection, etc.). A **provider** is a pluggable backend that Orchestrator uses to run builds and workflows. Examples include **AWS**, **Kubernetes**, or local execution. Each provider implements the [ProviderInterface](https://github.com/game-ci/unity-builder/blob/main/src/model/orchestrator/providers/provider-interface.ts), which defines the common lifecycle methods (setup, run, cleanup, garbage collection, etc.).
This abstraction makes Cloud Runner flexible: you can switch execution environments or add your own provider (via npm package, GitHub repo, or local path) without changing the rest of your pipeline. This abstraction makes Orchestrator flexible: you can switch execution environments or add your own provider (via npm package, GitHub repo, or local path) without changing the rest of your pipeline.
## Dynamic Provider Loading ## Dynamic Provider Loading

View File

@@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import * as core from '@actions/core'; import * as core from '@actions/core';
import { import {
CloudFormation, CloudFormation,
@@ -23,7 +23,7 @@ import crypto from 'node:crypto';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600; const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number { function getStackWaitTime(): number {
const overrideValue = Number(process.env.CLOUD_RUNNER_AWS_STACK_WAIT_TIME ?? ''); const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) { if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue; return overrideValue;
} }
@@ -87,7 +87,7 @@ export class AWSBaseStack {
}; };
try { try {
if (!stackExists) { if (!stackExists) {
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`); OrchestratorLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
let created = false; let created = false;
try { try {
await CF.send(new CreateStackCommand(createStackInput)); await CF.send(new CreateStackCommand(createStackInput));
@@ -95,13 +95,13 @@ export class AWSBaseStack {
} catch (error: any) { } catch (error: any) {
const message = `${error?.name ?? ''} ${error?.message ?? ''}`; const message = `${error?.name ?? ''} ${error?.message ?? ''}`;
if (message.includes('AlreadyExistsException')) { if (message.includes('AlreadyExistsException')) {
CloudRunnerLogger.log(`Base stack already exists, continuing with describe`); OrchestratorLogger.log(`Base stack already exists, continuing with describe`);
} else { } else {
throw error; throw error;
} }
} }
if (created) { if (created) {
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`); OrchestratorLogger.log(`created stack (version: ${parametersHash})`);
} }
} }
const CFState = await describeStack(); const CFState = await describeStack();
@@ -112,7 +112,7 @@ export class AWSBaseStack {
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue; const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
if (stack.StackStatus === 'CREATE_IN_PROGRESS') { if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`, `Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`,
); );
await waitUntilStackCreateComplete( await waitUntilStackCreateComplete(
@@ -125,22 +125,22 @@ export class AWSBaseStack {
} }
if (stackExists) { if (stackExists) {
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`); OrchestratorLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
if (parametersHash !== stackVersion) { if (parametersHash !== stackVersion) {
CloudRunnerLogger.log(`Attempting update of base stack`); OrchestratorLogger.log(`Attempting update of base stack`);
try { try {
await CF.send(new UpdateStackCommand(updateInput)); await CF.send(new UpdateStackCommand(updateInput));
} catch (error: any) { } catch (error: any) {
if (error['message'].includes('No updates are to be performed')) { if (error['message'].includes('No updates are to be performed')) {
CloudRunnerLogger.log(`No updates are to be performed`); OrchestratorLogger.log(`No updates are to be performed`);
} else { } else {
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`); OrchestratorLogger.log(`Update Failed (Stack name: ${baseStackName})`);
CloudRunnerLogger.log(error['message']); OrchestratorLogger.log(error['message']);
} }
CloudRunnerLogger.log(`Continuing...`); OrchestratorLogger.log(`Continuing...`);
} }
} else { } else {
CloudRunnerLogger.log(`No update required`); OrchestratorLogger.log(`No update required`);
} }
stack = (await describeStack()).Stacks?.[0]; stack = (await describeStack()).Stacks?.[0];
if (!stack) { if (!stack) {
@@ -149,7 +149,7 @@ export class AWSBaseStack {
); );
} }
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') { if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`, `Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`,
); );
await waitUntilStackUpdateComplete( await waitUntilStackUpdateComplete(
@@ -161,7 +161,7 @@ export class AWSBaseStack {
); );
} }
} }
CloudRunnerLogger.log('base stack is now ready'); OrchestratorLogger.log('base stack is now ready');
} catch (error) { } catch (error) {
core.error(JSON.stringify(await describeStack(), undefined, 4)); core.error(JSON.stringify(await describeStack(), undefined, 4));
throw error; throw error;

View File

@@ -4,7 +4,7 @@ import { Kinesis } from '@aws-sdk/client-kinesis';
import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs'; import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs';
import { S3 } from '@aws-sdk/client-s3'; import { S3 } from '@aws-sdk/client-s3';
import { Input } from '../../..'; import { Input } from '../../..';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
export class AwsClientFactory { export class AwsClientFactory {
private static cloudFormation: CloudFormation; private static cloudFormation: CloudFormation;
@@ -34,7 +34,7 @@ export class AwsClientFactory {
if (!this.cloudFormation) { if (!this.cloudFormation) {
this.cloudFormation = new CloudFormation({ this.cloudFormation = new CloudFormation({
region: Input.region, region: Input.region,
endpoint: CloudRunnerOptions.awsCloudFormationEndpoint, endpoint: OrchestratorOptions.awsCloudFormationEndpoint,
credentials: AwsClientFactory.getCredentials(), credentials: AwsClientFactory.getCredentials(),
}); });
} }
@@ -46,7 +46,7 @@ export class AwsClientFactory {
if (!this.ecs) { if (!this.ecs) {
this.ecs = new ECS({ this.ecs = new ECS({
region: Input.region, region: Input.region,
endpoint: CloudRunnerOptions.awsEcsEndpoint, endpoint: OrchestratorOptions.awsEcsEndpoint,
credentials: AwsClientFactory.getCredentials(), credentials: AwsClientFactory.getCredentials(),
}); });
} }
@@ -58,7 +58,7 @@ export class AwsClientFactory {
if (!this.kinesis) { if (!this.kinesis) {
this.kinesis = new Kinesis({ this.kinesis = new Kinesis({
region: Input.region, region: Input.region,
endpoint: CloudRunnerOptions.awsKinesisEndpoint, endpoint: OrchestratorOptions.awsKinesisEndpoint,
credentials: AwsClientFactory.getCredentials(), credentials: AwsClientFactory.getCredentials(),
}); });
} }
@@ -70,7 +70,7 @@ export class AwsClientFactory {
if (!this.cloudWatchLogs) { if (!this.cloudWatchLogs) {
this.cloudWatchLogs = new CloudWatchLogs({ this.cloudWatchLogs = new CloudWatchLogs({
region: Input.region, region: Input.region,
endpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint, endpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
credentials: AwsClientFactory.getCredentials(), credentials: AwsClientFactory.getCredentials(),
}); });
} }
@@ -82,7 +82,7 @@ export class AwsClientFactory {
if (!this.s3) { if (!this.s3) {
this.s3 = new S3({ this.s3 = new S3({
region: Input.region, region: Input.region,
endpoint: CloudRunnerOptions.awsS3Endpoint, endpoint: OrchestratorOptions.awsS3Endpoint,
forcePathStyle: true, forcePathStyle: true,
credentials: AwsClientFactory.getCredentials(), credentials: AwsClientFactory.getCredentials(),
}); });

View File

@@ -1,16 +1,16 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation'; import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
import * as core from '@actions/core'; import * as core from '@actions/core';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
export class AWSError { export class AWSError {
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) { static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
CloudRunnerLogger.log('aws error: '); OrchestratorLogger.log('aws error: ');
core.error(JSON.stringify(error, undefined, 4)); core.error(JSON.stringify(error, undefined, 4));
if (CloudRunner.buildParameters.cloudRunnerDebug) { if (Orchestrator.buildParameters.orchestratorDebug) {
CloudRunnerLogger.log('Getting events and resources for task stack'); OrchestratorLogger.log('Getting events and resources for task stack');
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents; const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4)); OrchestratorLogger.log(JSON.stringify(events, undefined, 4));
} }
} }
} }

View File

@@ -8,20 +8,20 @@ import {
ListStacksCommand, ListStacksCommand,
waitUntilStackCreateComplete, waitUntilStackCreateComplete,
} from '@aws-sdk/client-cloudformation'; } from '@aws-sdk/client-cloudformation';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def'; import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates'; import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { AWSError } from './aws-error'; import { AWSError } from './aws-error';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation'; import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation'; import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600; const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number { function getStackWaitTime(): number {
const overrideValue = Number(process.env.CLOUD_RUNNER_AWS_STACK_WAIT_TIME ?? ''); const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) { if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue; return overrideValue;
} }
@@ -43,23 +43,23 @@ export class AWSJobStack {
commands: string, commands: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<CloudRunnerAWSTaskDef> { ): Promise<OrchestratorAWSTaskDef> {
const taskDefStackName = `${this.baseStackName}-${buildGuid}`; const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate(); let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
taskDefCloudFormation = taskDefCloudFormation.replace( taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerCpu: `ContainerCpu:
Default: 1024`, Default: 1024`,
`ContainerCpu: `ContainerCpu:
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`, Default: ${Number.parseInt(Orchestrator.buildParameters.containerCpu)}`,
); );
taskDefCloudFormation = taskDefCloudFormation.replace( taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerMemory: `ContainerMemory:
Default: 2048`, Default: 2048`,
`ContainerMemory: `ContainerMemory:
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`, Default: ${Number.parseInt(Orchestrator.buildParameters.containerMemory)}`,
); );
if (!CloudRunnerOptions.asyncCloudRunner) { if (!OrchestratorOptions.asyncOrchestrator) {
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate( taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation, taskDefCloudFormation,
'# template resources logstream', '# template resources logstream',
@@ -133,8 +133,8 @@ export class AWSJobStack {
}, },
...secretsMappedToCloudFormationParameters, ...secretsMappedToCloudFormationParameters,
]; ];
CloudRunnerLogger.log( OrchestratorLogger.log(
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`, `Starting AWS job with memory: ${Orchestrator.buildParameters.containerMemory} cpu: ${Orchestrator.buildParameters.containerCpu}`,
); );
let previousStackExists = true; let previousStackExists = true;
while (previousStackExists) { while (previousStackExists) {
@@ -147,7 +147,7 @@ export class AWSJobStack {
const element = stacks.StackSummaries[index]; const element = stacks.StackSummaries[index];
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') { if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
previousStackExists = true; previousStackExists = true;
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`); OrchestratorLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
await new Promise((promise) => setTimeout(promise, 5000)); await new Promise((promise) => setTimeout(promise, 5000));
} }
} }
@@ -160,7 +160,7 @@ export class AWSJobStack {
}; };
try { try {
const stackWaitTimeSeconds = getStackWaitTime(); const stackWaitTimeSeconds = getStackWaitTime();
CloudRunnerLogger.log( OrchestratorLogger.log(
`Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`, `Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`,
); );
await CF.send(new CreateStackCommand(createStackInput)); await CF.send(new CreateStackCommand(createStackInput));
@@ -201,7 +201,7 @@ export class AWSJobStack {
}, },
{ {
ParameterKey: 'BUILDGUID', ParameterKey: 'BUILDGUID',
ParameterValue: CloudRunner.buildParameters.buildGuid, ParameterValue: Orchestrator.buildParameters.buildGuid,
}, },
{ {
ParameterKey: 'EnvironmentName', ParameterKey: 'EnvironmentName',
@@ -209,9 +209,9 @@ export class AWSJobStack {
}, },
], ],
}; };
if (CloudRunnerOptions.useCleanupCron) { if (OrchestratorOptions.useCleanupCron) {
try { try {
CloudRunnerLogger.log(`Creating job cleanup formation`); OrchestratorLogger.log(`Creating job cleanup formation`);
await CF.send(new CreateStackCommand(createCleanupStackInput)); await CF.send(new CreateStackCommand(createCleanupStackInput));
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise(); // await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();

View File

@@ -1,15 +1,15 @@
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs'; import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis'; import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import * as core from '@actions/core'; import * as core from '@actions/core';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def'; import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import * as zlib from 'node:zlib'; import * as zlib from 'node:zlib';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { Input } from '../../..'; import { Input } from '../../..';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { CommandHookService } from '../../services/hooks/command-hook-service'; import { CommandHookService } from '../../services/hooks/command-hook-service';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service'; import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import GitHub from '../../../github'; import GitHub from '../../../github';
import { AwsClientFactory } from './aws-client-factory'; import { AwsClientFactory } from './aws-client-factory';
@@ -22,8 +22,8 @@ class AWSTaskRunner {
* LocalStack on the host machine via host.docker.internal. * LocalStack on the host machine via host.docker.internal.
*/ */
private static transformEndpointsForContainer( private static transformEndpointsForContainer(
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
): CloudRunnerEnvironmentVariable[] { ): OrchestratorEnvironmentVariable[] {
const endpointEnvironmentNames = new Set([ const endpointEnvironmentNames = new Set([
'AWS_S3_ENDPOINT', 'AWS_S3_ENDPOINT',
'AWS_ENDPOINT', 'AWS_ENDPOINT',
@@ -46,7 +46,7 @@ class AWSTaskRunner {
value = value value = value
.replace('http://localhost', 'http://host.docker.internal') .replace('http://localhost', 'http://host.docker.internal')
.replace('http://127.0.0.1', 'http://host.docker.internal'); .replace('http://127.0.0.1', 'http://host.docker.internal');
CloudRunnerLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`); OrchestratorLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
} }
return { name: x.name, value }; return { name: x.name, value };
@@ -54,8 +54,8 @@ class AWSTaskRunner {
} }
static async runTask( static async runTask(
taskDef: CloudRunnerAWSTaskDef, taskDef: OrchestratorAWSTaskDef,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
commands: string, commands: string,
): Promise<{ output: string; shouldCleanup: boolean }> { ): Promise<{ output: string; shouldCleanup: boolean }> {
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || ''; const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
@@ -82,7 +82,7 @@ class AWSTaskRunner {
{ {
name: taskDef.taskDefStackName, name: taskDef.taskDefStackName,
environment: transformedEnvironment, environment: transformedEnvironment,
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)], command: ['-c', CommandHookService.ApplyHooksToCommands(commands, Orchestrator.buildParameters)],
}, },
], ],
}, },
@@ -97,28 +97,28 @@ class AWSTaskRunner {
}; };
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) { if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4)); OrchestratorLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
throw new Error(`Container Overrides length must be at most 8192`); throw new Error(`Container Overrides length must be at most 8192`);
} }
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any)); const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
const taskArn = task.tasks?.[0].taskArn || ''; const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting'); OrchestratorLogger.log('Orchestrator job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster); await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
CloudRunnerLogger.log( OrchestratorLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${ `Orchestrator job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
CloudRunnerOptions.asyncCloudRunner OrchestratorOptions.asyncOrchestrator
}`, }`,
); );
if (CloudRunnerOptions.asyncCloudRunner) { if (OrchestratorOptions.asyncOrchestrator) {
const shouldCleanup: boolean = false; const shouldCleanup: boolean = false;
const output: string = ''; const output: string = '';
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`); OrchestratorLogger.log(`Watch Orchestrator To End: false`);
return { output, shouldCleanup }; return { output, shouldCleanup };
} }
CloudRunnerLogger.log(`Streaming...`); OrchestratorLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName); const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
let exitCode; let exitCode;
let containerState; let containerState;
@@ -133,13 +133,13 @@ class AWSTaskRunner {
containerState = containers[0]; containerState = containers[0];
exitCode = containerState?.exitCode; exitCode = containerState?.exitCode;
} }
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`); OrchestratorLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
if (exitCode === undefined) { if (exitCode === undefined) {
CloudRunnerLogger.logWarning(`Undefined exitcode for container`); OrchestratorLogger.logWarning(`Undefined exitcode for container`);
} }
const wasSuccessful = exitCode === 0; const wasSuccessful = exitCode === 0;
if (wasSuccessful) { if (wasSuccessful) {
CloudRunnerLogger.log(`Cloud runner job has finished successfully`); OrchestratorLogger.log(`Orchestrator job has finished successfully`);
return { output, shouldCleanup }; return { output, shouldCleanup };
} }
@@ -166,7 +166,7 @@ class AWSTaskRunner {
const error = error_ as Error; const error = error_ as Error;
await new Promise((resolve) => setTimeout(resolve, 3000)); await new Promise((resolve) => setTimeout(resolve, 3000));
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn); const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
CloudRunnerLogger.log(`Cloud runner job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`); OrchestratorLogger.log(`Orchestrator job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
core.setFailed(error); core.setFailed(error);
core.error(error); core.error(error);
@@ -193,7 +193,7 @@ class AWSTaskRunner {
} }
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs)); const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
const sleepMs = delayMs + jitterMs; const sleepMs = delayMs + jitterMs;
CloudRunnerLogger.log( OrchestratorLogger.log(
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`, `AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
); );
await new Promise((r) => setTimeout(r, sleepMs)); await new Promise((r) => setTimeout(r, sleepMs));
@@ -204,12 +204,12 @@ class AWSTaskRunner {
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) { static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
await new Promise((resolve) => setTimeout(resolve, 3000)); await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(`Streaming...`); OrchestratorLogger.log(`Streaming...`);
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName); const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream); let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`; const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${Orchestrator.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${Orchestrator.buildParameters.awsStackName}-${Orchestrator.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`); OrchestratorLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``); await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
let shouldReadLogs = true; let shouldReadLogs = true;
let shouldCleanup = true; let shouldCleanup = true;
@@ -248,7 +248,7 @@ class AWSTaskRunner {
const baseBackoffMs = 1000; const baseBackoffMs = 1000;
const jitterMs = Math.floor(Math.random() * 1000); const jitterMs = Math.floor(Math.random() * 1000);
const sleepMs = baseBackoffMs + jitterMs; const sleepMs = baseBackoffMs + jitterMs;
CloudRunnerLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`); OrchestratorLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
await new Promise((r) => setTimeout(r, sleepMs)); await new Promise((r) => setTimeout(r, sleepMs));
return { iterator, shouldReadLogs, output, shouldCleanup }; return { iterator, shouldReadLogs, output, shouldCleanup };
@@ -269,18 +269,18 @@ class AWSTaskRunner {
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) { private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
if (taskData?.lastStatus === 'UNKNOWN') { if (taskData?.lastStatus === 'UNKNOWN') {
CloudRunnerLogger.log('## Cloud runner job unknwon'); OrchestratorLogger.log('## Orchestrator job unknwon');
} }
if (taskData?.lastStatus !== 'RUNNING') { if (taskData?.lastStatus !== 'RUNNING') {
if (timestamp === 0) { if (timestamp === 0) {
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs'); OrchestratorLogger.log('## Orchestrator job stopped, streaming end of logs');
timestamp = Date.now(); timestamp = Date.now();
} }
if (timestamp !== 0 && Date.now() - timestamp > 30000) { if (timestamp !== 0 && Date.now() - timestamp > 30000) {
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs'); OrchestratorLogger.log('## Orchestrator status is not RUNNING for 30 seconds, last query for logs');
shouldReadLogs = false; shouldReadLogs = false;
} }
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`); OrchestratorLogger.log(`## Status of job: ${taskData.lastStatus}`);
} }
return { timestamp, shouldReadLogs }; return { timestamp, shouldReadLogs };

View File

@@ -1,7 +1,7 @@
import CloudRunner from '../../../cloud-runner'; import Orchestrator from '../../../orchestrator';
export class TaskDefinitionFormation { export class TaskDefinitionFormation {
public static readonly description: string = `Game CI Cloud Runner Task Stack`; public static readonly description: string = `Game CI Orchestrator Task Stack`;
public static get formation(): string { public static get formation(): string {
return `AWSTemplateFormatVersion: 2010-09-09 return `AWSTemplateFormatVersion: 2010-09-09
Description: ${TaskDefinitionFormation.description} Description: ${TaskDefinitionFormation.description}
@@ -29,11 +29,11 @@ Parameters:
Default: 80 Default: 80
Description: What port number the application inside the docker container is binding to Description: What port number the application inside the docker container is binding to
ContainerCpu: ContainerCpu:
Default: ${CloudRunner.buildParameters.containerCpu} Default: ${Orchestrator.buildParameters.containerCpu}
Type: Number Type: Number
Description: How much CPU to give the container. 1024 is 1 CPU Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory: ContainerMemory:
Default: ${CloudRunner.buildParameters.containerMemory} Default: ${Orchestrator.buildParameters.containerMemory}
Type: Number Type: Number
Description: How much memory in megabytes to give the container Description: How much memory in megabytes to give the container
BUILDGUID: BUILDGUID:

View File

@@ -1,11 +1,11 @@
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation'; import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def'; import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import AwsTaskRunner from './aws-task-runner'; import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface'; import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack'; import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack'; import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..'; import { Input } from '../../..';
@@ -13,14 +13,14 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service'; import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import { AwsClientFactory } from './aws-client-factory'; import { AwsClientFactory } from './aws-client-factory';
import ResourceTracking from '../../services/core/resource-tracking'; import ResourceTracking from '../../services/core/resource-tracking';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600; const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number { function getStackWaitTime(): number {
const overrideValue = Number(process.env.CLOUD_RUNNER_AWS_STACK_WAIT_TIME ?? ''); const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) { if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue; return overrideValue;
} }
@@ -98,8 +98,8 @@ class AWSBuildEnvironment implements ProviderInterface {
commands: string, commands: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string> { ): Promise<string> {
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
ResourceTracking.logAllocationSummary('aws workflow'); ResourceTracking.logAllocationSummary('aws workflow');
@@ -107,7 +107,7 @@ class AWSBuildEnvironment implements ProviderInterface {
AwsClientFactory.getECS(); AwsClientFactory.getECS();
const CF = AwsClientFactory.getCloudFormation(); const CF = AwsClientFactory.getCloudFormation();
AwsClientFactory.getKinesis(); AwsClientFactory.getKinesis();
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`); OrchestratorLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh']; const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now(); const startTimeMs = Date.now();
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations( const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
@@ -124,30 +124,30 @@ class AWSBuildEnvironment implements ProviderInterface {
let postRunTaskTimeMs; let postRunTaskTimeMs;
try { try {
const postSetupStacksTimeMs = Date.now(); const postSetupStacksTimeMs = Date.now();
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`); OrchestratorLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands); const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
postRunTaskTimeMs = Date.now(); postRunTaskTimeMs = Date.now();
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`); OrchestratorLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
if (shouldCleanup) { if (shouldCleanup) {
await this.cleanupResources(CF, taskDef); await this.cleanupResources(CF, taskDef);
} }
const postCleanupTimeMs = Date.now(); const postCleanupTimeMs = Date.now();
if (postRunTaskTimeMs !== undefined) if (postRunTaskTimeMs !== undefined)
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`); OrchestratorLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
return output; return output;
} catch (error) { } catch (error) {
CloudRunnerLogger.log(`error running task ${error}`); OrchestratorLogger.log(`error running task ${error}`);
await this.cleanupResources(CF, taskDef); await this.cleanupResources(CF, taskDef);
throw error; throw error;
} }
} }
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) { async cleanupResources(CF: CloudFormation, taskDef: OrchestratorAWSTaskDef) {
const stackWaitTimeSeconds = getStackWaitTime(); const stackWaitTimeSeconds = getStackWaitTime();
CloudRunnerLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`); OrchestratorLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`);
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName })); await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
if (CloudRunnerOptions.useCleanupCron) { if (OrchestratorOptions.useCleanupCron) {
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` })); await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
} }
@@ -169,8 +169,8 @@ class AWSBuildEnvironment implements ProviderInterface {
StackName: `${taskDef.taskDefStackName}-cleanup`, StackName: `${taskDef.taskDefStackName}-cleanup`,
}, },
); );
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`); OrchestratorLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete'); OrchestratorLogger.log('Cleanup complete');
} }
} }
export default AWSBuildEnvironment; export default AWSBuildEnvironment;

View File

@@ -1,10 +1,10 @@
// eslint-disable-next-line import/named // eslint-disable-next-line import/named
import { StackResource } from '@aws-sdk/client-cloudformation'; import { StackResource } from '@aws-sdk/client-cloudformation';
class CloudRunnerAWSTaskDef { class OrchestratorAWSTaskDef {
public taskDefStackName!: string; public taskDefStackName!: string;
public taskDefCloudFormation!: string; public taskDefCloudFormation!: string;
public taskDefResources: StackResource[] | undefined; public taskDefResources: StackResource[] | undefined;
public baseResources: StackResource[] | undefined; public baseResources: StackResource[] | undefined;
} }
export default CloudRunnerAWSTaskDef; export default OrchestratorAWSTaskDef;

View File

@@ -2,7 +2,7 @@ import { DeleteStackCommand, DescribeStackResourcesCommand } from '@aws-sdk/clie
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs'; import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
import { StopTaskCommand } from '@aws-sdk/client-ecs'; import { StopTaskCommand } from '@aws-sdk/client-ecs';
import Input from '../../../../input'; import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../../services/core/orchestrator-logger';
import { TaskService } from './task-service'; import { TaskService } from './task-service';
import { AwsClientFactory } from '../aws-client-factory'; import { AwsClientFactory } from '../aws-client-factory';
@@ -25,7 +25,7 @@ export class GarbageCollectionService {
const { taskElement, element } = task; const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn); taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) { if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`); OrchestratorLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element })); await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
} }
} }
@@ -37,7 +37,7 @@ export class GarbageCollectionService {
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId), (x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
) )
) { ) {
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`); OrchestratorLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return; return;
} }
@@ -47,12 +47,12 @@ export class GarbageCollectionService {
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime))) (!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
) { ) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') { if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`); OrchestratorLogger.log(`Skipping ${element.StackName} ignore list`);
return; return;
} }
CloudRunnerLogger.log(`Deleting ${element.StackName}`); OrchestratorLogger.log(`Deleting ${element.StackName}`);
await CF.send(new DeleteStackCommand({ StackName: element.StackName })); await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
} }
} }
@@ -62,14 +62,14 @@ export class GarbageCollectionService {
deleteResources && deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!))) (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
) { ) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`); OrchestratorLogger.log(`Deleting ${element.logGroupName}`);
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' })); await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
} }
} }
const locks = await TaskService.getLocks(); const locks = await TaskService.getLocks();
for (const element of locks) { for (const element of locks) {
CloudRunnerLogger.log(`Lock: ${element.Key}`); OrchestratorLogger.log(`Lock: ${element.Key}`);
} }
} }
} }

View File

@@ -11,10 +11,10 @@ import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aw
import type { Task } from '@aws-sdk/client-ecs'; import type { Task } from '@aws-sdk/client-ecs';
import { ListObjectsV2Command } from '@aws-sdk/client-s3'; import { ListObjectsV2Command } from '@aws-sdk/client-s3';
import Input from '../../../../input'; import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../../services/core/orchestrator-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation'; import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner'; import AwsTaskRunner from '../aws-task-runner';
import CloudRunner from '../../../cloud-runner'; import Orchestrator from '../../../orchestrator';
import { AwsClientFactory } from '../aws-client-factory'; import { AwsClientFactory } from '../aws-client-factory';
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking'; import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
@@ -31,8 +31,8 @@ export class TaskService {
} }
public static async getCloudFormationJobStacks(): Promise<StackSummary[]> { public static async getCloudFormationJobStacks(): Promise<StackSummary[]> {
const result: StackSummary[] = []; const result: StackSummary[] = [];
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
CloudRunnerLogger.log(`List Cloud Formation Stacks`); OrchestratorLogger.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
const CF = AwsClientFactory.getCloudFormation(); const CF = AwsClientFactory.getCloudFormation();
const stacks = const stacks =
@@ -40,16 +40,16 @@ export class TaskService {
(_x) => (_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription, _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || []; ) || [];
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`); OrchestratorLogger.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) { for (const element of stacks) {
if (!element.CreationTime) { if (!element.CreationTime) {
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`); OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
} }
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0)); const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
CloudRunnerLogger.log( OrchestratorLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor( `Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24, ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
@@ -61,30 +61,30 @@ export class TaskService {
(_x) => (_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription, _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || []; ) || [];
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`); OrchestratorLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) { for (const element of baseStacks) {
if (!element.CreationTime) { if (!element.CreationTime) {
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`); OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
} }
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0)); const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
CloudRunnerLogger.log( OrchestratorLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor( `Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24, ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
); );
result.push(element); result.push(element);
} }
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
return result; return result;
} }
public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> { public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> {
const result: { taskElement: Task; element: string }[] = []; const result: { taskElement: Task; element: string }[] = [];
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
CloudRunnerLogger.log(`List Tasks`); OrchestratorLogger.log(`List Tasks`);
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
const ecs = AwsClientFactory.getECS(); const ecs = AwsClientFactory.getECS();
const clusters: string[] = []; const clusters: string[] = [];
@@ -96,7 +96,7 @@ export class TaskService {
nextToken = clusterResponse.nextToken; nextToken = clusterResponse.nextToken;
} while (nextToken); } while (nextToken);
} }
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`); OrchestratorLogger.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) { for (const element of clusters) {
const taskArns: string[] = []; const taskArns: string[] = [];
{ {
@@ -111,23 +111,23 @@ export class TaskService {
const describeInput = { tasks: taskArns, cluster: element }; const describeInput = { tasks: taskArns, cluster: element };
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || []; const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
if (describeList.length === 0) { if (describeList.length === 0) {
CloudRunnerLogger.log(`No Tasks`); OrchestratorLogger.log(`No Tasks`);
continue; continue;
} }
CloudRunnerLogger.log(`Tasks ${describeList.length}`); OrchestratorLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) { for (const taskElement of describeList) {
if (taskElement === undefined) { if (taskElement === undefined) {
continue; continue;
} }
if (taskElement.createdAt === undefined) { if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`); OrchestratorLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue; continue;
} }
result.push({ taskElement, element }); result.push({ taskElement, element });
} }
} }
} }
CloudRunnerLogger.log(``); OrchestratorLogger.log(``);
return result; return result;
} }
@@ -143,7 +143,7 @@ export class TaskService {
throw new Error('stack not defined'); throw new Error('stack not defined');
} }
if (!stack.CreationTime) { if (!stack.CreationTime) {
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`); OrchestratorLogger.log(`${stack.StackName} due to undefined CreationTime`);
} }
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0)); const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
const message = ` const message = `
@@ -153,11 +153,11 @@ export class TaskService {
${JSON.stringify(stackInfo, undefined, 4)} ${JSON.stringify(stackInfo, undefined, 4)}
${JSON.stringify(stackInfo2, undefined, 4)} ${JSON.stringify(stackInfo2, undefined, 4)}
`; `;
CloudRunnerLogger.log(message); OrchestratorLogger.log(message);
return message; return message;
} catch (error) { } catch (error) {
CloudRunnerLogger.error( OrchestratorLogger.error(
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`, `Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
); );
throw error; throw error;
@@ -181,15 +181,15 @@ export class TaskService {
logGroups.push(...(logGroupsDescribe?.logGroups || [])); logGroups.push(...(logGroupsDescribe?.logGroups || []));
} }
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`); OrchestratorLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) { for (const element of logGroups) {
if (element.creationTime === undefined) { if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`); OrchestratorLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue; continue;
} }
const ageDate: Date = new Date(Date.now() - element.creationTime); const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log( OrchestratorLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor( `Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24, ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
@@ -201,7 +201,7 @@ export class TaskService {
} }
public static async getLocks(): Promise<Array<{ Key: string }>> { public static async getLocks(): Promise<Array<{ Key: string }>> {
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
if (CloudRunner.buildParameters.storageProvider === 'rclone') { if (Orchestrator.buildParameters.storageProvider === 'rclone') {
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
type ListObjectsFunction = (prefix: string) => Promise<string[]>; type ListObjectsFunction = (prefix: string) => Promise<string[]>;
const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects(''); const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects('');
@@ -210,7 +210,7 @@ export class TaskService {
} }
const s3 = AwsClientFactory.getS3(); const s3 = AwsClientFactory.getS3();
const listRequest = { const listRequest = {
Bucket: CloudRunner.buildParameters.awsStackName, Bucket: Orchestrator.buildParameters.awsStackName,
}; };
const results = await s3.send(new ListObjectsV2Command(listRequest)); const results = await s3.send(new ListObjectsV2Command(listRequest));

View File

@@ -1,20 +1,20 @@
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { ProviderInterface } from '../provider-interface'; import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import Docker from '../../../docker'; import Docker from '../../../docker';
import { Action } from '../../..'; import { Action } from '../../..';
import { writeFileSync } from 'node:fs'; import { writeFileSync } from 'node:fs';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service'; import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types'; import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerCloudRunner implements ProviderInterface { class LocalDockerOrchestrator implements ProviderInterface {
public buildParameters!: BuildParameters; public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> { listResources(): Promise<ProviderResource[]> {
@@ -50,15 +50,15 @@ class LocalDockerCloudRunner implements ProviderInterface {
const { workspace } = Action; const { workspace } = Action;
if ( if (
fs.existsSync( fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${ `${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`, }`,
) )
) { ) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`); await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache/cache/build/`);
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${ `rm -r ${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`, }`,
); );
} }
@@ -80,11 +80,11 @@ class LocalDockerCloudRunner implements ProviderInterface {
commands: string, commands: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string> { ): Promise<string> {
CloudRunnerLogger.log(buildGuid); OrchestratorLogger.log(buildGuid);
CloudRunnerLogger.log(commands); OrchestratorLogger.log(commands);
const { workspace, actionFolder } = Action; const { workspace, actionFolder } = Action;
const content: StringKeyValuePair[] = []; const content: StringKeyValuePair[] = [];
@@ -115,12 +115,12 @@ class LocalDockerCloudRunner implements ProviderInterface {
value = value value = value
.replace('http://localhost', 'http://host.docker.internal') .replace('http://localhost', 'http://host.docker.internal')
.replace('http://127.0.0.1', 'http://host.docker.internal'); .replace('http://127.0.0.1', 'http://host.docker.internal');
CloudRunnerLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`); OrchestratorLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
} }
content.push({ name: x.name, value }); content.push({ name: x.name, value });
} }
// if (this.buildParameters?.cloudRunnerIntegrationTests) { // if (this.buildParameters?.orchestratorIntegrationTests) {
// core.info(JSON.stringify(content, undefined, 4)); // core.info(JSON.stringify(content, undefined, 4));
// core.info(JSON.stringify(secrets, undefined, 4)); // core.info(JSON.stringify(secrets, undefined, 4));
// core.info(JSON.stringify(environment, undefined, 4)); // core.info(JSON.stringify(environment, undefined, 4));
@@ -142,28 +142,28 @@ class LocalDockerCloudRunner implements ProviderInterface {
const fileContents = `#!/bin/sh const fileContents = `#!/bin/sh
set -e set -e
mkdir -p /github/workspace/cloud-runner-cache mkdir -p /github/workspace/orchestrator-cache
mkdir -p /data/cache mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder} cp -a /github/workspace/orchestrator-cache/. ${sharedFolder}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)} ${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
# Only copy cache directory, exclude retained workspaces to avoid running out of disk space # Only copy cache directory, exclude retained workspaces to avoid running out of disk space
if [ -d "${sharedFolder}cache" ]; then if [ -d "${sharedFolder}cache" ]; then
cp -a ${sharedFolder}cache/. /github/workspace/cloud-runner-cache/cache/ || true cp -a ${sharedFolder}cache/. /github/workspace/orchestrator-cache/cache/ || true
fi fi
# Copy test files from /data/ root to workspace for test assertions # Copy test files from /data/ root to workspace for test assertions
# This allows tests to write files to /data/ and have them available in the workspace # This allows tests to write files to /data/ and have them available in the workspace
find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/cloud-runner-cache/ \\; || true find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/orchestrator-cache/ \\; || true
`; `;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, { writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
flag: 'w', flag: 'w',
}); });
if (CloudRunner.buildParameters.cloudRunnerDebug) { if (Orchestrator.buildParameters.orchestratorDebug) {
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`); OrchestratorLogger.log(`Running local-docker: \n ${fileContents}`);
} }
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) { if (fs.existsSync(`${workspace}/orchestrator-cache`)) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`); await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache && du -sh ${workspace}/orchestrator-cache`);
} }
const exitCode = await Docker.run( const exitCode = await Docker.run(
image, image,
@@ -193,4 +193,4 @@ find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/w
return myOutput; return myOutput;
} }
} }
export default LocalDockerCloudRunner; export default LocalDockerOrchestrator;

View File

@@ -2,21 +2,21 @@ import * as k8s from '@kubernetes/client-node';
import { BuildParameters } from '../../..'; import { BuildParameters } from '../../..';
import * as core from '@actions/core'; import * as core from '@actions/core';
import { ProviderInterface } from '../provider-interface'; import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import KubernetesStorage from './kubernetes-storage'; import KubernetesStorage from './kubernetes-storage';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import KubernetesTaskRunner from './kubernetes-task-runner'; import KubernetesTaskRunner from './kubernetes-task-runner';
import KubernetesSecret from './kubernetes-secret'; import KubernetesSecret from './kubernetes-secret';
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory'; import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
import KubernetesServiceAccount from './kubernetes-service-account'; import KubernetesServiceAccount from './kubernetes-service-account';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { CoreV1Api } from '@kubernetes/client-node'; import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger'; import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import { KubernetesRole } from './kubernetes-role'; import { KubernetesRole } from './kubernetes-role';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import ResourceTracking from '../../services/core/resource-tracking'; import ResourceTracking from '../../services/core/resource-tracking';
class Kubernetes implements ProviderInterface { class Kubernetes implements ProviderInterface {
@@ -47,7 +47,7 @@ class Kubernetes implements ProviderInterface {
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api); this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api); this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
this.namespace = buildParameters.containerNamespace ? buildParameters.containerNamespace : 'default'; this.namespace = buildParameters.containerNamespace ? buildParameters.containerNamespace : 'default';
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment'); OrchestratorLogger.log('Loaded default Kubernetes configuration for this environment');
} }
async PushLogUpdate(logs: string) { async PushLogUpdate(logs: string) {
@@ -63,7 +63,7 @@ class Kubernetes implements ProviderInterface {
// logs to base64 // logs to base64
logs = Buffer.from(logs).toString('base64'); logs = Buffer.from(logs).toString('base64');
const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true); const response = await OrchestratorSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`); RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
} }
@@ -133,11 +133,11 @@ class Kubernetes implements ProviderInterface {
commands: string, commands: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string> { ): Promise<string> {
try { try {
CloudRunnerLogger.log('Cloud Runner K8s workflow!'); OrchestratorLogger.log('Orchestrator K8s workflow!');
ResourceTracking.logAllocationSummary('k8s workflow'); ResourceTracking.logAllocationSummary('k8s workflow');
await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)'); await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)');
await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)'); await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)');
@@ -145,7 +145,7 @@ class Kubernetes implements ProviderInterface {
// Setup // Setup
const id = const id =
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters) BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
? CloudRunner.lockedWorkspace ? Orchestrator.lockedWorkspace
: this.buildParameters.buildGuid; : this.buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`; this.pvcName = `unity-builder-pvc-${id}`;
await KubernetesStorage.createPersistentVolumeClaim( await KubernetesStorage.createPersistentVolumeClaim(
@@ -162,11 +162,11 @@ class Kubernetes implements ProviderInterface {
// For tests, clean up old images before creating job to free space for image pull // For tests, clean up old images before creating job to free space for image pull
// IMPORTANT: Preserve the Unity image to avoid re-pulling it // IMPORTANT: Preserve the Unity image to avoid re-pulling it
if (process.env['cloudRunnerTests'] === 'true') { if (process.env['orchestratorTests'] === 'true') {
try { try {
CloudRunnerLogger.log('Cleaning up old images in k3d node before pulling new image...'); OrchestratorLogger.log('Cleaning up old images in k3d node before pulling new image...');
const { CloudRunnerSystem: CloudRunnerSystemModule } = await import( const { OrchestratorSystem: OrchestratorSystemModule } = await import(
'../../services/core/cloud-runner-system' '../../services/core/orchestrator-system'
); );
// Aggressive cleanup: remove stopped containers and non-Unity images // Aggressive cleanup: remove stopped containers and non-Unity images
@@ -185,15 +185,15 @@ class Kubernetes implements ProviderInterface {
for (const cmd of cleanupCommands) { for (const cmd of cleanupCommands) {
try { try {
await CloudRunnerSystemModule.Run(cmd, true, true); await OrchestratorSystemModule.Run(cmd, true, true);
} catch (cmdError) { } catch (cmdError) {
// Ignore individual command failures - cleanup is best effort // Ignore individual command failures - cleanup is best effort
CloudRunnerLogger.log(`Cleanup command failed (non-fatal): ${cmdError}`); OrchestratorLogger.log(`Cleanup command failed (non-fatal): ${cmdError}`);
} }
} }
CloudRunnerLogger.log('Cleanup completed (containers and non-Unity images removed, Unity images preserved)'); OrchestratorLogger.log('Cleanup completed (containers and non-Unity images removed, Unity images preserved)');
} catch (cleanupError) { } catch (cleanupError) {
CloudRunnerLogger.logWarning(`Failed to cleanup images before job creation: ${cleanupError}`); OrchestratorLogger.logWarning(`Failed to cleanup images before job creation: ${cleanupError}`);
// Continue anyway - image might already be cached // Continue anyway - image might already be cached
} }
@@ -203,14 +203,14 @@ class Kubernetes implements ProviderInterface {
try { try {
// Before creating the job, verify we have the Unity image cached on the agent node // Before creating the job, verify we have the Unity image cached on the agent node
// If not cached, try to ensure it's available to avoid disk pressure during pull // If not cached, try to ensure it's available to avoid disk pressure during pull
if (process.env['cloudRunnerTests'] === 'true' && image.includes('unityci/editor')) { if (process.env['orchestratorTests'] === 'true' && image.includes('unityci/editor')) {
try { try {
const { CloudRunnerSystem: CloudRunnerSystemModule2 } = await import( const { OrchestratorSystem: OrchestratorSystemModule2 } = await import(
'../../services/core/cloud-runner-system' '../../services/core/orchestrator-system'
); );
// Check if image is cached on agent node (where pods run) // Check if image is cached on agent node (where pods run)
const agentImageCheck = await CloudRunnerSystemModule2.Run( const agentImageCheck = await OrchestratorSystemModule2.Run(
`docker exec k3d-unity-builder-agent-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`, `docker exec k3d-unity-builder-agent-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
true, true,
true, true,
@@ -218,20 +218,20 @@ class Kubernetes implements ProviderInterface {
if (agentImageCheck.includes('not_cached')) { if (agentImageCheck.includes('not_cached')) {
// Check if image is on server node // Check if image is on server node
const serverImageCheck = await CloudRunnerSystemModule2.Run( const serverImageCheck = await OrchestratorSystemModule2.Run(
`docker exec k3d-unity-builder-server-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`, `docker exec k3d-unity-builder-server-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
true, true,
true, true,
); );
// Check available disk space on agent node // Check available disk space on agent node
const diskInfo = await CloudRunnerSystemModule2.Run( const diskInfo = await OrchestratorSystemModule2.Run(
'docker exec k3d-unity-builder-agent-0 sh -c "df -h /var/lib/rancher/k3s 2>/dev/null | tail -1 || df -h / 2>/dev/null | tail -1 || echo unknown" || echo unknown', 'docker exec k3d-unity-builder-agent-0 sh -c "df -h /var/lib/rancher/k3s 2>/dev/null | tail -1 || df -h / 2>/dev/null | tail -1 || echo unknown" || echo unknown',
true, true,
true, true,
); );
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Unity image not cached on agent node (where pods run). Server node: ${ `Unity image not cached on agent node (where pods run). Server node: ${
serverImageCheck.includes('cached') ? 'has image' : 'no image' serverImageCheck.includes('cached') ? 'has image' : 'no image'
}. Disk info: ${diskInfo.trim()}. Pod will attempt to pull image (3.9GB) which may fail due to disk pressure.`, }. Disk info: ${diskInfo.trim()}. Pod will attempt to pull image (3.9GB) which may fail due to disk pressure.`,
@@ -244,7 +244,7 @@ class Kubernetes implements ProviderInterface {
// 3. The pod will attempt to pull during scheduling anyway // 3. The pod will attempt to pull during scheduling anyway
// 4. If the pull fails, Kubernetes will provide proper error messages // 4. If the pull fails, Kubernetes will provide proper error messages
if (serverImageCheck.includes('cached')) { if (serverImageCheck.includes('cached')) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
'Unity image exists on server node but not agent node. Pod will attempt to pull during scheduling. If pull fails due to disk pressure, ensure cleanup runs before this test.', 'Unity image exists on server node but not agent node. Pod will attempt to pull during scheduling. If pull fails due to disk pressure, ensure cleanup runs before this test.',
); );
} else { } else {
@@ -264,7 +264,7 @@ class Kubernetes implements ProviderInterface {
// Unity image is ~3.9GB, need at least 4.5GB to be safe // Unity image is ~3.9GB, need at least 4.5GB to be safe
if (availableGB < 4.5) { if (availableGB < 4.5) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`CRITICAL: Unity image not cached and only ${availableGB.toFixed( `CRITICAL: Unity image not cached and only ${availableGB.toFixed(
2, 2,
)}GB available. Image pull (3.9GB) will likely fail. Consider running cleanup or ensuring pre-pull step succeeds.`, )}GB available. Image pull (3.9GB) will likely fail. Consider running cleanup or ensuring pre-pull step succeeds.`,
@@ -273,20 +273,20 @@ class Kubernetes implements ProviderInterface {
} }
} }
} else { } else {
CloudRunnerLogger.log('Unity image is cached on agent node - pod should start without pulling'); OrchestratorLogger.log('Unity image is cached on agent node - pod should start without pulling');
} }
} catch (checkError) { } catch (checkError) {
// Ignore check errors - continue with job creation // Ignore check errors - continue with job creation
CloudRunnerLogger.logWarning(`Failed to verify Unity image cache: ${checkError}`); OrchestratorLogger.logWarning(`Failed to verify Unity image cache: ${checkError}`);
} }
} }
CloudRunnerLogger.log('Job does not exist'); OrchestratorLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets); await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
CloudRunnerLogger.log('Watching pod until running'); OrchestratorLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace); await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
CloudRunnerLogger.log('Pod is running'); OrchestratorLogger.log('Pod is running');
output += await KubernetesTaskRunner.runTask( output += await KubernetesTaskRunner.runTask(
this.kubeConfig, this.kubeConfig,
this.kubeClient, this.kubeClient,
@@ -296,9 +296,9 @@ class Kubernetes implements ProviderInterface {
this.namespace, this.namespace,
); );
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`error running k8s workflow ${error}`); OrchestratorLogger.log(`error running k8s workflow ${error}`);
await new Promise((resolve) => setTimeout(resolve, 3000)); await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log( OrchestratorLogger.log(
JSON.stringify( JSON.stringify(
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items (await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
.map((x) => { .map((x) => {
@@ -321,7 +321,7 @@ class Kubernetes implements ProviderInterface {
return output; return output;
} catch (error) { } catch (error) {
CloudRunnerLogger.log('Running job failed'); OrchestratorLogger.log('Running job failed');
core.error(JSON.stringify(error, undefined, 4)); core.error(JSON.stringify(error, undefined, 4));
// await this.cleanupTaskResources(); // await this.cleanupTaskResources();
@@ -334,8 +334,8 @@ class Kubernetes implements ProviderInterface {
image: string, image: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
) { ) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets); await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace); const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
@@ -359,8 +359,8 @@ class Kubernetes implements ProviderInterface {
image: string, image: string,
mountdir: string, mountdir: string,
workingdir: string, workingdir: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
) { ) {
for (let index = 0; index < 3; index++) { for (let index = 0; index < 3; index++) {
try { try {
@@ -385,13 +385,13 @@ class Kubernetes implements ProviderInterface {
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api); // await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec); const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
CloudRunnerLogger.log(`Build job created`); OrchestratorLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000)); await new Promise((promise) => setTimeout(promise, 5000));
CloudRunnerLogger.log('Job created'); OrchestratorLogger.log('Job created');
return result.body.metadata?.name; return result.body.metadata?.name;
} catch (error) { } catch (error) {
CloudRunnerLogger.log(`Error occured creating job: ${error}`); OrchestratorLogger.log(`Error occured creating job: ${error}`);
throw error; throw error;
} }
} }
@@ -403,26 +403,26 @@ class Kubernetes implements ProviderInterface {
} }
async cleanupTaskResources() { async cleanupTaskResources() {
CloudRunnerLogger.log('cleaning up'); OrchestratorLogger.log('cleaning up');
try { try {
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace); await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api); await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup`); OrchestratorLogger.log(`Failed to cleanup`);
if (error.response.body.reason !== `NotFound`) { if (error.response.body.reason !== `NotFound`) {
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`); OrchestratorLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
throw error; throw error;
} }
} }
try { try {
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup secret`); OrchestratorLogger.log(`Failed to cleanup secret`);
CloudRunnerLogger.log(error.response.body.reason); OrchestratorLogger.log(error.response.body.reason);
} }
CloudRunnerLogger.log('cleaned up Secret, Job and Pod'); OrchestratorLogger.log('cleaned up Secret, Job and Pod');
CloudRunnerLogger.log('cleaning up finished'); OrchestratorLogger.log('cleaning up finished');
} }
async cleanupWorkflow( async cleanupWorkflow(
@@ -435,14 +435,14 @@ class Kubernetes implements ProviderInterface {
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) { if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
return; return;
} }
CloudRunnerLogger.log(`deleting PVC`); OrchestratorLogger.log(`deleting PVC`);
try { try {
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace); await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace); await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
CloudRunnerLogger.log('cleaned up PVC and Service Account'); OrchestratorLogger.log('cleaned up PVC and Service Account');
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`); OrchestratorLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
throw error; throw error;
} }
} }

View File

@@ -1,10 +1,10 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node'; import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import { CommandHookService } from '../../services/hooks/command-hook-service'; import { CommandHookService } from '../../services/hooks/command-hook-service';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
class KubernetesJobSpecFactory { class KubernetesJobSpecFactory {
static getJobSpec( static getJobSpec(
@@ -12,8 +12,8 @@ class KubernetesJobSpecFactory {
image: string, image: string,
mountdir: string, mountdir: string,
workingDirectory: string, workingDirectory: string,
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
buildGuid: string, buildGuid: string,
buildParameters: BuildParameters, buildParameters: BuildParameters,
secretName: string, secretName: string,
@@ -38,7 +38,7 @@ class KubernetesJobSpecFactory {
// Priority: K8S_LOCALSTACK_HOST env var > localstack-main (container name on shared network) // Priority: K8S_LOCALSTACK_HOST env var > localstack-main (container name on shared network)
// Note: Using K8S_LOCALSTACK_HOST instead of LOCALSTACK_HOST to avoid conflict with awslocal CLI // Note: Using K8S_LOCALSTACK_HOST instead of LOCALSTACK_HOST to avoid conflict with awslocal CLI
const localstackHost = process.env['K8S_LOCALSTACK_HOST'] || 'localstack-main'; const localstackHost = process.env['K8S_LOCALSTACK_HOST'] || 'localstack-main';
CloudRunnerLogger.log(`K8s pods will use LocalStack host: ${localstackHost}`); OrchestratorLogger.log(`K8s pods will use LocalStack host: ${localstackHost}`);
const adjustedEnvironment = environment.map((x) => { const adjustedEnvironment = environment.map((x) => {
let value = x.value; let value = x.value;
@@ -52,10 +52,10 @@ class KubernetesJobSpecFactory {
value = value value = value
.replace('http://localhost', `http://${localstackHost}`) .replace('http://localhost', `http://${localstackHost}`)
.replace('http://127.0.0.1', `http://${localstackHost}`); .replace('http://127.0.0.1', `http://${localstackHost}`);
CloudRunnerLogger.log(`Replaced localhost with ${localstackHost} for ${x.name}: ${value}`); OrchestratorLogger.log(`Replaced localhost with ${localstackHost} for ${x.name}: ${value}`);
} }
return { name: x.name, value } as CloudRunnerEnvironmentVariable; return { name: x.name, value } as OrchestratorEnvironmentVariable;
}); });
const job = new k8s.V1Job(); const job = new k8s.V1Job();
@@ -71,7 +71,7 @@ class KubernetesJobSpecFactory {
// Reduce TTL for tests to free up resources faster (default 9999s = ~2.8 hours) // Reduce TTL for tests to free up resources faster (default 9999s = ~2.8 hours)
// For CI/test environments, use shorter TTL (300s = 5 minutes) to prevent disk pressure // For CI/test environments, use shorter TTL (300s = 5 minutes) to prevent disk pressure
const jobTTL = process.env['cloudRunnerTests'] === 'true' ? 300 : 9999; const jobTTL = process.env['orchestratorTests'] === 'true' ? 300 : 9999;
job.spec = { job.spec = {
ttlSecondsAfterFinished: jobTTL, ttlSecondsAfterFinished: jobTTL,
backoffLimit: 0, backoffLimit: 0,
@@ -91,11 +91,11 @@ class KubernetesJobSpecFactory {
ttlSecondsAfterFinished: 9999, ttlSecondsAfterFinished: 9999,
name: containerName, name: containerName,
image, image,
imagePullPolicy: process.env['cloudRunnerTests'] === 'true' ? 'IfNotPresent' : 'Always', imagePullPolicy: process.env['orchestratorTests'] === 'true' ? 'IfNotPresent' : 'Always',
command: ['/bin/sh'], command: ['/bin/sh'],
args: [ args: [
'-c', '-c',
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`, `${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, Orchestrator.buildParameters)}`,
], ],
workingDir: `${workingDirectory}`, workingDir: `${workingDirectory}`,
@@ -106,7 +106,7 @@ class KubernetesJobSpecFactory {
const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu']; const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu'];
const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage)); const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage));
if (isLightweightContainer && process.env['cloudRunnerTests'] === 'true') { if (isLightweightContainer && process.env['orchestratorTests'] === 'true') {
// For test environments, use minimal resources for hook containers // For test environments, use minimal resources for hook containers
return { return {
memory: '128Mi', memory: '128Mi',
@@ -179,7 +179,7 @@ class KubernetesJobSpecFactory {
}, },
}; };
if (process.env['CLOUD_RUNNER_MINIKUBE']) { if (process.env['ORCHESTRATOR_MINIKUBE']) {
job.spec.template.spec.volumes[0] = { job.spec.template.spec.volumes[0] = {
name: 'build-mount', name: 'build-mount',
hostPath: { hostPath: {
@@ -195,7 +195,7 @@ class KubernetesJobSpecFactory {
// For production, use 2Gi to allow for larger builds // For production, use 2Gi to allow for larger builds
// The node needs some free space headroom, so requesting too much causes evictions // The node needs some free space headroom, so requesting too much causes evictions
// With node at 96% usage and only ~2.7GB free, we can't request much without triggering evictions // With node at 96% usage and only ~2.7GB free, we can't request much without triggering evictions
if (process.env['cloudRunnerTests'] !== 'true') { if (process.env['orchestratorTests'] !== 'true') {
// Only set ephemeral-storage request for production builds // Only set ephemeral-storage request for production builds
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '2Gi'; job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '2Gi';
} }

View File

@@ -1,11 +1,11 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { CoreV1Api } from '@kubernetes/client-node'; import { CoreV1Api } from '@kubernetes/client-node';
class KubernetesPods { class KubernetesPods {
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) { public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name); const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`); const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
const phase = pods[0]?.status?.phase || 'undefined status'; const phase = pods[0]?.status?.phase || 'undefined status';
CloudRunnerLogger.log(`Getting pod status: ${phase}`); OrchestratorLogger.log(`Getting pod status: ${phase}`);
if (phase === `Failed`) { if (phase === `Failed`) {
const pod = pods[0]; const pod = pods[0];
const containerStatuses = pod.status?.containerStatuses || []; const containerStatuses = pod.status?.containerStatuses || [];
@@ -70,15 +70,15 @@ class KubernetesPods {
if (containerSucceeded && containerExitCode === 0) { if (containerSucceeded && containerExitCode === 0) {
// Container succeeded - PreStopHook failure is non-critical // Container succeeded - PreStopHook failure is non-critical
if (hasPreStopHookFailure) { if (hasPreStopHookFailure) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`, `Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`,
); );
} else { } else {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`, `Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`,
); );
} }
CloudRunnerLogger.log(`Pod details: ${errorDetails.join('\n')}`); OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
// Don't throw error - container succeeded, PreStopHook failure is non-critical // Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure return false; // Pod is not running, but we don't treat it as a failure
@@ -87,7 +87,7 @@ class KubernetesPods {
// If pod was killed and we have PreStopHook failure, wait for container status // If pod was killed and we have PreStopHook failure, wait for container status
// The container might have succeeded but status hasn't been updated yet // The container might have succeeded but status hasn't been updated yet
if (wasKilled && hasPreStopHookFailure && (containerExitCode === undefined || !containerSucceeded)) { if (wasKilled && hasPreStopHookFailure && (containerExitCode === undefined || !containerSucceeded)) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Pod ${podName} was killed with PreStopHook failure. Waiting for container status to determine if container succeeded...`, `Pod ${podName} was killed with PreStopHook failure. Waiting for container status to determine if container succeeded...`,
); );
@@ -103,13 +103,13 @@ class KubernetesPods {
if (updatedContainerStatus.state?.terminated) { if (updatedContainerStatus.state?.terminated) {
const updatedExitCode = updatedContainerStatus.state.terminated.exitCode; const updatedExitCode = updatedContainerStatus.state.terminated.exitCode;
if (updatedExitCode === 0) { if (updatedExitCode === 0) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} container succeeded (exit code 0) after waiting. PreStopHook failure is non-fatal.`, `Pod ${podName} container succeeded (exit code 0) after waiting. PreStopHook failure is non-fatal.`,
); );
return false; // Pod is not running, but container succeeded return false; // Pod is not running, but container succeeded
} else { } else {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Pod ${podName} container failed with exit code ${updatedExitCode} after waiting.`, `Pod ${podName} container failed with exit code ${updatedExitCode} after waiting.`,
); );
errorDetails.push(`Container terminated after wait: exit code ${updatedExitCode}`); errorDetails.push(`Container terminated after wait: exit code ${updatedExitCode}`);
@@ -120,27 +120,27 @@ class KubernetesPods {
} }
} }
} catch (waitError) { } catch (waitError) {
CloudRunnerLogger.log(`Error while waiting for container status: ${waitError}`); OrchestratorLogger.log(`Error while waiting for container status: ${waitError}`);
} }
} }
// If we still don't have container status after waiting, but only PreStopHook failed, // If we still don't have container status after waiting, but only PreStopHook failed,
// be lenient - the container might have succeeded but status wasn't updated // be lenient - the container might have succeeded but status wasn't updated
if (containerExitCode === undefined && hasPreStopHookFailure && !hasExceededGracePeriod) { if (containerExitCode === undefined && hasPreStopHookFailure && !hasExceededGracePeriod) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} container status not available after waiting, but only PreStopHook failed (no ExceededGracePeriod). Assuming container may have succeeded.`, `Pod ${podName} container status not available after waiting, but only PreStopHook failed (no ExceededGracePeriod). Assuming container may have succeeded.`,
); );
return false; // Be lenient - PreStopHook failure alone is not fatal return false; // Be lenient - PreStopHook failure alone is not fatal
} }
CloudRunnerLogger.log( OrchestratorLogger.log(
`Container status check completed. Exit code: ${containerExitCode}, PreStopHook failure: ${hasPreStopHookFailure}`, `Container status check completed. Exit code: ${containerExitCode}, PreStopHook failure: ${hasPreStopHookFailure}`,
); );
} }
// If we only have PreStopHook failure and no actual container failure, be lenient // If we only have PreStopHook failure and no actual container failure, be lenient
if (hasPreStopHookFailure && !hasExceededGracePeriod && containerExitCode === undefined) { if (hasPreStopHookFailure && !hasExceededGracePeriod && containerExitCode === undefined) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} has PreStopHook failure but no container failure detected. Treating as non-fatal.`, `Pod ${podName} has PreStopHook failure but no container failure detected. Treating as non-fatal.`,
); );
@@ -153,8 +153,8 @@ class KubernetesPods {
); );
if (wasEvicted) { if (wasEvicted) {
const evictionMessage = `Pod ${podName} was evicted due to disk pressure. This is a test infrastructure issue - the cluster doesn't have enough disk space.`; const evictionMessage = `Pod ${podName} was evicted due to disk pressure. This is a test infrastructure issue - the cluster doesn't have enough disk space.`;
CloudRunnerLogger.logWarning(evictionMessage); OrchestratorLogger.logWarning(evictionMessage);
CloudRunnerLogger.log(`Pod details: ${errorDetails.join('\n')}`); OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
throw new Error( throw new Error(
`${evictionMessage}\nThis indicates the test environment needs more disk space or better cleanup.\n${errorDetails.join( `${evictionMessage}\nThis indicates the test environment needs more disk space or better cleanup.\n${errorDetails.join(
'\n', '\n',
@@ -166,18 +166,18 @@ class KubernetesPods {
// If this happened with PreStopHook failure, it might be a resource issue, not a real failure // If this happened with PreStopHook failure, it might be a resource issue, not a real failure
// Be lenient if we only have PreStopHook/ExceededGracePeriod issues // Be lenient if we only have PreStopHook/ExceededGracePeriod issues
if (containerExitCode === 137 && (hasPreStopHookFailure || hasExceededGracePeriod)) { if (containerExitCode === 137 && (hasPreStopHookFailure || hasExceededGracePeriod)) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} was killed (exit code 137 - likely OOM or resource limit) with PreStopHook/grace period issues. This may be a resource constraint issue rather than a build failure.`, `Pod ${podName} was killed (exit code 137 - likely OOM or resource limit) with PreStopHook/grace period issues. This may be a resource constraint issue rather than a build failure.`,
); );
// Still log the details but don't fail the test - the build might have succeeded before being killed // Still log the details but don't fail the test - the build might have succeeded before being killed
CloudRunnerLogger.log(`Pod details: ${errorDetails.join('\n')}`); OrchestratorLogger.log(`Pod details: ${errorDetails.join('\n')}`);
return false; // Don't treat system kills as test failures if only PreStopHook issues return false; // Don't treat system kills as test failures if only PreStopHook issues
} }
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`; const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
CloudRunnerLogger.log(errorMessage); OrchestratorLogger.log(errorMessage);
throw new Error(errorMessage); throw new Error(errorMessage);
} }

View File

@@ -1,12 +1,12 @@
import { CoreV1Api } from '@kubernetes/client-node'; import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import * as k8s from '@kubernetes/client-node'; import * as k8s from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import * as base64 from 'base-64'; import * as base64 from 'base-64';
class KubernetesSecret { class KubernetesSecret {
static async createSecret( static async createSecret(
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
secretName: string, secretName: string,
namespace: string, namespace: string,
kubeClient: CoreV1Api, kubeClient: CoreV1Api,
@@ -23,20 +23,20 @@ class KubernetesSecret {
for (const buildSecret of secrets) { for (const buildSecret of secrets) {
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue); secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
} }
CloudRunnerLogger.log(`Creating secret: ${secretName}`); OrchestratorLogger.log(`Creating secret: ${secretName}`);
const existingSecrets = await kubeClient.listNamespacedSecret(namespace); const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
const mappedSecrets = existingSecrets.body.items.map((x) => { const mappedSecrets = existingSecrets.body.items.map((x) => {
return x.metadata?.name || `no name`; return x.metadata?.name || `no name`;
}); });
CloudRunnerLogger.log( OrchestratorLogger.log(
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`, `ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
); );
await new Promise((promise) => setTimeout(promise, 15000)); await new Promise((promise) => setTimeout(promise, 15000));
await kubeClient.createNamespacedSecret(namespace, secret); await kubeClient.createNamespacedSecret(namespace, secret);
CloudRunnerLogger.log('Created secret'); OrchestratorLogger.log('Created secret');
} catch (error) { } catch (error) {
CloudRunnerLogger.log(`Created secret failed ${error}`); OrchestratorLogger.log(`Created secret failed ${error}`);
throw new Error(`Failed to create kubernetes secret`); throw new Error(`Failed to create kubernetes secret`);
} }
} }

View File

@@ -2,7 +2,7 @@ import { waitUntil } from 'async-wait-until';
import * as core from '@actions/core'; import * as core from '@actions/core';
import * as k8s from '@kubernetes/client-node'; import * as k8s from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { IncomingMessage } from 'node:http'; import { IncomingMessage } from 'node:http';
import GitHub from '../../../github'; import GitHub from '../../../github';
@@ -14,24 +14,24 @@ class KubernetesStorage {
namespace: string, namespace: string,
) { ) {
if (buildParameters.kubeVolume !== ``) { if (buildParameters.kubeVolume !== ``) {
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`); OrchestratorLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
pvcName = buildParameters.kubeVolume; pvcName = buildParameters.kubeVolume;
return; return;
} }
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items; const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
const pvcList = allPvc.map((x) => x.metadata?.name); const pvcList = allPvc.map((x) => x.metadata?.name);
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`); OrchestratorLogger.log(`Current PVCs in namespace ${namespace}`);
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4)); OrchestratorLogger.log(JSON.stringify(pvcList, undefined, 4));
if (pvcList.includes(pvcName)) { if (pvcList.includes(pvcName)) {
CloudRunnerLogger.log(`pvc ${pvcName} already exists`); OrchestratorLogger.log(`pvc ${pvcName} already exists`);
if (GitHub.githubInputEnabled) { if (GitHub.githubInputEnabled) {
core.setOutput('volume', pvcName); core.setOutput('volume', pvcName);
} }
return; return;
} }
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`); OrchestratorLogger.log(`Creating PVC ${pvcName} (does not exist)`);
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace); const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName); await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
} }
@@ -49,7 +49,7 @@ class KubernetesStorage {
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) { public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
let checkCount = 0; let checkCount = 0;
try { try {
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`); OrchestratorLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
// Check if storage class uses WaitForFirstConsumer binding mode // Check if storage class uses WaitForFirstConsumer binding mode
// If so, skip waiting - PVC will bind when pod is created // If so, skip waiting - PVC will bind when pod is created
@@ -68,33 +68,33 @@ class KubernetesStorage {
const volumeBindingMode = sc.body.volumeBindingMode; const volumeBindingMode = sc.body.volumeBindingMode;
if (volumeBindingMode === 'WaitForFirstConsumer') { if (volumeBindingMode === 'WaitForFirstConsumer') {
CloudRunnerLogger.log( OrchestratorLogger.log(
`StorageClass "${storageClassName}" uses WaitForFirstConsumer binding mode. PVC will bind when pod is created. Skipping wait.`, `StorageClass "${storageClassName}" uses WaitForFirstConsumer binding mode. PVC will bind when pod is created. Skipping wait.`,
); );
shouldSkipWait = true; shouldSkipWait = true;
} }
} catch (scError) { } catch (scError) {
// If we can't check the storage class, proceed with normal wait // If we can't check the storage class, proceed with normal wait
CloudRunnerLogger.log( OrchestratorLogger.log(
`Could not check storage class binding mode: ${scError}. Proceeding with normal wait.`, `Could not check storage class binding mode: ${scError}. Proceeding with normal wait.`,
); );
} }
} }
} catch (pvcReadError) { } catch (pvcReadError) {
// If we can't read PVC, proceed with normal wait // If we can't read PVC, proceed with normal wait
CloudRunnerLogger.log( OrchestratorLogger.log(
`Could not read PVC to check storage class: ${pvcReadError}. Proceeding with normal wait.`, `Could not read PVC to check storage class: ${pvcReadError}. Proceeding with normal wait.`,
); );
} }
if (shouldSkipWait) { if (shouldSkipWait) {
CloudRunnerLogger.log(`Skipping PVC wait - will bind when pod is created`); OrchestratorLogger.log(`Skipping PVC wait - will bind when pod is created`);
return; return;
} }
const initialPhase = await this.getPVCPhase(kubeClient, name, namespace); const initialPhase = await this.getPVCPhase(kubeClient, name, namespace);
CloudRunnerLogger.log(`Initial PVC phase: ${initialPhase}`); OrchestratorLogger.log(`Initial PVC phase: ${initialPhase}`);
// Wait until PVC is NOT Pending (i.e., Bound or Available) // Wait until PVC is NOT Pending (i.e., Bound or Available)
await waitUntil( await waitUntil(
@@ -104,7 +104,7 @@ class KubernetesStorage {
// Log progress every 4 checks (every ~60 seconds) // Log progress every 4 checks (every ~60 seconds)
if (checkCount % 4 === 0) { if (checkCount % 4 === 0) {
CloudRunnerLogger.log(`PVC ${name} still ${phase} (check ${checkCount})`); OrchestratorLogger.log(`PVC ${name} still ${phase} (check ${checkCount})`);
// Fetch and log PVC events for diagnostics // Fetch and log PVC events for diagnostics
try { try {
@@ -120,7 +120,7 @@ class KubernetesStorage {
.slice(-5); // Get last 5 events .slice(-5); // Get last 5 events
if (pvcEvents.length > 0) { if (pvcEvents.length > 0) {
CloudRunnerLogger.log(`PVC Events: ${JSON.stringify(pvcEvents, undefined, 2)}`); OrchestratorLogger.log(`PVC Events: ${JSON.stringify(pvcEvents, undefined, 2)}`);
// Check if event indicates WaitForFirstConsumer // Check if event indicates WaitForFirstConsumer
const waitForConsumerEvent = pvcEvents.find( const waitForConsumerEvent = pvcEvents.find(
@@ -128,7 +128,7 @@ class KubernetesStorage {
event.reason === 'WaitForFirstConsumer' || event.message?.includes('waiting for first consumer'), event.reason === 'WaitForFirstConsumer' || event.message?.includes('waiting for first consumer'),
); );
if (waitForConsumerEvent) { if (waitForConsumerEvent) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`PVC is waiting for first consumer. This is normal for WaitForFirstConsumer storage classes. Proceeding without waiting.`, `PVC is waiting for first consumer. This is normal for WaitForFirstConsumer storage classes. Proceeding without waiting.`,
); );
@@ -149,7 +149,7 @@ class KubernetesStorage {
); );
const finalPhase = await this.getPVCPhase(kubeClient, name, namespace); const finalPhase = await this.getPVCPhase(kubeClient, name, namespace);
CloudRunnerLogger.log(`PVC phase after wait: ${finalPhase}`); OrchestratorLogger.log(`PVC phase after wait: ${finalPhase}`);
if (finalPhase === 'Pending') { if (finalPhase === 'Pending') {
throw new Error(`PVC ${name} is still Pending after timeout`); throw new Error(`PVC ${name} is still Pending after timeout`);
@@ -266,9 +266,9 @@ class KubernetesStorage {
pvcName: string, pvcName: string,
) { ) {
const name = result.body.metadata?.name || ''; const name = result.body.metadata?.name || '';
CloudRunnerLogger.log(`PVC ${name} created`); OrchestratorLogger.log(`PVC ${name} created`);
await this.watchUntilPVCNotPending(kubeClient, name, namespace); await this.watchUntilPVCNotPending(kubeClient, name, namespace);
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`); OrchestratorLogger.log(`PVC ${name} is ready and not pending`);
core.setOutput('volume', pvcName); core.setOutput('volume', pvcName);
} }
} }

View File

@@ -1,8 +1,8 @@
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node'; import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { waitUntil } from 'async-wait-until'; import { waitUntil } from 'async-wait-until';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import KubernetesPods from './kubernetes-pods'; import KubernetesPods from './kubernetes-pods';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service'; import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
@@ -27,8 +27,8 @@ class KubernetesTaskRunner {
// eslint-disable-next-line no-constant-condition // eslint-disable-next-line no-constant-condition
while (true) { while (true) {
await new Promise((resolve) => setTimeout(resolve, 3000)); await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log( OrchestratorLogger.log(
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`, `Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${Orchestrator.buildParameters.kubeVolumeSize}/${Orchestrator.buildParameters.containerCpu}/${Orchestrator.buildParameters.containerMemory}`,
); );
const isRunning = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient); const isRunning = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
@@ -37,7 +37,7 @@ class KubernetesTaskRunner {
// These errors pollute the output and don't contain useful information // These errors pollute the output and don't contain useful information
const lowerChunk = outputChunk.toLowerCase(); const lowerChunk = outputChunk.toLowerCase();
if (lowerChunk.includes('unable to retrieve container logs')) { if (lowerChunk.includes('unable to retrieve container logs')) {
CloudRunnerLogger.log(`Filtered kubectl error: ${outputChunk.trim()}`); OrchestratorLogger.log(`Filtered kubectl error: ${outputChunk.trim()}`);
return; return;
} }
@@ -61,7 +61,7 @@ class KubernetesTaskRunner {
try { try {
// Always specify container name explicitly to avoid containerd:// errors // Always specify container name explicitly to avoid containerd:// errors
// Use -f for running pods, --previous for terminated pods // Use -f for running pods, --previous for terminated pods
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`kubectl logs ${podName} -c ${containerName} -n ${namespace}${isRunning ? ' -f' : ' --previous'}`, `kubectl logs ${podName} -c ${containerName} -n ${namespace}${isRunning ? ' -f' : ' --previous'}`,
false, false,
true, true,
@@ -74,7 +74,7 @@ class KubernetesTaskRunner {
kubectlLogsFailedCount++; kubectlLogsFailedCount++;
await new Promise((resolve) => setTimeout(resolve, 3000)); await new Promise((resolve) => setTimeout(resolve, 3000));
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient); const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`); OrchestratorLogger.log(`K8s logging error ${error} ${continueStreaming}`);
// Filter out kubectl error messages from the error output // Filter out kubectl error messages from the error output
const errorMessage = error?.message || error?.toString() || ''; const errorMessage = error?.message || error?.toString() || '';
@@ -83,14 +83,14 @@ class KubernetesTaskRunner {
errorMessage.toLowerCase().includes('unable to retrieve container logs'); errorMessage.toLowerCase().includes('unable to retrieve container logs');
if (isKubectlLogsError) { if (isKubectlLogsError) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Kubectl unable to retrieve logs, attempt ${kubectlLogsFailedCount}/${maxKubectlLogsFailures}`, `Kubectl unable to retrieve logs, attempt ${kubectlLogsFailedCount}/${maxKubectlLogsFailures}`,
); );
// If kubectl logs has failed multiple times, try reading the log file directly from the pod // If kubectl logs has failed multiple times, try reading the log file directly from the pod
// This works even if the pod is terminated, as long as it hasn't been deleted // This works even if the pod is terminated, as long as it hasn't been deleted
if (kubectlLogsFailedCount >= maxKubectlLogsFailures && !isRunning && !continueStreaming) { if (kubectlLogsFailedCount >= maxKubectlLogsFailures && !isRunning && !continueStreaming) {
CloudRunnerLogger.log(`Attempting to read log file directly from pod as fallback...`); OrchestratorLogger.log(`Attempting to read log file directly from pod as fallback...`);
try { try {
// Try to read the log file from the pod // Try to read the log file from the pod
// Use kubectl exec for running pods, or try to access via PVC if pod is terminated // Use kubectl exec for running pods, or try to access via PVC if pod is terminated
@@ -98,7 +98,7 @@ class KubernetesTaskRunner {
if (isRunning) { if (isRunning) {
// Pod is still running, try exec // Pod is still running, try exec
logFileContent = await CloudRunnerSystem.Run( logFileContent = await OrchestratorSystem.Run(
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`, `kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
true, true,
true, true,
@@ -106,15 +106,15 @@ class KubernetesTaskRunner {
} else { } else {
// Pod is terminated, try to create a temporary pod to read from the PVC // Pod is terminated, try to create a temporary pod to read from the PVC
// First, check if we can still access the pod's filesystem // First, check if we can still access the pod's filesystem
CloudRunnerLogger.log(`Pod is terminated, attempting to read log file via temporary pod...`); OrchestratorLogger.log(`Pod is terminated, attempting to read log file via temporary pod...`);
// For terminated pods, we might not be able to exec, so we'll skip this fallback // For terminated pods, we might not be able to exec, so we'll skip this fallback
// and rely on the log file being written to the PVC (if mounted) // and rely on the log file being written to the PVC (if mounted)
CloudRunnerLogger.logWarning(`Cannot read log file from terminated pod via exec`); OrchestratorLogger.logWarning(`Cannot read log file from terminated pod via exec`);
} }
if (logFileContent && logFileContent.trim()) { if (logFileContent && logFileContent.trim()) {
CloudRunnerLogger.log(`Successfully read log file from pod (${logFileContent.length} chars)`); OrchestratorLogger.log(`Successfully read log file from pod (${logFileContent.length} chars)`);
// Process the log file content line by line // Process the log file content line by line
for (const line of logFileContent.split(`\n`)) { for (const line of logFileContent.split(`\n`)) {
@@ -131,18 +131,18 @@ class KubernetesTaskRunner {
// Check if we got the end of transmission marker // Check if we got the end of transmission marker
if (FollowLogStreamService.DidReceiveEndOfTransmission) { if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream (from log file)'); OrchestratorLogger.log('end of log stream (from log file)');
break; break;
} }
} else { } else {
CloudRunnerLogger.logWarning(`Log file read returned empty content, continuing with available logs`); OrchestratorLogger.logWarning(`Log file read returned empty content, continuing with available logs`);
// If we can't read the log file, break out of the loop to return whatever logs we have // If we can't read the log file, break out of the loop to return whatever logs we have
// This prevents infinite retries when kubectl logs consistently fails // This prevents infinite retries when kubectl logs consistently fails
break; break;
} }
} catch (execError: any) { } catch (execError: any) {
CloudRunnerLogger.logWarning(`Failed to read log file from pod: ${execError}`); OrchestratorLogger.logWarning(`Failed to read log file from pod: ${execError}`);
// If we've exhausted all options, break to return whatever logs we have // If we've exhausted all options, break to return whatever logs we have
break; break;
@@ -152,9 +152,9 @@ class KubernetesTaskRunner {
// If pod is not running and we tried --previous but it failed, try without --previous // If pod is not running and we tried --previous but it failed, try without --previous
if (!isRunning && !continueStreaming && error?.message?.includes('previous terminated container')) { if (!isRunning && !continueStreaming && error?.message?.includes('previous terminated container')) {
CloudRunnerLogger.log(`Previous container not found, trying current container logs...`); OrchestratorLogger.log(`Previous container not found, trying current container logs...`);
try { try {
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`kubectl logs ${podName} -c ${containerName} -n ${namespace}`, `kubectl logs ${podName} -c ${containerName} -n ${namespace}`,
false, false,
true, true,
@@ -163,7 +163,7 @@ class KubernetesTaskRunner {
// If we successfully got logs, check for end of transmission // If we successfully got logs, check for end of transmission
if (FollowLogStreamService.DidReceiveEndOfTransmission) { if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream'); OrchestratorLogger.log('end of log stream');
break; break;
} }
@@ -176,7 +176,7 @@ class KubernetesTaskRunner {
// If we've exhausted retries, break // If we've exhausted retries, break
break; break;
} catch (fallbackError: any) { } catch (fallbackError: any) {
CloudRunnerLogger.log(`Fallback log fetch also failed: ${fallbackError}`); OrchestratorLogger.log(`Fallback log fetch also failed: ${fallbackError}`);
// If both fail, continue retrying if we haven't exhausted retries // If both fail, continue retrying if we haven't exhausted retries
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) { if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
@@ -185,7 +185,7 @@ class KubernetesTaskRunner {
} }
// Only break if we've exhausted all retries // Only break if we've exhausted all retries
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Could not fetch any container logs after ${KubernetesTaskRunner.maxRetry} retries`, `Could not fetch any container logs after ${KubernetesTaskRunner.maxRetry} retries`,
); );
break; break;
@@ -206,13 +206,13 @@ class KubernetesTaskRunner {
} }
// For previous container errors, we've already tried fallback, so just break // For previous container errors, we've already tried fallback, so just break
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Could not fetch previous container logs after retries, but continuing with available logs`, `Could not fetch previous container logs after retries, but continuing with available logs`,
); );
break; break;
} }
if (FollowLogStreamService.DidReceiveEndOfTransmission) { if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream'); OrchestratorLogger.log('end of log stream');
break; break;
} }
} }
@@ -225,7 +225,7 @@ class KubernetesTaskRunner {
const missingCollectedLogs = !output.includes('Collected Logs'); const missingCollectedLogs = !output.includes('Collected Logs');
if (needsFallback) { if (needsFallback) {
CloudRunnerLogger.log('Output is empty, attempting aggressive log collection fallback...'); OrchestratorLogger.log('Output is empty, attempting aggressive log collection fallback...');
// Give the pod a moment to finish writing logs before we try to read them // Give the pod a moment to finish writing logs before we try to read them
await new Promise((resolve) => setTimeout(resolve, 5000)); await new Promise((resolve) => setTimeout(resolve, 5000));
@@ -243,7 +243,7 @@ class KubernetesTaskRunner {
: missingCollectedLogs : missingCollectedLogs
? 'Collected Logs missing from output' ? 'Collected Logs missing from output'
: 'pod is terminated'; : 'pod is terminated';
CloudRunnerLogger.log( OrchestratorLogger.log(
`Pod is ${isPodStillRunning ? 'running' : 'terminated'} and ${reason}, reading log file as fallback...`, `Pod is ${isPodStillRunning ? 'running' : 'terminated'} and ${reason}, reading log file as fallback...`,
); );
try { try {
@@ -274,17 +274,17 @@ class KubernetesTaskRunner {
for (const attempt of attempts) { for (const attempt of attempts) {
// If we already have content with "Collected Logs", no need to try more // If we already have content with "Collected Logs", no need to try more
if (logFileContent && logFileContent.trim() && logFileContent.includes('Collected Logs')) { if (logFileContent && logFileContent.trim() && logFileContent.includes('Collected Logs')) {
CloudRunnerLogger.log('Found "Collected Logs" in fallback content, stopping attempts.'); OrchestratorLogger.log('Found "Collected Logs" in fallback content, stopping attempts.');
break; break;
} }
try { try {
CloudRunnerLogger.log(`Trying fallback method: ${attempt.slice(0, 80)}...`); OrchestratorLogger.log(`Trying fallback method: ${attempt.slice(0, 80)}...`);
const result = await CloudRunnerSystem.Run(attempt, true, true); const result = await OrchestratorSystem.Run(attempt, true, true);
if (result && result.trim()) { if (result && result.trim()) {
// Prefer content that has "Collected Logs" over content that doesn't // Prefer content that has "Collected Logs" over content that doesn't
if (!logFileContent || !logFileContent.includes('Collected Logs')) { if (!logFileContent || !logFileContent.includes('Collected Logs')) {
logFileContent = result; logFileContent = result;
CloudRunnerLogger.log( OrchestratorLogger.log(
`Successfully read logs using fallback method (${logFileContent.length} chars): ${attempt.slice( `Successfully read logs using fallback method (${logFileContent.length} chars): ${attempt.slice(
0, 0,
50, 50,
@@ -293,17 +293,17 @@ class KubernetesTaskRunner {
// If this content has "Collected Logs", we're done // If this content has "Collected Logs", we're done
if (logFileContent.includes('Collected Logs')) { if (logFileContent.includes('Collected Logs')) {
CloudRunnerLogger.log('Fallback method successfully captured "Collected Logs".'); OrchestratorLogger.log('Fallback method successfully captured "Collected Logs".');
break; break;
} }
} else { } else {
CloudRunnerLogger.log(`Skipping this result - already have content with "Collected Logs".`); OrchestratorLogger.log(`Skipping this result - already have content with "Collected Logs".`);
} }
} else { } else {
CloudRunnerLogger.log(`Fallback method returned empty result: ${attempt.slice(0, 50)}...`); OrchestratorLogger.log(`Fallback method returned empty result: ${attempt.slice(0, 50)}...`);
} }
} catch (attemptError: any) { } catch (attemptError: any) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Fallback method failed: ${attempt.slice(0, 50)}... Error: ${attemptError?.message || attemptError}`, `Fallback method failed: ${attempt.slice(0, 50)}... Error: ${attemptError?.message || attemptError}`,
); );
@@ -312,13 +312,13 @@ class KubernetesTaskRunner {
} }
if (!logFileContent || !logFileContent.trim()) { if (!logFileContent || !logFileContent.trim()) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
'Could not read log file from pod after all fallback attempts (may be OOM-killed or pod not accessible).', 'Could not read log file from pod after all fallback attempts (may be OOM-killed or pod not accessible).',
); );
} }
if (logFileContent && logFileContent.trim()) { if (logFileContent && logFileContent.trim()) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Read log file from pod as fallback (${logFileContent.length} chars) to capture missing messages`, `Read log file from pod as fallback (${logFileContent.length} chars) to capture missing messages`,
); );
@@ -348,7 +348,7 @@ class KubernetesTaskRunner {
} }
} }
} catch (logFileError: any) { } catch (logFileError: any) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Could not read log file from pod as fallback: ${logFileError?.message || logFileError}`, `Could not read log file from pod as fallback: ${logFileError?.message || logFileError}`,
); );
@@ -359,7 +359,7 @@ class KubernetesTaskRunner {
// If output is still empty or missing "Collected Logs" after fallback attempts, add a warning message // If output is still empty or missing "Collected Logs" after fallback attempts, add a warning message
// This ensures BuildResults is not completely empty, which would cause test failures // This ensures BuildResults is not completely empty, which would cause test failures
if ((needsFallback && output.trim().length === 0) || (!output.includes('Collected Logs') && shouldTryFallback)) { if ((needsFallback && output.trim().length === 0) || (!output.includes('Collected Logs') && shouldTryFallback)) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
'Could not retrieve "Collected Logs" from pod after all attempts. Pod may have been killed before logs were written.', 'Could not retrieve "Collected Logs" from pod after all attempts. Pod may have been killed before logs were written.',
); );
@@ -374,7 +374,7 @@ class KubernetesTaskRunner {
} }
} }
} catch (fallbackError: any) { } catch (fallbackError: any) {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Error checking pod status for log file fallback: ${fallbackError?.message || fallbackError}`, `Error checking pod status for log file fallback: ${fallbackError?.message || fallbackError}`,
); );
@@ -397,7 +397,7 @@ class KubernetesTaskRunner {
const originalLineCount = lines.length; const originalLineCount = lines.length;
const filteredLineCount = filteredLines.length; const filteredLineCount = filteredLines.length;
if (originalLineCount > filteredLineCount) { if (originalLineCount > filteredLineCount) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Filtered out ${originalLineCount - filteredLineCount} kubectl error message(s) from output`, `Filtered out ${originalLineCount - filteredLineCount} kubectl error message(s) from output`,
); );
} }
@@ -410,7 +410,7 @@ class KubernetesTaskRunner {
let message = ``; let message = ``;
let lastPhase = ''; let lastPhase = '';
let consecutivePendingCount = 0; let consecutivePendingCount = 0;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`); OrchestratorLogger.log(`Watching ${podName} ${namespace}`);
try { try {
await waitUntil( await waitUntil(
@@ -422,7 +422,7 @@ class KubernetesTaskRunner {
// Log phase changes // Log phase changes
if (phase !== lastPhase) { if (phase !== lastPhase) {
CloudRunnerLogger.log(`Pod ${podName} phase changed: ${lastPhase} -> ${phase}`); OrchestratorLogger.log(`Pod ${podName} phase changed: ${lastPhase} -> ${phase}`);
lastPhase = phase; lastPhase = phase;
consecutivePendingCount = 0; consecutivePendingCount = 0;
} }
@@ -481,7 +481,7 @@ class KubernetesTaskRunner {
// Ignore event fetch errors // Ignore event fetch errors
} }
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
// For permanent failures, mark as incomplete and store the error message // For permanent failures, mark as incomplete and store the error message
// We'll throw an error after the wait loop exits // We'll throw an error after the wait loop exits
@@ -496,7 +496,7 @@ class KubernetesTaskRunner {
// If pod completed (Succeeded/Failed), log it but don't throw - we'll try to get logs // If pod completed (Succeeded/Failed), log it but don't throw - we'll try to get logs
if (waitComplete && phase !== 'Running') { if (waitComplete && phase !== 'Running') {
CloudRunnerLogger.log(`Pod ${podName} completed with phase: ${phase}. Will attempt to retrieve logs.`); OrchestratorLogger.log(`Pod ${podName} completed with phase: ${phase}. Will attempt to retrieve logs.`);
} }
if (phase === 'Pending') { if (phase === 'Pending') {
@@ -515,7 +515,7 @@ class KubernetesTaskRunner {
.map((x) => `${x.reason}: ${x.message || ''}`) .map((x) => `${x.reason}: ${x.message || ''}`)
.join('; '); .join('; ');
message = `Pod ${podName} cannot be scheduled:\n${schedulingMessage}`; message = `Pod ${podName} cannot be scheduled:\n${schedulingMessage}`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
waitComplete = false; waitComplete = false;
return true; // Exit wait loop to throw error return true; // Exit wait loop to throw error
@@ -531,7 +531,7 @@ class KubernetesTaskRunner {
if (hasImagePullError) { if (hasImagePullError) {
message = `Pod ${podName} failed to pull image. Check image availability and credentials.`; message = `Pod ${podName} failed to pull image. Check image availability and credentials.`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
waitComplete = false; waitComplete = false;
return true; // Exit wait loop to throw error return true; // Exit wait loop to throw error
@@ -540,7 +540,7 @@ class KubernetesTaskRunner {
// If actively pulling image, reset pending count to allow more time // If actively pulling image, reset pending count to allow more time
// Large images (like Unity 3.9GB) can take 3-5 minutes to pull // Large images (like Unity 3.9GB) can take 3-5 minutes to pull
if (isPullingImage && consecutivePendingCount > 4) { if (isPullingImage && consecutivePendingCount > 4) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Pod ${podName} is pulling image (check ${consecutivePendingCount}). This may take several minutes for large images.`, `Pod ${podName} is pulling image (check ${consecutivePendingCount}). This may take several minutes for large images.`,
); );
@@ -553,7 +553,7 @@ class KubernetesTaskRunner {
// For tests, allow more time if image is being pulled (large images need 5+ minutes) // For tests, allow more time if image is being pulled (large images need 5+ minutes)
// Otherwise fail faster if stuck in Pending (2 minutes = 8 checks at 15s interval) // Otherwise fail faster if stuck in Pending (2 minutes = 8 checks at 15s interval)
const isTest = process.env['cloudRunnerTests'] === 'true'; const isTest = process.env['orchestratorTests'] === 'true';
const isPullingImage = const isPullingImage =
containerStatuses.some( containerStatuses.some(
(cs: any) => cs.state?.waiting?.reason === 'ImagePull' || cs.state?.waiting?.reason === 'ErrImagePull', (cs: any) => cs.state?.waiting?.reason === 'ImagePull' || cs.state?.waiting?.reason === 'ErrImagePull',
@@ -657,7 +657,7 @@ class KubernetesTaskRunner {
} catch { } catch {
// Ignore event fetch errors // Ignore event fetch errors
} }
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
waitComplete = false; waitComplete = false;
return true; // Exit wait loop to throw error return true; // Exit wait loop to throw error
@@ -669,7 +669,7 @@ class KubernetesTaskRunner {
const conditionMessages = conditions const conditionMessages = conditions
.map((c: any) => `${c.type}: ${c.reason || 'N/A'} - ${c.message || 'N/A'}`) .map((c: any) => `${c.type}: ${c.reason || 'N/A'} - ${c.message || 'N/A'}`)
.join('; '); .join('; ');
CloudRunnerLogger.log(`${pendingMessage}. Conditions: ${conditionMessages || 'None'}`); OrchestratorLogger.log(`${pendingMessage}. Conditions: ${conditionMessages || 'None'}`);
// Log events periodically to help diagnose // Log events periodically to help diagnose
if (consecutivePendingCount % 8 === 0) { if (consecutivePendingCount % 8 === 0) {
@@ -681,7 +681,7 @@ class KubernetesTaskRunner {
.map((x) => `${x.type}: ${x.reason} - ${x.message}`) .map((x) => `${x.type}: ${x.reason} - ${x.message}`)
.join('; '); .join('; ');
if (podEvents) { if (podEvents) {
CloudRunnerLogger.log(`Recent pod events: ${podEvents}`); OrchestratorLogger.log(`Recent pod events: ${podEvents}`);
} }
} catch { } catch {
// Ignore event fetch errors // Ignore event fetch errors
@@ -699,7 +699,7 @@ class KubernetesTaskRunner {
return false; return false;
}, },
{ {
timeout: process.env['cloudRunnerTests'] === 'true' ? 300000 : 2000000, // 5 minutes for tests, ~33 minutes for production timeout: process.env['orchestratorTests'] === 'true' ? 300000 : 2000000, // 5 minutes for tests, ~33 minutes for production
intervalBetweenAttempts: 15000, // 15 seconds intervalBetweenAttempts: 15000, // 15 seconds
}, },
); );
@@ -726,10 +726,10 @@ class KubernetesTaskRunner {
// Ignore event fetch errors // Ignore event fetch errors
} }
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
} catch { } catch {
message = `Pod ${podName} timed out and could not retrieve final status: ${waitError?.message || waitError}`; message = `Pod ${podName} timed out and could not retrieve final status: ${waitError?.message || waitError}`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
} }
throw new Error(`Pod ${podName} failed to start within timeout. ${message}`); throw new Error(`Pod ${podName} failed to start within timeout. ${message}`);
@@ -743,7 +743,7 @@ class KubernetesTaskRunner {
const finalStatus = await kubeClient.readNamespacedPodStatus(podName, namespace); const finalStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
const finalPhase = finalStatus?.body.status?.phase || 'Unknown'; const finalPhase = finalStatus?.body.status?.phase || 'Unknown';
if (finalPhase === 'Failed' || finalPhase === 'Succeeded') { if (finalPhase === 'Failed' || finalPhase === 'Succeeded') {
CloudRunnerLogger.logWarning( OrchestratorLogger.logWarning(
`Pod ${podName} completed with phase ${finalPhase} before reaching Running state. Will attempt to retrieve logs.`, `Pod ${podName} completed with phase ${finalPhase} before reaching Running state. Will attempt to retrieve logs.`,
); );
@@ -752,7 +752,7 @@ class KubernetesTaskRunner {
} catch { } catch {
// If we can't check status, fall through to throw error // If we can't check status, fall through to throw error
} }
CloudRunnerLogger.logWarning(`Pod ${podName} did not reach running state: ${message}`); OrchestratorLogger.logWarning(`Pod ${podName} did not reach running state: ${message}`);
throw new Error(`Pod ${podName} did not start successfully: ${message}`); throw new Error(`Pod ${podName} did not start successfully: ${message}`);
} }

View File

@@ -1,14 +1,14 @@
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { ProviderInterface } from '../provider-interface'; import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
import { quote } from 'shell-quote'; import { quote } from 'shell-quote';
class LocalCloudRunner implements ProviderInterface { class LocalOrchestrator implements ProviderInterface {
listResources(): Promise<ProviderResource[]> { listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');
} }
@@ -59,13 +59,13 @@ class LocalCloudRunner implements ProviderInterface {
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
workingdir: string, workingdir: string,
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string> { ): Promise<string> {
CloudRunnerLogger.log(image); OrchestratorLogger.log(image);
CloudRunnerLogger.log(buildGuid); OrchestratorLogger.log(buildGuid);
CloudRunnerLogger.log(commands); OrchestratorLogger.log(commands);
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available. // On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
if (process.platform === 'win32') { if (process.platform === 'win32') {
@@ -78,10 +78,10 @@ class LocalCloudRunner implements ProviderInterface {
// Use shell-quote to properly escape the command string, preventing command injection // Use shell-quote to properly escape the command string, preventing command injection
const bashWrapped = `bash -lc ${quote([inline])}`; const bashWrapped = `bash -lc ${quote([inline])}`;
return await CloudRunnerSystem.Run(bashWrapped); return await OrchestratorSystem.Run(bashWrapped);
} }
return await CloudRunnerSystem.Run(commands); return await OrchestratorSystem.Run(commands);
} }
} }
export default LocalCloudRunner; export default LocalOrchestrator;

View File

@@ -2,7 +2,7 @@ import { exec } from 'child_process';
import { promisify } from 'util'; import { promisify } from 'util';
import * as fs from 'fs'; import * as fs from 'fs';
import path from 'path'; import path from 'path';
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser'; import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser';
const execAsync = promisify(exec); const execAsync = promisify(exec);
@@ -32,7 +32,7 @@ export class ProviderGitManager {
private static ensureCacheDir(): void { private static ensureCacheDir(): void {
if (!fs.existsSync(this.CACHE_DIR)) { if (!fs.existsSync(this.CACHE_DIR)) {
fs.mkdirSync(this.CACHE_DIR, { recursive: true }); fs.mkdirSync(this.CACHE_DIR, { recursive: true });
CloudRunnerLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`); OrchestratorLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`);
} }
} }
@@ -69,15 +69,15 @@ export class ProviderGitManager {
// Remove existing directory if it exists // Remove existing directory if it exists
if (fs.existsSync(localPath)) { if (fs.existsSync(localPath)) {
CloudRunnerLogger.log(`Removing existing directory: ${localPath}`); OrchestratorLogger.log(`Removing existing directory: ${localPath}`);
fs.rmSync(localPath, { recursive: true, force: true }); fs.rmSync(localPath, { recursive: true, force: true });
} }
try { try {
CloudRunnerLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`); OrchestratorLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`; const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
CloudRunnerLogger.log(`Executing: ${cloneCommand}`); OrchestratorLogger.log(`Executing: ${cloneCommand}`);
const { stderr } = await execAsync(cloneCommand, { const { stderr } = await execAsync(cloneCommand, {
timeout: this.GIT_TIMEOUT, timeout: this.GIT_TIMEOUT,
@@ -85,10 +85,10 @@ export class ProviderGitManager {
}); });
if (stderr && !stderr.includes('warning')) { if (stderr && !stderr.includes('warning')) {
CloudRunnerLogger.log(`Git clone stderr: ${stderr}`); OrchestratorLogger.log(`Git clone stderr: ${stderr}`);
} }
CloudRunnerLogger.log(`Successfully cloned repository to: ${localPath}`); OrchestratorLogger.log(`Successfully cloned repository to: ${localPath}`);
return { return {
success: true, success: true,
@@ -96,7 +96,7 @@ export class ProviderGitManager {
}; };
} catch (error: any) { } catch (error: any) {
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`; const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
CloudRunnerLogger.log(`Error: ${errorMessage}`); OrchestratorLogger.log(`Error: ${errorMessage}`);
return { return {
success: false, success: false,
@@ -123,7 +123,7 @@ export class ProviderGitManager {
} }
try { try {
CloudRunnerLogger.log(`Updating repository: ${localPath}`); OrchestratorLogger.log(`Updating repository: ${localPath}`);
// Fetch latest changes // Fetch latest changes
await execAsync('git fetch origin', { await execAsync('git fetch origin', {
@@ -141,7 +141,7 @@ export class ProviderGitManager {
statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded'); statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
if (hasUpdates) { if (hasUpdates) {
CloudRunnerLogger.log(`Updates available, pulling latest changes...`); OrchestratorLogger.log(`Updates available, pulling latest changes...`);
// Reset to origin/branch to get latest changes // Reset to origin/branch to get latest changes
await execAsync(`git reset --hard origin/${urlInfo.branch}`, { await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
@@ -149,14 +149,14 @@ export class ProviderGitManager {
cwd: localPath, cwd: localPath,
}); });
CloudRunnerLogger.log(`Repository updated successfully`); OrchestratorLogger.log(`Repository updated successfully`);
return { return {
success: true, success: true,
updated: true, updated: true,
}; };
} else { } else {
CloudRunnerLogger.log(`Repository is already up to date`); OrchestratorLogger.log(`Repository is already up to date`);
return { return {
success: true, success: true,
@@ -165,7 +165,7 @@ export class ProviderGitManager {
} }
} catch (error: any) { } catch (error: any) {
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`; const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
CloudRunnerLogger.log(`Error: ${errorMessage}`); OrchestratorLogger.log(`Error: ${errorMessage}`);
return { return {
success: false, success: false,
@@ -184,11 +184,11 @@ export class ProviderGitManager {
this.ensureCacheDir(); this.ensureCacheDir();
if (this.isRepositoryCloned(urlInfo)) { if (this.isRepositoryCloned(urlInfo)) {
CloudRunnerLogger.log(`Repository already exists locally, checking for updates...`); OrchestratorLogger.log(`Repository already exists locally, checking for updates...`);
const updateResult = await this.updateRepository(urlInfo); const updateResult = await this.updateRepository(urlInfo);
if (!updateResult.success) { if (!updateResult.success) {
CloudRunnerLogger.log(`Failed to update repository, attempting fresh clone...`); OrchestratorLogger.log(`Failed to update repository, attempting fresh clone...`);
const cloneResult = await this.cloneRepository(urlInfo); const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) { if (!cloneResult.success) {
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`); throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
@@ -199,7 +199,7 @@ export class ProviderGitManager {
return this.getLocalPath(urlInfo); return this.getLocalPath(urlInfo);
} else { } else {
CloudRunnerLogger.log(`Repository not found locally, cloning...`); OrchestratorLogger.log(`Repository not found locally, cloning...`);
const cloneResult = await this.cloneRepository(urlInfo); const cloneResult = await this.cloneRepository(urlInfo);
if (!cloneResult.success) { if (!cloneResult.success) {
@@ -236,14 +236,14 @@ export class ProviderGitManager {
for (const entryPoint of commonEntryPoints) { for (const entryPoint of commonEntryPoints) {
const fullPath = path.join(localPath, entryPoint); const fullPath = path.join(localPath, entryPoint);
if (fs.existsSync(fullPath)) { if (fs.existsSync(fullPath)) {
CloudRunnerLogger.log(`Found provider entry point: ${entryPoint}`); OrchestratorLogger.log(`Found provider entry point: ${entryPoint}`);
return fullPath; return fullPath;
} }
} }
// Default to repository root // Default to repository root
CloudRunnerLogger.log(`No specific entry point found, using repository root`); OrchestratorLogger.log(`No specific entry point found, using repository root`);
return localPath; return localPath;
} }
@@ -266,13 +266,13 @@ export class ProviderGitManager {
const stats = fs.statSync(entryPath); const stats = fs.statSync(entryPath);
if (now - stats.mtime.getTime() > maxAge) { if (now - stats.mtime.getTime() > maxAge) {
CloudRunnerLogger.log(`Cleaning up old repository: ${entry.name}`); OrchestratorLogger.log(`Cleaning up old repository: ${entry.name}`);
fs.rmSync(entryPath, { recursive: true, force: true }); fs.rmSync(entryPath, { recursive: true, force: true });
} }
} }
} }
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Error during cleanup: ${error.message}`); OrchestratorLogger.log(`Error during cleanup: ${error.message}`);
} }
} }
} }

View File

@@ -1,6 +1,6 @@
import BuildParameters from '../../build-parameters'; import BuildParameters from '../../build-parameters';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../options/orchestrator-environment-variable';
import CloudRunnerSecret from '../options/cloud-runner-secret'; import OrchestratorSecret from '../options/orchestrator-secret';
import { ProviderResource } from './provider-resource'; import { ProviderResource } from './provider-resource';
import { ProviderWorkflow } from './provider-workflow'; import { ProviderWorkflow } from './provider-workflow';
@@ -35,9 +35,9 @@ export interface ProviderInterface {
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
workingdir: string, workingdir: string,
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string>; ): Promise<string>;
garbageCollect( garbageCollect(
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars

View File

@@ -1,6 +1,6 @@
import { ProviderInterface } from './provider-interface'; import { ProviderInterface } from './provider-interface';
import BuildParameters from '../../build-parameters'; import BuildParameters from '../../build-parameters';
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser'; import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser';
import { ProviderGitManager } from './provider-git-manager'; import { ProviderGitManager } from './provider-git-manager';
@@ -16,7 +16,7 @@ export default async function loadProvider(
providerSource: string, providerSource: string,
buildParameters: BuildParameters, buildParameters: BuildParameters,
): Promise<ProviderInterface> { ): Promise<ProviderInterface> {
CloudRunnerLogger.log(`Loading provider: ${providerSource}`); OrchestratorLogger.log(`Loading provider: ${providerSource}`);
// Parse the provider source to determine its type // Parse the provider source to determine its type
const sourceInfo = parseProviderSource(providerSource); const sourceInfo = parseProviderSource(providerSource);
@@ -29,7 +29,7 @@ export default async function loadProvider(
// Handle different source types // Handle different source types
switch (sourceInfo.type) { switch (sourceInfo.type) {
case 'github': { case 'github': {
CloudRunnerLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`); OrchestratorLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
// Ensure the repository is available locally // Ensure the repository is available locally
const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo); const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
@@ -37,19 +37,19 @@ export default async function loadProvider(
// Get the path to the provider module within the repository // Get the path to the provider module within the repository
modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath); modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
CloudRunnerLogger.log(`Loading provider from: ${modulePath}`); OrchestratorLogger.log(`Loading provider from: ${modulePath}`);
break; break;
} }
case 'local': { case 'local': {
modulePath = sourceInfo.path; modulePath = sourceInfo.path;
CloudRunnerLogger.log(`Loading provider from local path: ${modulePath}`); OrchestratorLogger.log(`Loading provider from local path: ${modulePath}`);
break; break;
} }
case 'npm': { case 'npm': {
modulePath = sourceInfo.packageName; modulePath = sourceInfo.packageName;
CloudRunnerLogger.log(`Loading provider from NPM package: ${modulePath}`); OrchestratorLogger.log(`Loading provider from NPM package: ${modulePath}`);
break; break;
} }
@@ -65,7 +65,7 @@ export default async function loadProvider(
}; };
modulePath = providerModuleMap[providerSource] || providerSource; modulePath = providerModuleMap[providerSource] || providerSource;
CloudRunnerLogger.log(`Loading provider from module path: ${modulePath}`); OrchestratorLogger.log(`Loading provider from module path: ${modulePath}`);
break; break;
} }
} }
@@ -111,7 +111,7 @@ export default async function loadProvider(
} }
} }
CloudRunnerLogger.log(`Successfully loaded provider: ${providerSource}`); OrchestratorLogger.log(`Successfully loaded provider: ${providerSource}`);
return instance as ProviderInterface; return instance as ProviderInterface;
} }

View File

@@ -1,4 +1,4 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
export interface GitHubUrlInfo { export interface GitHubUrlInfo {
type: 'github'; type: 'github';
@@ -115,24 +115,24 @@ export function isGitHubSource(source: string): boolean {
* @param parsed The parsed source information * @param parsed The parsed source information
*/ */
export function logProviderSource(source: string, parsed: ProviderSourceInfo): void { export function logProviderSource(source: string, parsed: ProviderSourceInfo): void {
CloudRunnerLogger.log(`Provider source: ${source}`); OrchestratorLogger.log(`Provider source: ${source}`);
switch (parsed.type) { switch (parsed.type) {
case 'github': case 'github':
CloudRunnerLogger.log(` Type: GitHub repository`); OrchestratorLogger.log(` Type: GitHub repository`);
CloudRunnerLogger.log(` Owner: ${parsed.owner}`); OrchestratorLogger.log(` Owner: ${parsed.owner}`);
CloudRunnerLogger.log(` Repository: ${parsed.repo}`); OrchestratorLogger.log(` Repository: ${parsed.repo}`);
CloudRunnerLogger.log(` Branch: ${parsed.branch}`); OrchestratorLogger.log(` Branch: ${parsed.branch}`);
if (parsed.path) { if (parsed.path) {
CloudRunnerLogger.log(` Path: ${parsed.path}`); OrchestratorLogger.log(` Path: ${parsed.path}`);
} }
break; break;
case 'local': case 'local':
CloudRunnerLogger.log(` Type: Local path`); OrchestratorLogger.log(` Type: Local path`);
CloudRunnerLogger.log(` Path: ${parsed.path}`); OrchestratorLogger.log(` Path: ${parsed.path}`);
break; break;
case 'npm': case 'npm':
CloudRunnerLogger.log(` Type: NPM package`); OrchestratorLogger.log(` Type: NPM package`);
CloudRunnerLogger.log(` Package: ${parsed.packageName}`); OrchestratorLogger.log(` Package: ${parsed.packageName}`);
break; break;
} }
} }

View File

@@ -1,12 +1,12 @@
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { ProviderInterface } from '../provider-interface'; import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
class TestCloudRunner implements ProviderInterface { class TestOrchestrator implements ProviderInterface {
listResources(): Promise<ProviderResource[]> { listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');
} }
@@ -51,17 +51,17 @@ class TestCloudRunner implements ProviderInterface {
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
workingdir: string, workingdir: string,
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[], environment: OrchestratorEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[], secrets: OrchestratorSecret[],
): Promise<string> { ): Promise<string> {
CloudRunnerLogger.log(image); OrchestratorLogger.log(image);
CloudRunnerLogger.log(buildGuid); OrchestratorLogger.log(buildGuid);
CloudRunnerLogger.log(commands); OrchestratorLogger.log(commands);
return await new Promise((result) => { return await new Promise((result) => {
result(commands); result(commands);
}); });
} }
} }
export default TestCloudRunner; export default TestOrchestrator;

View File

@@ -1,10 +1,10 @@
import { assert } from 'node:console'; import { assert } from 'node:console';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders'; import { OrchestratorFolders } from '../options/orchestrator-folders';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../services/core/orchestrator-system';
import { LfsHashing } from '../services/utility/lfs-hashing'; import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger'; import { RemoteClientLogger } from './remote-client-logger';
import { Cli } from '../../cli/cli'; import { Cli } from '../../cli/cli';
@@ -17,14 +17,14 @@ export class Caching {
static async cachePush() { static async cachePush() {
try { try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}'); const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter; Orchestrator.buildParameters = buildParameter;
await Caching.PushToCache( await Caching.PushToCache(
Cli.options!['cachePushTo'], Cli.options!['cachePushTo'],
Cli.options!['cachePushFrom'], Cli.options!['cachePushFrom'],
Cli.options!['artifactName'] || '', Cli.options!['artifactName'] || '',
); );
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`${error}`); OrchestratorLogger.log(`${error}`);
} }
} }
@@ -32,46 +32,46 @@ export class Caching {
static async cachePull() { static async cachePull() {
try { try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}'); const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter; Orchestrator.buildParameters = buildParameter;
await Caching.PullFromCache( await Caching.PullFromCache(
Cli.options!['cachePushFrom'], Cli.options!['cachePushFrom'],
Cli.options!['cachePushTo'], Cli.options!['cachePushTo'],
Cli.options!['artifactName'] || '', Cli.options!['artifactName'] || '',
); );
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`${error}`); OrchestratorLogger.log(`${error}`);
} }
} }
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) { public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`); OrchestratorLogger.log(`Pushing to cache ${sourceFolder}`);
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd(); const startPath = process.cwd();
let compressionSuffix = ''; let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) { if (Orchestrator.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`; compressionSuffix = `.lz4`;
} }
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`); OrchestratorLogger.log(`Compression: ${Orchestrator.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
try { try {
if (!(await fileExists(cacheFolder))) { if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`); await OrchestratorSystem.Run(`mkdir -p ${cacheFolder}`);
} }
process.chdir(path.resolve(sourceFolder, '..')); process.chdir(path.resolve(sourceFolder, '..'));
if (CloudRunner.buildParameters.cloudRunnerDebug === true) { if (Orchestrator.buildParameters.orchestratorDebug === true) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename( `Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
sourceFolder, sourceFolder,
)}`, )}`,
); );
} }
const contents = await fs.promises.readdir(path.basename(sourceFolder)); const contents = await fs.promises.readdir(path.basename(sourceFolder));
CloudRunnerLogger.log( OrchestratorLogger.log(
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`, `There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
); );
if (contents.length === 0) { if (contents.length === 0) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`, `Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
); );
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
@@ -82,8 +82,8 @@ export class Caching {
// Check disk space before creating tar archive and clean up if needed // Check disk space before creating tar archive and clean up if needed
let diskUsagePercent = 0; let diskUsagePercent = 0;
try { try {
const diskCheckOutput = await CloudRunnerSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`); const diskCheckOutput = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
CloudRunnerLogger.log(`Disk space before tar: ${diskCheckOutput}`); OrchestratorLogger.log(`Disk space before tar: ${diskCheckOutput}`);
// Parse disk usage percentage (e.g., "72G 72G 196M 100%") // Parse disk usage percentage (e.g., "72G 72G 196M 100%")
const usageMatch = diskCheckOutput.match(/(\d+)%/); const usageMatch = diskCheckOutput.match(/(\d+)%/);
@@ -96,50 +96,52 @@ export class Caching {
// If disk usage is high (>90%), proactively clean up old cache files // If disk usage is high (>90%), proactively clean up old cache files
if (diskUsagePercent > 90) { if (diskUsagePercent > 90) {
CloudRunnerLogger.log(`Disk usage is ${diskUsagePercent}% - cleaning up old cache files before tar operation`); OrchestratorLogger.log(`Disk usage is ${diskUsagePercent}% - cleaning up old cache files before tar operation`);
try { try {
const cacheParent = path.dirname(cacheFolder); const cacheParent = path.dirname(cacheFolder);
if (await fileExists(cacheParent)) { if (await fileExists(cacheParent)) {
// Try to fix permissions first to avoid permission denied errors // Try to fix permissions first to avoid permission denied errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`, `chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`,
); );
// Remove cache files older than 6 hours (more aggressive than 1 day) // Remove cache files older than 6 hours (more aggressive than 1 day)
// Use multiple methods to handle permission issues // Use multiple methods to handle permission issues
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`, `find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`,
); );
// Try with sudo if available // Try with sudo if available
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`, `sudo find ${cacheParent} -name "*.tar*" -type f -mmin +360 -delete 2>/dev/null || true`,
); );
// As last resort, try to remove files one by one // As last resort, try to remove files one by one
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheParent} -name "*.tar*" -type f -mmin +360 -exec rm -f {} + 2>/dev/null || true`, `find ${cacheParent} -name "*.tar*" -type f -mmin +360 -exec rm -f {} + 2>/dev/null || true`,
); );
// Also try to remove old cache directories // Also try to remove old cache directories
await CloudRunnerSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`); await OrchestratorSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`);
// If disk is still very high (>95%), be even more aggressive // If disk is still very high (>95%), be even more aggressive
if (diskUsagePercent > 95) { if (diskUsagePercent > 95) {
CloudRunnerLogger.log(`Disk usage is very high (${diskUsagePercent}%), performing aggressive cleanup...`); OrchestratorLogger.log(
`Disk usage is very high (${diskUsagePercent}%), performing aggressive cleanup...`,
);
// Remove files older than 1 hour // Remove files older than 1 hour
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`, `find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
); );
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`, `sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
); );
} }
CloudRunnerLogger.log(`Cleanup completed. Checking disk space again...`); OrchestratorLogger.log(`Cleanup completed. Checking disk space again...`);
const diskCheckAfter = await CloudRunnerSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`); const diskCheckAfter = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
CloudRunnerLogger.log(`Disk space after cleanup: ${diskCheckAfter}`); OrchestratorLogger.log(`Disk space after cleanup: ${diskCheckAfter}`);
// Check disk usage again after cleanup // Check disk usage again after cleanup
let diskUsageAfterCleanup = 0; let diskUsageAfterCleanup = 0;
@@ -157,7 +159,7 @@ export class Caching {
// due to shared CI disk pressure. // due to shared CI disk pressure.
if (diskUsageAfterCleanup >= 100) { if (diskUsageAfterCleanup >= 100) {
const message = `Cannot create cache archive: disk is still at ${diskUsageAfterCleanup}% after cleanup. Tar operation would hang. Skipping cache push; please free up disk space manually if this persists.`; const message = `Cannot create cache archive: disk is still at ${diskUsageAfterCleanup}% after cleanup. Tar operation would hang. Skipping cache push; please free up disk space manually if this persists.`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
RemoteClientLogger.log(message); RemoteClientLogger.log(message);
// Restore working directory before early return // Restore working directory before early return
@@ -171,13 +173,13 @@ export class Caching {
if (cleanupError instanceof Error && cleanupError.message.includes('Cannot create cache archive')) { if (cleanupError instanceof Error && cleanupError.message.includes('Cannot create cache archive')) {
throw cleanupError; throw cleanupError;
} }
CloudRunnerLogger.log(`Proactive cleanup failed: ${cleanupError}`); OrchestratorLogger.log(`Proactive cleanup failed: ${cleanupError}`);
} }
} }
// Clean up any existing incomplete tar files // Clean up any existing incomplete tar files
try { try {
await CloudRunnerSystem.Run(`rm -f ${cacheArtifactName}.tar${compressionSuffix} 2>/dev/null || true`); await OrchestratorSystem.Run(`rm -f ${cacheArtifactName}.tar${compressionSuffix} 2>/dev/null || true`);
} catch { } catch {
// Ignore cleanup errors // Ignore cleanup errors
} }
@@ -190,7 +192,7 @@ export class Caching {
let tarCommandToRun = tarCommand; let tarCommandToRun = tarCommand;
try { try {
// Check if timeout command is available // Check if timeout command is available
await CloudRunnerSystem.Run(`which timeout > /dev/null 2>&1`, true, true); await OrchestratorSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
// Use timeout if available (600 seconds = 10 minutes) // Use timeout if available (600 seconds = 10 minutes)
tarCommandToRun = `timeout 600 ${tarCommand}`; tarCommandToRun = `timeout 600 ${tarCommand}`;
@@ -200,7 +202,7 @@ export class Caching {
tarCommandToRun = tarCommand; tarCommandToRun = tarCommand;
} }
await CloudRunnerSystem.Run(tarCommandToRun); await OrchestratorSystem.Run(tarCommandToRun);
} catch (error: any) { } catch (error: any) {
// Check if error is due to disk space or timeout // Check if error is due to disk space or timeout
const errorMessage = error?.message || error?.toString() || ''; const errorMessage = error?.message || error?.toString() || '';
@@ -210,56 +212,56 @@ export class Caching {
errorMessage.includes('timeout') || errorMessage.includes('timeout') ||
errorMessage.includes('Terminated') errorMessage.includes('Terminated')
) { ) {
CloudRunnerLogger.log(`Disk space error detected. Attempting aggressive cleanup...`); OrchestratorLogger.log(`Disk space error detected. Attempting aggressive cleanup...`);
// Try to clean up old cache files more aggressively // Try to clean up old cache files more aggressively
try { try {
const cacheParent = path.dirname(cacheFolder); const cacheParent = path.dirname(cacheFolder);
if (await fileExists(cacheParent)) { if (await fileExists(cacheParent)) {
// Try to fix permissions first to avoid permission denied errors // Try to fix permissions first to avoid permission denied errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`, `chmod -R u+w ${cacheParent} 2>/dev/null || chown -R $(whoami) ${cacheParent} 2>/dev/null || true`,
); );
// Remove cache files older than 1 hour (very aggressive) // Remove cache files older than 1 hour (very aggressive)
// Use multiple methods to handle permission issues // Use multiple methods to handle permission issues
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`, `find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
); );
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`, `sudo find ${cacheParent} -name "*.tar*" -type f -mmin +60 -delete 2>/dev/null || true`,
); );
// As last resort, try to remove files one by one // As last resort, try to remove files one by one
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheParent} -name "*.tar*" -type f -mmin +60 -exec rm -f {} + 2>/dev/null || true`, `find ${cacheParent} -name "*.tar*" -type f -mmin +60 -exec rm -f {} + 2>/dev/null || true`,
); );
// Remove empty cache directories // Remove empty cache directories
await CloudRunnerSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`); await OrchestratorSystem.Run(`find ${cacheParent} -type d -empty -delete 2>/dev/null || true`);
// Also try to clean up the entire cache folder if it's getting too large // Also try to clean up the entire cache folder if it's getting too large
const cacheRoot = path.resolve(cacheParent, '..'); const cacheRoot = path.resolve(cacheParent, '..');
if (await fileExists(cacheRoot)) { if (await fileExists(cacheRoot)) {
// Try to fix permissions for cache root too // Try to fix permissions for cache root too
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${cacheRoot} 2>/dev/null || chown -R $(whoami) ${cacheRoot} 2>/dev/null || true`, `chmod -R u+w ${cacheRoot} 2>/dev/null || chown -R $(whoami) ${cacheRoot} 2>/dev/null || true`,
); );
// Remove cache entries older than 30 minutes // Remove cache entries older than 30 minutes
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`, `find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`,
); );
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`sudo find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`, `sudo find ${cacheRoot} -name "*.tar*" -type f -mmin +30 -delete 2>/dev/null || true`,
); );
} }
CloudRunnerLogger.log(`Aggressive cleanup completed. Retrying tar operation...`); OrchestratorLogger.log(`Aggressive cleanup completed. Retrying tar operation...`);
// Retry the tar operation once after cleanup // Retry the tar operation once after cleanup
let retrySucceeded = false; let retrySucceeded = false;
try { try {
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`, `tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
); );
@@ -285,7 +287,7 @@ export class Caching {
); );
} }
} catch (cleanupError: any) { } catch (cleanupError: any) {
CloudRunnerLogger.log(`Cleanup attempt failed: ${cleanupError}`); OrchestratorLogger.log(`Cleanup attempt failed: ${cleanupError}`);
throw new Error( throw new Error(
`Failed to create cache archive due to insufficient disk space. Error: ${errorMessage}. Cleanup failed: ${ `Failed to create cache archive due to insufficient disk space. Error: ${errorMessage}. Cleanup failed: ${
cleanupError?.message || cleanupError cleanupError?.message || cleanupError
@@ -296,16 +298,16 @@ export class Caching {
throw error; throw error;
} }
} }
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`); await OrchestratorSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists'); assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists'); assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
// Ensure the cache folder directory exists before moving the file // Ensure the cache folder directory exists before moving the file
// (it might have been deleted by cleanup if it was empty) // (it might have been deleted by cleanup if it was empty)
if (!(await fileExists(cacheFolder))) { if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`); await OrchestratorSystem.Run(`mkdir -p ${cacheFolder}`);
} }
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`); await OrchestratorSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`); RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert( assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`), await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
@@ -318,15 +320,15 @@ export class Caching {
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
} }
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) { public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`); OrchestratorLogger.log(`Pulling from cache ${destinationFolder} ${Orchestrator.buildParameters.skipCache}`);
if (`${CloudRunner.buildParameters.skipCache}` === `true`) { if (`${Orchestrator.buildParameters.skipCache}` === `true`) {
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`); OrchestratorLogger.log(`Skipping cache debugSkipCache is true`);
return; return;
} }
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = ''; let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) { if (Orchestrator.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`; compressionSuffix = `.lz4`;
} }
const startPath = process.cwd(); const startPath = process.cwd();
@@ -341,7 +343,7 @@ export class Caching {
} }
const latestInBranch = await ( const latestInBranch = await (
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`) await OrchestratorSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
) )
.replace(/\n/g, ``) .replace(/\n/g, ``)
.replace(`.tar${compressionSuffix}`, ''); .replace(`.tar${compressionSuffix}`, '');
@@ -352,13 +354,13 @@ export class Caching {
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`)) cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName ? cacheArtifactName
: latestInBranch; : latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); await OrchestratorLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) { if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
// Check disk space before extraction to prevent hangs // Check disk space before extraction to prevent hangs
let diskUsagePercent = 0; let diskUsagePercent = 0;
try { try {
const diskCheckOutput = await CloudRunnerSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`); const diskCheckOutput = await OrchestratorSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
const usageMatch = diskCheckOutput.match(/(\d+)%/); const usageMatch = diskCheckOutput.match(/(\d+)%/);
if (usageMatch) { if (usageMatch) {
diskUsagePercent = Number.parseInt(usageMatch[1], 10); diskUsagePercent = Number.parseInt(usageMatch[1], 10);
@@ -370,7 +372,7 @@ export class Caching {
// If disk is at 100%, skip cache extraction to prevent hangs // If disk is at 100%, skip cache extraction to prevent hangs
if (diskUsagePercent >= 100) { if (diskUsagePercent >= 100) {
const message = `Disk is at ${diskUsagePercent}% - skipping cache extraction to prevent hang. Cache may be incomplete or corrupted.`; const message = `Disk is at ${diskUsagePercent}% - skipping cache extraction to prevent hang. Cache may be incomplete or corrupted.`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
RemoteClientLogger.logWarning(message); RemoteClientLogger.logWarning(message);
// Continue without cache - build will proceed without cached Library // Continue without cache - build will proceed without cached Library
@@ -383,12 +385,12 @@ export class Caching {
try { try {
// Use tar -t to test the archive without extracting (fast check) // Use tar -t to test the archive without extracting (fast check)
// This will fail if the archive is corrupted // This will fail if the archive is corrupted
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`tar -tf ${cacheSelection}.tar${compressionSuffix} > /dev/null 2>&1 || (echo "Tar file validation failed" && exit 1)`, `tar -tf ${cacheSelection}.tar${compressionSuffix} > /dev/null 2>&1 || (echo "Tar file validation failed" && exit 1)`,
); );
} catch { } catch {
const message = `Cache archive ${cacheSelection}.tar${compressionSuffix} appears to be corrupted or incomplete. Skipping cache extraction.`; const message = `Cache archive ${cacheSelection}.tar${compressionSuffix} appears to be corrupted or incomplete. Skipping cache extraction.`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
RemoteClientLogger.logWarning(message); RemoteClientLogger.logWarning(message);
// Continue without cache - build will proceed without cached Library // Continue without cache - build will proceed without cached Library
@@ -397,8 +399,8 @@ export class Caching {
return; return;
} }
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`; const resultsFolder = `results${Orchestrator.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); await OrchestratorSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`); RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder); const fullResultsFolder = path.join(cacheFolder, resultsFolder);
@@ -408,13 +410,13 @@ export class Caching {
// Add timeout if available (600 seconds = 10 minutes) // Add timeout if available (600 seconds = 10 minutes)
try { try {
await CloudRunnerSystem.Run(`which timeout > /dev/null 2>&1`, true, true); await OrchestratorSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
tarExtractCommand = `timeout 600 ${tarExtractCommand}`; tarExtractCommand = `timeout 600 ${tarExtractCommand}`;
} catch { } catch {
// timeout command not available, use regular tar // timeout command not available, use regular tar
} }
await CloudRunnerSystem.Run(tarExtractCommand); await OrchestratorSystem.Run(tarExtractCommand);
} catch (extractError: any) { } catch (extractError: any) {
const errorMessage = extractError?.message || extractError?.toString() || ''; const errorMessage = extractError?.message || extractError?.toString() || '';
@@ -427,7 +429,7 @@ export class Caching {
errorMessage.includes('Terminated') errorMessage.includes('Terminated')
) { ) {
const message = `Cache extraction failed (likely due to corrupted archive or disk space): ${errorMessage}. Continuing without cache.`; const message = `Cache extraction failed (likely due to corrupted archive or disk space): ${errorMessage}. Continuing without cache.`;
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
RemoteClientLogger.logWarning(message); RemoteClientLogger.logWarning(message);
// Continue without cache - build will proceed without cached Library // Continue without cache - build will proceed without cached Library
@@ -447,13 +449,13 @@ export class Caching {
if (await fileExists(destinationFolder)) { if (await fileExists(destinationFolder)) {
await fs.promises.rmdir(destinationFolder, { recursive: true }); await fs.promises.rmdir(destinationFolder, { recursive: true });
} }
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`, `mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
); );
const contents = await fs.promises.readdir( const contents = await fs.promises.readdir(
path.join(destinationParentFolder, path.basename(destinationFolder)), path.join(destinationParentFolder, path.basename(destinationFolder)),
); );
CloudRunnerLogger.log( OrchestratorLogger.log(
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`, `There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
); );
} else { } else {
@@ -474,8 +476,8 @@ export class Caching {
public static async handleCachePurging() { public static async handleCachePurging() {
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) { if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`); RemoteClientLogger.log(`purging ${OrchestratorFolders.purgeRemoteCaching}`);
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true }); fs.promises.rmdir(OrchestratorFolders.cacheFolder, { recursive: true });
} }
} }
} }

View File

@@ -1,25 +1,25 @@
import fs from 'node:fs'; import fs from 'node:fs';
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import { CloudRunnerFolders } from '../options/cloud-runner-folders'; import { OrchestratorFolders } from '../options/orchestrator-folders';
import { Caching } from './caching'; import { Caching } from './caching';
import { LfsHashing } from '../services/utility/lfs-hashing'; import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger'; import { RemoteClientLogger } from './remote-client-logger';
import path from 'node:path'; import path from 'node:path';
import { assert } from 'node:console'; import { assert } from 'node:console';
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
import { CliFunction } from '../../cli/cli-functions-repository'; import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../services/core/orchestrator-system';
import YAML from 'yaml'; import YAML from 'yaml';
import GitHub from '../../github'; import GitHub from '../../github';
import BuildParameters from '../../build-parameters'; import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli'; import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../options/cloud-runner-options'; import OrchestratorOptions from '../options/orchestrator-options';
import ResourceTracking from '../services/core/resource-tracking'; import ResourceTracking from '../services/core/resource-tracking';
export class RemoteClient { export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`) @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() { static async setupRemoteClient() {
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`); OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)'); await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
if (!(await RemoteClient.handleRetainedWorkspace())) { if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository(); await RemoteClient.bootstrapRepository();
@@ -35,7 +35,7 @@ export class RemoteClient {
process.stdin.setEncoding('utf8'); process.stdin.setEncoding('utf8');
// For K8s, ensure stdout is unbuffered so messages are captured immediately // For K8s, ensure stdout is unbuffered so messages are captured immediately
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
process.stdout.setDefaultEncoding('utf8'); process.stdout.setDefaultEncoding('utf8');
} }
@@ -54,13 +54,13 @@ export class RemoteClient {
} }
// For K8s, also write to stdout so kubectl logs can capture it // For K8s, also write to stdout so kubectl logs can capture it
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
// Write to stdout so kubectl logs can capture it - ensure newline is included // Write to stdout so kubectl logs can capture it - ensure newline is included
// Stdout flushes automatically on newline, so no explicit flush needed // Stdout flushes automatically on newline, so no explicit flush needed
process.stdout.write(`${element}\n`); process.stdout.write(`${element}\n`);
} }
CloudRunnerLogger.log(element); OrchestratorLogger.log(element);
} }
}); });
@@ -70,28 +70,28 @@ export class RemoteClient {
fs.appendFileSync(logFile, `${lingeringLine}\n`); fs.appendFileSync(logFile, `${lingeringLine}\n`);
// For K8s, also write to stdout so kubectl logs can capture it // For K8s, also write to stdout so kubectl logs can capture it
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
// Stdout flushes automatically on newline // Stdout flushes automatically on newline
process.stdout.write(`${lingeringLine}\n`); process.stdout.write(`${lingeringLine}\n`);
} }
} }
CloudRunnerLogger.log(lingeringLine); OrchestratorLogger.log(lingeringLine);
}); });
} }
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`) @CliFunction(`remote-cli-post-build`, `runs a orchestrator build`)
public static async remoteClientPostBuild(): Promise<string> { public static async remoteClientPostBuild(): Promise<string> {
try { try {
RemoteClientLogger.log(`Running POST build tasks`); RemoteClientLogger.log(`Running POST build tasks`);
// Ensure cache key is present in logs for assertions // Ensure cache key is present in logs for assertions
RemoteClientLogger.log(`CACHE_KEY=${CloudRunner.buildParameters.cacheKey}`); RemoteClientLogger.log(`CACHE_KEY=${Orchestrator.buildParameters.cacheKey}`);
CloudRunnerLogger.log(`${CloudRunner.buildParameters.cacheKey}`); OrchestratorLogger.log(`${Orchestrator.buildParameters.cacheKey}`);
// Guard: only push Library cache if the folder exists and has contents // Guard: only push Library cache if the folder exists and has contents
try { try {
const libraryFolderHost = CloudRunnerFolders.libraryFolderAbsolute; const libraryFolderHost = OrchestratorFolders.libraryFolderAbsolute;
if (fs.existsSync(libraryFolderHost)) { if (fs.existsSync(libraryFolderHost)) {
let libraryEntries: string[] = []; let libraryEntries: string[] = [];
try { try {
@@ -101,9 +101,9 @@ export class RemoteClient {
} }
if (libraryEntries.length > 0) { if (libraryEntries.length > 0) {
await Caching.PushToCache( await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`), OrchestratorFolders.ToLinuxFolder(`${OrchestratorFolders.cacheFolderForCacheKeyFull}/Library`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryFolderAbsolute),
`lib-${CloudRunner.buildParameters.buildGuid}`, `lib-${Orchestrator.buildParameters.buildGuid}`,
); );
} else { } else {
RemoteClientLogger.log(`Skipping Library cache push (folder is empty)`); RemoteClientLogger.log(`Skipping Library cache push (folder is empty)`);
@@ -117,7 +117,7 @@ export class RemoteClient {
// Guard: only push Build cache if the folder exists and has contents // Guard: only push Build cache if the folder exists and has contents
try { try {
const buildFolderHost = CloudRunnerFolders.projectBuildFolderAbsolute; const buildFolderHost = OrchestratorFolders.projectBuildFolderAbsolute;
if (fs.existsSync(buildFolderHost)) { if (fs.existsSync(buildFolderHost)) {
let buildEntries: string[] = []; let buildEntries: string[] = [];
try { try {
@@ -127,9 +127,9 @@ export class RemoteClient {
} }
if (buildEntries.length > 0) { if (buildEntries.length > 0) {
await Caching.PushToCache( await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`), OrchestratorFolders.ToLinuxFolder(`${OrchestratorFolders.cacheFolderForCacheKeyFull}/build`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.projectBuildFolderAbsolute),
`build-${CloudRunner.buildParameters.buildGuid}`, `build-${Orchestrator.buildParameters.buildGuid}`,
); );
} else { } else {
RemoteClientLogger.log(`Skipping Build cache push (folder is empty)`); RemoteClientLogger.log(`Skipping Build cache push (folder is empty)`);
@@ -141,15 +141,15 @@ export class RemoteClient {
RemoteClientLogger.logWarning(`Build cache push skipped with error: ${error.message}`); RemoteClientLogger.logWarning(`Build cache push skipped with error: ${error.message}`);
} }
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) { if (!BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)) {
const uniqueJobFolderLinux = CloudRunnerFolders.ToLinuxFolder( const uniqueJobFolderLinux = OrchestratorFolders.ToLinuxFolder(
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute,
); );
if ( if (
fs.existsSync(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute) || fs.existsSync(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute) ||
fs.existsSync(uniqueJobFolderLinux) fs.existsSync(uniqueJobFolderLinux)
) { ) {
await CloudRunnerSystem.Run(`rm -r ${uniqueJobFolderLinux} || true`); await OrchestratorSystem.Run(`rm -r ${uniqueJobFolderLinux} || true`);
} else { } else {
RemoteClientLogger.log(`Skipping cleanup; unique job folder missing`); RemoteClientLogger.log(`Skipping cleanup; unique job folder missing`);
} }
@@ -162,7 +162,7 @@ export class RemoteClient {
} catch (error: any) { } catch (error: any) {
// Log error but don't fail - post-build tasks are best-effort // Log error but don't fail - post-build tasks are best-effort
RemoteClientLogger.logWarning(`Post-build task error: ${error.message}`); RemoteClientLogger.logWarning(`Post-build task error: ${error.message}`);
CloudRunnerLogger.log(`Post-build task error: ${error.message}`); OrchestratorLogger.log(`Post-build task error: ${error.message}`);
} }
// Ensure success marker is always present in logs for tests, even if post-build tasks failed // Ensure success marker is always present in logs for tests, even if post-build tasks failed
@@ -174,7 +174,7 @@ export class RemoteClient {
// Write directly to log file first to ensure it's captured even if pipe fails // Write directly to log file first to ensure it's captured even if pipe fails
// This is critical for all providers, especially K8s where timing matters // This is critical for all providers, especially K8s where timing matters
try { try {
const logFilePath = CloudRunner.isCloudRunnerEnvironment const logFilePath = Orchestrator.isOrchestratorEnvironment
? `/home/job-log.txt` ? `/home/job-log.txt`
: path.join(process.cwd(), 'temp', 'job-log.txt'); : path.join(process.cwd(), 'temp', 'job-log.txt');
if (fs.existsSync(path.dirname(logFilePath))) { if (fs.existsSync(path.dirname(logFilePath))) {
@@ -191,7 +191,7 @@ export class RemoteClient {
// For K8s, also write to stderr as a backup since kubectl logs reads from both stdout and stderr // For K8s, also write to stderr as a backup since kubectl logs reads from both stdout and stderr
// This ensures the message is captured even if stdout pipe has issues // This ensures the message is captured even if stdout pipe has issues
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
process.stderr.write(`${successMessage}\n`, 'utf8'); process.stderr.write(`${successMessage}\n`, 'utf8');
} }
@@ -202,19 +202,19 @@ export class RemoteClient {
await new Promise((resolve) => setTimeout(resolve, 100)); await new Promise((resolve) => setTimeout(resolve, 100));
} }
// Also log via CloudRunnerLogger and RemoteClientLogger for GitHub Actions and log file // Also log via OrchestratorLogger and RemoteClientLogger for GitHub Actions and log file
// This ensures the message appears in log files for providers that read from log files // This ensures the message appears in log files for providers that read from log files
// RemoteClientLogger.log writes directly to the log file, which is important for providers // RemoteClientLogger.log writes directly to the log file, which is important for providers
// that read from the log file rather than stdout // that read from the log file rather than stdout
RemoteClientLogger.log(successMessage); RemoteClientLogger.log(successMessage);
CloudRunnerLogger.log(successMessage); OrchestratorLogger.log(successMessage);
await ResourceTracking.logDiskUsageSnapshot('remote-cli-post-build (end)'); await ResourceTracking.logDiskUsageSnapshot('remote-cli-post-build (end)');
return new Promise((result) => result(``)); return new Promise((result) => result(``));
} }
static async runCustomHookFiles(hookLifecycle: string) { static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`); RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`); const gameCiCustomHooksPath = path.join(OrchestratorFolders.repoPathAbsolute, `game-ci`, `hooks`);
try { try {
const files = fs.readdirSync(gameCiCustomHooksPath); const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) { for (const file of files) {
@@ -222,7 +222,7 @@ export class RemoteClient {
const fileContentsObject = YAML.parse(fileContents.toString()); const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) { if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`); RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands); await OrchestratorSystem.Run(fileContentsObject.commands);
} }
} }
} catch (error) { } catch (error) {
@@ -230,124 +230,124 @@ export class RemoteClient {
} }
} }
public static async bootstrapRepository() { public static async bootstrapRepository() {
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`, `mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
); );
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`, `mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.cacheFolderForCacheKeyFull)}`,
); );
await RemoteClient.cloneRepoWithoutLFSFiles(); await RemoteClient.cloneRepoWithoutLFSFiles();
await RemoteClient.sizeOfFolder( await RemoteClient.sizeOfFolder(
'repo before lfs cache pull', 'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.repoPathAbsolute),
); );
const lfsHashes = await LfsHashing.createLFSHashFiles(); const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) { if (fs.existsSync(OrchestratorFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`); RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
} }
await Caching.PullFromCache( await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`, `${lfsHashes.lfsGuidSum}`,
); );
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute); await RemoteClient.sizeOfFolder('repo after lfs cache pull', OrchestratorFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS(); await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute); await RemoteClient.sizeOfFolder('repo before lfs git pull', OrchestratorFolders.repoPathAbsolute);
await Caching.PushToCache( await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`, `${lfsHashes.lfsGuidSum}`,
); );
await Caching.PullFromCache( await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute), OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.libraryFolderAbsolute),
); );
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute); await RemoteClient.sizeOfFolder('repo after library cache pull', OrchestratorFolders.repoPathAbsolute);
await Caching.handleCachePurging(); await Caching.handleCachePurging();
} }
private static async sizeOfFolder(message: string, folder: string) { private static async sizeOfFolder(message: string, folder: string) {
if (CloudRunner.buildParameters.cloudRunnerDebug) { if (Orchestrator.buildParameters.orchestratorDebug) {
CloudRunnerLogger.log(`Size of ${message}`); OrchestratorLogger.log(`Size of ${message}`);
await CloudRunnerSystem.Run(`du -sh ${folder}`); await OrchestratorSystem.Run(`du -sh ${folder}`);
} }
} }
private static async cloneRepoWithoutLFSFiles() { private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); process.chdir(`${OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute}`);
if ( if (
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) && fs.existsSync(OrchestratorFolders.repoPathAbsolute) &&
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)) !fs.existsSync(path.join(OrchestratorFolders.repoPathAbsolute, `.git`))
) { ) {
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`); await OrchestratorSystem.Run(`rm -r ${OrchestratorFolders.repoPathAbsolute}`);
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`); OrchestratorLogger.log(`${OrchestratorFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
} }
if ( if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) && BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters) &&
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)) fs.existsSync(path.join(OrchestratorFolders.repoPathAbsolute, `.git`))
) { ) {
process.chdir(CloudRunnerFolders.repoPathAbsolute); process.chdir(OrchestratorFolders.repoPathAbsolute);
RemoteClientLogger.log( RemoteClientLogger.log(
`${ `${
CloudRunnerFolders.repoPathAbsolute OrchestratorFolders.repoPathAbsolute
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode( } repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
CloudRunner.buildParameters, Orchestrator.buildParameters,
)}`, )}`,
); );
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`); await OrchestratorSystem.Run(`git fetch && git reset --hard ${Orchestrator.buildParameters.gitSha}`);
return; return;
} }
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`); RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`); await OrchestratorSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`); RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`); await OrchestratorSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`); await OrchestratorSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
try { try {
const depthArgument = CloudRunnerOptions.cloneDepth !== '0' ? `--depth ${CloudRunnerOptions.cloneDepth}` : ''; const depthArgument = OrchestratorOptions.cloneDepth !== '0' ? `--depth ${OrchestratorOptions.cloneDepth}` : '';
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`git clone ${depthArgument} ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename( `git clone ${depthArgument} ${OrchestratorFolders.targetBuildRepoUrl} ${path.basename(
CloudRunnerFolders.repoPathAbsolute, OrchestratorFolders.repoPathAbsolute,
)}`.trim(), )}`.trim(),
); );
} catch (error: any) { } catch (error: any) {
throw error; throw error;
} }
process.chdir(CloudRunnerFolders.repoPathAbsolute); process.chdir(OrchestratorFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`); await OrchestratorSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists'); assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`); RemoteClientLogger.log(`${Orchestrator.buildParameters.branch}`);
// Ensure refs exist (tags and PR refs) // Ensure refs exist (tags and PR refs)
await CloudRunnerSystem.Run(`git fetch --all --tags || true`); await OrchestratorSystem.Run(`git fetch --all --tags || true`);
const branchForPrFetch = CloudRunner.buildParameters.branch || ''; const branchForPrFetch = Orchestrator.buildParameters.branch || '';
if (branchForPrFetch.startsWith('pull/')) { if (branchForPrFetch.startsWith('pull/')) {
// Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731) // Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731)
const prNumber = branchForPrFetch.split('/')[1]; const prNumber = branchForPrFetch.split('/')[1];
if (prNumber) { if (prNumber) {
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`, `git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`,
); );
} }
} }
const targetSha = CloudRunner.buildParameters.gitSha; const targetSha = Orchestrator.buildParameters.gitSha;
const targetBranch = CloudRunner.buildParameters.branch; const targetBranch = Orchestrator.buildParameters.branch;
if (targetSha) { if (targetSha) {
try { try {
await CloudRunnerSystem.Run(`git checkout ${targetSha}`); await OrchestratorSystem.Run(`git checkout ${targetSha}`);
} catch { } catch {
try { try {
await CloudRunnerSystem.Run(`git fetch origin ${targetSha} || true`); await OrchestratorSystem.Run(`git fetch origin ${targetSha} || true`);
await CloudRunnerSystem.Run(`git checkout ${targetSha}`); await OrchestratorSystem.Run(`git checkout ${targetSha}`);
} catch (error) { } catch (error) {
RemoteClientLogger.logWarning(`Falling back to branch checkout; SHA not found: ${targetSha}`); RemoteClientLogger.logWarning(`Falling back to branch checkout; SHA not found: ${targetSha}`);
try { try {
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`); await OrchestratorSystem.Run(`git checkout ${targetBranch}`);
} catch { } catch {
if ((targetBranch || '').startsWith('pull/')) { if ((targetBranch || '').startsWith('pull/')) {
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`); await OrchestratorSystem.Run(`git checkout origin/${targetBranch}`);
} else { } else {
throw error; throw error;
} }
@@ -356,10 +356,10 @@ export class RemoteClient {
} }
} else { } else {
try { try {
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`); await OrchestratorSystem.Run(`git checkout ${targetBranch}`);
} catch (_error) { } catch (_error) {
if ((targetBranch || '').startsWith('pull/')) { if ((targetBranch || '').startsWith('pull/')) {
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`); await OrchestratorSystem.Run(`git checkout origin/${targetBranch}`);
} else { } else {
throw _error; throw _error;
} }
@@ -368,27 +368,27 @@ export class RemoteClient {
} }
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching'); assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`); RemoteClientLogger.log(`Checked out ${Orchestrator.buildParameters.branch}`);
} }
static async replaceLargePackageReferencesWithSharedReferences() { static async replaceLargePackageReferencesWithSharedReferences() {
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`); OrchestratorLogger.log(`Use Shared Pkgs ${Orchestrator.buildParameters.useLargePackages}`);
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``); GitHub.updateGitHubCheck(`Use Shared Pkgs ${Orchestrator.buildParameters.useLargePackages}`, ``);
if (CloudRunner.buildParameters.useLargePackages) { if (Orchestrator.buildParameters.useLargePackages) {
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`); const filePath = path.join(OrchestratorFolders.projectPathAbsolute, `Packages/manifest.json`);
let manifest = fs.readFileSync(filePath, 'utf8'); let manifest = fs.readFileSync(filePath, 'utf8');
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent'); manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
fs.writeFileSync(filePath, manifest); fs.writeFileSync(filePath, manifest);
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`); OrchestratorLogger.log(`Package Manifest \n ${manifest}`);
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``); GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
} }
} }
private static async pullLatestLFS() { private static async pullLatestLFS() {
process.chdir(CloudRunnerFolders.repoPathAbsolute); process.chdir(OrchestratorFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`); await OrchestratorSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`); await OrchestratorSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
if (CloudRunner.buildParameters.skipLfs) { if (Orchestrator.buildParameters.skipLfs) {
RemoteClientLogger.log(`Skipping LFS pull (skipLfs=true)`); RemoteClientLogger.log(`Skipping LFS pull (skipLfs=true)`);
return; return;
@@ -396,8 +396,8 @@ export class RemoteClient {
// Best effort: try plain pull first (works for public repos or pre-configured auth) // Best effort: try plain pull first (works for public repos or pre-configured auth)
try { try {
await CloudRunnerSystem.Run(`git lfs pull`, true); await OrchestratorSystem.Run(`git lfs pull`, true);
await CloudRunnerSystem.Run(`git lfs checkout || true`, true); await OrchestratorSystem.Run(`git lfs checkout || true`, true);
RemoteClientLogger.log(`Pulled LFS files without explicit token configuration`); RemoteClientLogger.log(`Pulled LFS files without explicit token configuration`);
return; return;
@@ -411,14 +411,14 @@ export class RemoteClient {
const gitPrivateToken = process.env.GIT_PRIVATE_TOKEN; const gitPrivateToken = process.env.GIT_PRIVATE_TOKEN;
if (gitPrivateToken) { if (gitPrivateToken) {
RemoteClientLogger.log(`Attempting to pull LFS files with GIT_PRIVATE_TOKEN...`); RemoteClientLogger.log(`Attempting to pull LFS files with GIT_PRIVATE_TOKEN...`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`git config --global url."https://${gitPrivateToken}@github.com/".insteadOf "https://github.com/"`, `git config --global url."https://${gitPrivateToken}@github.com/".insteadOf "https://github.com/"`,
); );
await CloudRunnerSystem.Run(`git lfs pull`, true); await OrchestratorSystem.Run(`git lfs pull`, true);
await CloudRunnerSystem.Run(`git lfs checkout || true`, true); await OrchestratorSystem.Run(`git lfs checkout || true`, true);
RemoteClientLogger.log(`Successfully pulled LFS files with GIT_PRIVATE_TOKEN`); RemoteClientLogger.log(`Successfully pulled LFS files with GIT_PRIVATE_TOKEN`);
return; return;
@@ -432,14 +432,14 @@ export class RemoteClient {
const githubToken = process.env.GITHUB_TOKEN; const githubToken = process.env.GITHUB_TOKEN;
if (githubToken) { if (githubToken) {
RemoteClientLogger.log(`Attempting to pull LFS files with GITHUB_TOKEN fallback...`); RemoteClientLogger.log(`Attempting to pull LFS files with GITHUB_TOKEN fallback...`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`); await OrchestratorSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`git config --global url."https://${githubToken}@github.com/".insteadOf "https://github.com/"`, `git config --global url."https://${githubToken}@github.com/".insteadOf "https://github.com/"`,
); );
await CloudRunnerSystem.Run(`git lfs pull`, true); await OrchestratorSystem.Run(`git lfs pull`, true);
await CloudRunnerSystem.Run(`git lfs checkout || true`, true); await OrchestratorSystem.Run(`git lfs checkout || true`, true);
RemoteClientLogger.log(`Successfully pulled LFS files with GITHUB_TOKEN`); RemoteClientLogger.log(`Successfully pulled LFS files with GITHUB_TOKEN`);
return; return;
@@ -453,43 +453,43 @@ export class RemoteClient {
} }
static async handleRetainedWorkspace() { static async handleRetainedWorkspace() {
RemoteClientLogger.log( RemoteClientLogger.log(
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`, `Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)}`,
); );
// Log cache key explicitly to aid debugging and assertions // Log cache key explicitly to aid debugging and assertions
CloudRunnerLogger.log(`Cache Key: ${CloudRunner.buildParameters.cacheKey}`); OrchestratorLogger.log(`Cache Key: ${Orchestrator.buildParameters.cacheKey}`);
if ( if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) && BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) && fs.existsSync(OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))) fs.existsSync(OrchestratorFolders.ToLinuxFolder(path.join(OrchestratorFolders.repoPathAbsolute, `.git`)))
) { ) {
CloudRunnerLogger.log(`Retained Workspace Already Exists!`); OrchestratorLogger.log(`Retained Workspace Already Exists!`);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)); process.chdir(OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.repoPathAbsolute));
await CloudRunnerSystem.Run(`git fetch --all --tags || true`); await OrchestratorSystem.Run(`git fetch --all --tags || true`);
const retainedBranchForPrFetch = CloudRunner.buildParameters.branch || ''; const retainedBranchForPrFetch = Orchestrator.buildParameters.branch || '';
if (retainedBranchForPrFetch.startsWith('pull/')) { if (retainedBranchForPrFetch.startsWith('pull/')) {
// Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731) // Extract PR number and fetch only that specific ref (e.g., pull/731/merge -> 731)
const prNumber = retainedBranchForPrFetch.split('/')[1]; const prNumber = retainedBranchForPrFetch.split('/')[1];
if (prNumber) { if (prNumber) {
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`, `git fetch origin +refs/pull/${prNumber}/merge:refs/remotes/origin/pull/${prNumber}/merge +refs/pull/${prNumber}/head:refs/remotes/origin/pull/${prNumber}/head || true`,
); );
} }
} }
await CloudRunnerSystem.Run(`git lfs pull`); await OrchestratorSystem.Run(`git lfs pull`);
await CloudRunnerSystem.Run(`git lfs checkout || true`); await OrchestratorSystem.Run(`git lfs checkout || true`);
const sha = CloudRunner.buildParameters.gitSha; const sha = Orchestrator.buildParameters.gitSha;
const branch = CloudRunner.buildParameters.branch; const branch = Orchestrator.buildParameters.branch;
try { try {
await CloudRunnerSystem.Run(`git reset --hard "${sha}"`); await OrchestratorSystem.Run(`git reset --hard "${sha}"`);
await CloudRunnerSystem.Run(`git checkout ${sha}`); await OrchestratorSystem.Run(`git checkout ${sha}`);
} catch { } catch {
RemoteClientLogger.logWarning(`Retained workspace: SHA not found, falling back to branch ${branch}`); RemoteClientLogger.logWarning(`Retained workspace: SHA not found, falling back to branch ${branch}`);
try { try {
await CloudRunnerSystem.Run(`git checkout ${branch}`); await OrchestratorSystem.Run(`git checkout ${branch}`);
} catch (error) { } catch (error) {
if ((branch || '').startsWith('pull/')) { if ((branch || '').startsWith('pull/')) {
await CloudRunnerSystem.Run(`git checkout origin/${branch}`); await OrchestratorSystem.Run(`git checkout origin/${branch}`);
} else { } else {
throw error; throw error;
} }

View File

@@ -1,8 +1,8 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import CloudRunnerOptions from '../options/cloud-runner-options'; import OrchestratorOptions from '../options/orchestrator-options';
export class RemoteClientLogger { export class RemoteClientLogger {
private static get LogFilePath() { private static get LogFilePath() {
@@ -17,23 +17,23 @@ export class RemoteClientLogger {
public static log(message: string) { public static log(message: string) {
const finalMessage = `[Client] ${message}`; const finalMessage = `[Client] ${message}`;
this.appendToFile(finalMessage); this.appendToFile(finalMessage);
CloudRunnerLogger.log(finalMessage); OrchestratorLogger.log(finalMessage);
} }
public static logCliError(message: string) { public static logCliError(message: string) {
CloudRunnerLogger.log(`[Client][Error] ${message}`); OrchestratorLogger.log(`[Client][Error] ${message}`);
} }
public static logCliDiagnostic(message: string) { public static logCliDiagnostic(message: string) {
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`); OrchestratorLogger.log(`[Client][Diagnostic] ${message}`);
} }
public static logWarning(message: string) { public static logWarning(message: string) {
CloudRunnerLogger.logWarning(message); OrchestratorLogger.logWarning(message);
} }
public static appendToFile(message: string) { public static appendToFile(message: string) {
if (CloudRunner.isCloudRunnerEnvironment) { if (Orchestrator.isOrchestratorEnvironment) {
// Ensure the directory exists before writing // Ensure the directory exists before writing
const logDirectory = path.dirname(RemoteClientLogger.LogFilePath); const logDirectory = path.dirname(RemoteClientLogger.LogFilePath);
if (!fs.existsSync(logDirectory)) { if (!fs.existsSync(logDirectory)) {
@@ -45,7 +45,7 @@ export class RemoteClientLogger {
} }
public static async handleLogManagementPostJob() { public static async handleLogManagementPostJob() {
if (CloudRunnerOptions.providerStrategy !== 'k8s') { if (OrchestratorOptions.providerStrategy !== 'k8s') {
return; return;
} }
const collectedLogsMessage = `Collected Logs`; const collectedLogsMessage = `Collected Logs`;
@@ -57,7 +57,7 @@ export class RemoteClientLogger {
// For K8s, write to stdout/stderr so kubectl logs can capture it // For K8s, write to stdout/stderr so kubectl logs can capture it
// This is critical because kubectl logs reads from stdout/stderr, not from GitHub Actions logs // This is critical because kubectl logs reads from stdout/stderr, not from GitHub Actions logs
// Write multiple times to increase chance of capture if kubectl is having issues // Write multiple times to increase chance of capture if kubectl is having issues
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
// Write to stdout multiple times to increase chance of capture // Write to stdout multiple times to increase chance of capture
for (let index = 0; index < 3; index++) { for (let index = 0; index < 3; index++) {
process.stdout.write(`${collectedLogsMessage}\n`, 'utf8'); process.stdout.write(`${collectedLogsMessage}\n`, 'utf8');
@@ -70,33 +70,33 @@ export class RemoteClientLogger {
} }
} }
// Also log via CloudRunnerLogger for GitHub Actions // Also log via OrchestratorLogger for GitHub Actions
CloudRunnerLogger.log(collectedLogsMessage); OrchestratorLogger.log(collectedLogsMessage);
// check for log file not existing // check for log file not existing
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) { if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
const logFileMissingMessage = `Log file does not exist`; const logFileMissingMessage = `Log file does not exist`;
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
process.stdout.write(`${logFileMissingMessage}\n`, 'utf8'); process.stdout.write(`${logFileMissingMessage}\n`, 'utf8');
} }
CloudRunnerLogger.log(logFileMissingMessage); OrchestratorLogger.log(logFileMissingMessage);
// check if CloudRunner.isCloudRunnerEnvironment is true, log // check if Orchestrator.isOrchestratorEnvironment is true, log
if (!CloudRunner.isCloudRunnerEnvironment) { if (!Orchestrator.isOrchestratorEnvironment) {
const notCloudEnvironmentMessage = `Cloud Runner is not running in a cloud environment, not collecting logs`; const notCloudEnvironmentMessage = `Orchestrator is not running in a cloud environment, not collecting logs`;
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
process.stdout.write(`${notCloudEnvironmentMessage}\n`, 'utf8'); process.stdout.write(`${notCloudEnvironmentMessage}\n`, 'utf8');
} }
CloudRunnerLogger.log(notCloudEnvironmentMessage); OrchestratorLogger.log(notCloudEnvironmentMessage);
} }
return; return;
} }
const logFileExistsMessage = `Log file exist`; const logFileExistsMessage = `Log file exist`;
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
process.stdout.write(`${logFileExistsMessage}\n`, 'utf8'); process.stdout.write(`${logFileExistsMessage}\n`, 'utf8');
} }
CloudRunnerLogger.log(logFileExistsMessage); OrchestratorLogger.log(logFileExistsMessage);
await new Promise((resolve) => setTimeout(resolve, 1)); await new Promise((resolve) => setTimeout(resolve, 1));
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString(); // let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
@@ -104,11 +104,11 @@ export class RemoteClientLogger {
// hashedLogs = md5(hashedLogs); // hashedLogs = md5(hashedLogs);
// //
// for (let index = 0; index < 3; index++) { // for (let index = 0; index < 3; index++) {
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`); // OrchestratorLogger.log(`LOGHASH: ${hashedLogs}`);
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString(); // const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`); // OrchestratorLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
// CloudRunnerLogger.log( // OrchestratorLogger.log(
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`, // `Game CI's "Orchestrator System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
// ); // );
// //
// // wait for 15 seconds to allow the log to be sent // // wait for 15 seconds to allow the log to be sent

View File

@@ -1,7 +1,7 @@
import GitHub from '../../../github'; import GitHub from '../../../github';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { CloudRunnerStatics } from '../../options/cloud-runner-statics'; import { OrchestratorStatics } from '../../options/orchestrator-statics';
import CloudRunnerLogger from './cloud-runner-logger'; import OrchestratorLogger from './orchestrator-logger';
import * as core from '@actions/core'; import * as core from '@actions/core';
export class FollowLogStreamService { export class FollowLogStreamService {
@@ -11,8 +11,8 @@ export class FollowLogStreamService {
static errors = ``; static errors = ``;
public static DidReceiveEndOfTransmission = false; public static DidReceiveEndOfTransmission = false;
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) { public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) { if (message.includes(`---${Orchestrator.buildParameters.logId}`)) {
CloudRunnerLogger.log('End of log transmission received'); OrchestratorLogger.log('End of log transmission received');
FollowLogStreamService.DidReceiveEndOfTransmission = true; FollowLogStreamService.DidReceiveEndOfTransmission = true;
shouldReadLogs = false; shouldReadLogs = false;
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) { } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
@@ -50,7 +50,7 @@ export class FollowLogStreamService {
// Always append log lines to output so tests can assert on BuildResults // Always append log lines to output so tests can assert on BuildResults
output += `${message}\n`; output += `${message}\n`;
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`); OrchestratorLogger.log(`[${OrchestratorStatics.logPrefix}] ${message}`);
return { shouldReadLogs, shouldCleanup, output }; return { shouldReadLogs, shouldCleanup, output };
} }

View File

@@ -1,6 +1,6 @@
import * as core from '@actions/core'; import * as core from '@actions/core';
class CloudRunnerLogger { class OrchestratorLogger {
private static timestamp: number; private static timestamp: number;
private static globalTimestamp: number; private static globalTimestamp: number;
@@ -44,4 +44,4 @@ class CloudRunnerLogger {
return Date.now(); return Date.now();
} }
} }
export default CloudRunnerLogger; export default OrchestratorLogger;

View File

@@ -1,6 +1,6 @@
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
class CloudRunnerResult { class OrchestratorResult {
public BuildParameters: BuildParameters; public BuildParameters: BuildParameters;
public BuildResults: string; public BuildResults: string;
public BuildSucceeded: boolean; public BuildSucceeded: boolean;
@@ -21,4 +21,4 @@ class CloudRunnerResult {
this.LibraryCacheUsed = libraryCacheUsed; this.LibraryCacheUsed = libraryCacheUsed;
} }
} }
export default CloudRunnerResult; export default OrchestratorResult;

View File

@@ -1,9 +1,9 @@
import { exec } from 'child_process'; import { exec } from 'child_process';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger'; import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
export class CloudRunnerSystem { export class OrchestratorSystem {
public static async RunAndReadLines(command: string): Promise<string[]> { public static async RunAndReadLines(command: string): Promise<string[]> {
const result = await CloudRunnerSystem.Run(command, false, true); const result = await OrchestratorSystem.Run(command, false, true);
return result return result
.split(`\n`) .split(`\n`)

View File

@@ -1,14 +1,14 @@
import CloudRunnerLogger from './cloud-runner-logger'; import OrchestratorLogger from './orchestrator-logger';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { CloudRunnerSystem } from './cloud-runner-system'; import { OrchestratorSystem } from './orchestrator-system';
class ResourceTracking { class ResourceTracking {
static isEnabled(): boolean { static isEnabled(): boolean {
return ( return (
CloudRunnerOptions.resourceTracking || OrchestratorOptions.resourceTracking ||
CloudRunnerOptions.cloudRunnerDebug || OrchestratorOptions.orchestratorDebug ||
process.env['cloudRunnerTests'] === 'true' process.env['orchestratorTests'] === 'true'
); );
} }
@@ -17,7 +17,7 @@ class ResourceTracking {
return; return;
} }
const buildParameters = CloudRunner.buildParameters; const buildParameters = Orchestrator.buildParameters;
const allocations = { const allocations = {
providerStrategy: buildParameters.providerStrategy, providerStrategy: buildParameters.providerStrategy,
containerCpu: buildParameters.containerCpu, containerCpu: buildParameters.containerCpu,
@@ -35,11 +35,11 @@ class ResourceTracking {
maxRetainedWorkspaces: buildParameters.maxRetainedWorkspaces, maxRetainedWorkspaces: buildParameters.maxRetainedWorkspaces,
useCompressionStrategy: buildParameters.useCompressionStrategy, useCompressionStrategy: buildParameters.useCompressionStrategy,
useLargePackages: buildParameters.useLargePackages, useLargePackages: buildParameters.useLargePackages,
ephemeralStorageRequest: process.env['cloudRunnerTests'] === 'true' ? 'not set' : '2Gi', ephemeralStorageRequest: process.env['orchestratorTests'] === 'true' ? 'not set' : '2Gi',
}; };
CloudRunnerLogger.log(`[ResourceTracking] Allocation summary (${context}):`); OrchestratorLogger.log(`[ResourceTracking] Allocation summary (${context}):`);
CloudRunnerLogger.log(JSON.stringify(allocations, undefined, 2)); OrchestratorLogger.log(JSON.stringify(allocations, undefined, 2));
} }
static async logDiskUsageSnapshot(context: string) { static async logDiskUsageSnapshot(context: string) {
@@ -47,10 +47,10 @@ class ResourceTracking {
return; return;
} }
CloudRunnerLogger.log(`[ResourceTracking] Disk usage snapshot (${context})`); OrchestratorLogger.log(`[ResourceTracking] Disk usage snapshot (${context})`);
await ResourceTracking.runAndLog('df -h', 'df -h'); await ResourceTracking.runAndLog('df -h', 'df -h');
await ResourceTracking.runAndLog('du -sh .', 'du -sh .'); await ResourceTracking.runAndLog('du -sh .', 'du -sh .');
await ResourceTracking.runAndLog('du -sh ./cloud-runner-cache', 'du -sh ./cloud-runner-cache'); await ResourceTracking.runAndLog('du -sh ./orchestrator-cache', 'du -sh ./orchestrator-cache');
await ResourceTracking.runAndLog('du -sh ./temp', 'du -sh ./temp'); await ResourceTracking.runAndLog('du -sh ./temp', 'du -sh ./temp');
await ResourceTracking.runAndLog('du -sh ./logs', 'du -sh ./logs'); await ResourceTracking.runAndLog('du -sh ./logs', 'du -sh ./logs');
} }
@@ -61,7 +61,7 @@ class ResourceTracking {
} }
const nodes = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0']; const nodes = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
CloudRunnerLogger.log(`[ResourceTracking] K3d node disk usage (${context})`); OrchestratorLogger.log(`[ResourceTracking] K3d node disk usage (${context})`);
for (const node of nodes) { for (const node of nodes) {
await ResourceTracking.runAndLog( await ResourceTracking.runAndLog(
`k3d node ${node}`, `k3d node ${node}`,
@@ -72,11 +72,11 @@ class ResourceTracking {
private static async runAndLog(label: string, command: string) { private static async runAndLog(label: string, command: string) {
try { try {
const output = await CloudRunnerSystem.Run(command, true, true); const output = await OrchestratorSystem.Run(command, true, true);
const trimmed = output.trim(); const trimmed = output.trim();
CloudRunnerLogger.log(`[ResourceTracking] ${label}:\n${trimmed || 'no output'}`); OrchestratorLogger.log(`[ResourceTracking] ${label}:\n${trimmed || 'no output'}`);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`[ResourceTracking] ${label} failed: ${error?.message || error}`); OrchestratorLogger.log(`[ResourceTracking] ${label} failed: ${error?.message || error}`);
} }
} }
} }

View File

@@ -1,6 +1,6 @@
import CloudRunnerLogger from './cloud-runner-logger'; import OrchestratorLogger from './orchestrator-logger';
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import Input from '../../../input'; import Input from '../../../input';
import { import {
CreateBucketCommand, CreateBucketCommand,
@@ -25,7 +25,7 @@ export class SharedWorkspaceLocking {
return SharedWorkspaceLocking._s3; return SharedWorkspaceLocking._s3;
} }
private static get useRclone() { private static get useRclone() {
return CloudRunner.buildParameters.storageProvider === 'rclone'; return Orchestrator.buildParameters.storageProvider === 'rclone';
} }
private static async rclone(command: string): Promise<string> { private static async rclone(command: string): Promise<string> {
const { stdout } = await exec(`rclone ${command}`); const { stdout } = await exec(`rclone ${command}`);
@@ -34,8 +34,8 @@ export class SharedWorkspaceLocking {
} }
private static get bucket() { private static get bucket() {
return SharedWorkspaceLocking.useRclone return SharedWorkspaceLocking.useRclone
? CloudRunner.buildParameters.rcloneRemote ? Orchestrator.buildParameters.rcloneRemote
: CloudRunner.buildParameters.awsStackName; : Orchestrator.buildParameters.awsStackName;
} }
public static get workspaceBucketRoot() { public static get workspaceBucketRoot() {
return SharedWorkspaceLocking.useRclone return SharedWorkspaceLocking.useRclone
@@ -131,7 +131,7 @@ export class SharedWorkspaceLocking {
} }
public static NewWorkspaceName() { public static NewWorkspaceName() {
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`; return `${Orchestrator.retainedWorkspacePrefix}-${Orchestrator.buildParameters.buildGuid}`;
} }
public static async GetAllLocksForWorkspace( public static async GetAllLocksForWorkspace(
workspace: string, workspace: string,
@@ -156,10 +156,10 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) { if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext); const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`); OrchestratorLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
for (const element of workspaces) { for (const element of workspaces) {
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext); const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log( OrchestratorLogger.log(
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`, `run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
); );
@@ -171,12 +171,12 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) { if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
workspace = SharedWorkspaceLocking.NewWorkspaceName(); workspace = SharedWorkspaceLocking.NewWorkspaceName();
CloudRunner.lockedWorkspace = workspace; Orchestrator.lockedWorkspace = workspace;
} }
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext); const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext); const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
CloudRunnerLogger.log( OrchestratorLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`, `run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
); );
@@ -204,7 +204,7 @@ export class SharedWorkspaceLocking {
.sort((x) => x.timestamp); .sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId)); const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0; const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
CloudRunnerLogger.log( OrchestratorLogger.log(
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${ `Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
lockMatches.length lockMatches.length
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`, } Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
@@ -219,7 +219,7 @@ export class SharedWorkspaceLocking {
for (const element of workspaces) { for (const element of workspaces) {
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext); const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext); const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`); OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
if (!isLocked && isBelowMax) { if (!isLocked && isBelowMax) {
result.push(element); result.push(element);
} }
@@ -309,9 +309,9 @@ export class SharedWorkspaceLocking {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext); const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`All workspaces ${workspaces}`); OrchestratorLogger.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) { if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`); OrchestratorLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext); await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false; return false;
@@ -340,7 +340,7 @@ export class SharedWorkspaceLocking {
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext); const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
if (hasLock) { if (hasLock) {
CloudRunner.lockedWorkspace = workspace; Orchestrator.lockedWorkspace = workspace;
} else { } else {
await (SharedWorkspaceLocking.useRclone await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`) ? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`)
@@ -358,9 +358,9 @@ export class SharedWorkspaceLocking {
await SharedWorkspaceLocking.ensureBucketExists(); await SharedWorkspaceLocking.ensureBucketExists();
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext); const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId)); const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`); OrchestratorLogger.log(`All Locks ${files} ${workspace} ${runId}`);
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`); OrchestratorLogger.log(`Deleting lock ${workspace}/${file}`);
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`); OrchestratorLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
if (file) { if (file) {
await (SharedWorkspaceLocking.useRclone await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone( ? SharedWorkspaceLocking.rclone(

View File

@@ -1,10 +1,10 @@
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import Input from '../../../input'; import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable'; import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader'; import OrchestratorOptionsReader from '../../options/orchestrator-options-reader';
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override'; import OrchestratorQueryOverride from '../../options/orchestrator-query-override';
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
import { CommandHookService } from '../hooks/command-hook-service'; import { CommandHookService } from '../hooks/command-hook-service';
export class TaskParameterSerializer { export class TaskParameterSerializer {
@@ -19,10 +19,10 @@ export class TaskParameterSerializer {
'NAME', 'NAME',
'CUSTOM_JOB', 'CUSTOM_JOB',
]); ]);
public static createCloudRunnerEnvironmentVariables( public static createOrchestratorEnvironmentVariables(
buildParameters: BuildParameters, buildParameters: BuildParameters,
): CloudRunnerEnvironmentVariable[] { ): OrchestratorEnvironmentVariable[] {
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy( const result: OrchestratorEnvironmentVariable[] = this.uniqBy(
[ [
...[ ...[
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform }, { name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
@@ -31,7 +31,7 @@ export class TaskParameterSerializer {
], ],
...TaskParameterSerializer.serializeFromObject(buildParameters), ...TaskParameterSerializer.serializeFromObject(buildParameters),
...TaskParameterSerializer.serializeInput(), ...TaskParameterSerializer.serializeInput(),
...TaskParameterSerializer.serializeCloudRunnerOptions(), ...TaskParameterSerializer.serializeOrchestratorOptions(),
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)), ...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
// Include AWS environment variables for LocalStack compatibility // Include AWS environment variables for LocalStack compatibility
@@ -50,14 +50,14 @@ export class TaskParameterSerializer {
return x; return x;
}), }),
(item: CloudRunnerEnvironmentVariable) => item.name, (item: OrchestratorEnvironmentVariable) => item.name,
); );
return result; return result;
} }
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) { static uniqBy(a: OrchestratorEnvironmentVariable[], key: (parameters: OrchestratorEnvironmentVariable) => string) {
const seen: { [key: string]: boolean } = {}; const seen: { [key: string]: boolean } = {};
return a.filter(function (item) { return a.filter(function (item) {
@@ -90,8 +90,8 @@ export class TaskParameterSerializer {
return TaskParameterSerializer.serializeFromType(Input); return TaskParameterSerializer.serializeFromType(Input);
} }
private static serializeCloudRunnerOptions() { private static serializeOrchestratorOptions() {
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions); return TaskParameterSerializer.serializeFromType(OrchestratorOptions);
} }
private static serializeAwsEnvironmentVariables() { private static serializeAwsEnvironmentVariables() {
@@ -117,7 +117,7 @@ export class TaskParameterSerializer {
} }
public static ToEnvVarFormat(input: string): string { public static ToEnvVarFormat(input: string): string {
return CloudRunnerOptions.ToEnvVarFormat(input); return OrchestratorOptions.ToEnvVarFormat(input);
} }
public static UndoEnvVarFormat(element: string): string { public static UndoEnvVarFormat(element: string): string {
@@ -153,7 +153,7 @@ export class TaskParameterSerializer {
private static serializeFromType(type: any) { private static serializeFromType(type: any) {
const array: any[] = []; const array: any[] = [];
const input = CloudRunnerOptionsReader.GetProperties(); const input = OrchestratorOptionsReader.GetProperties();
for (const element of input) { for (const element of input) {
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) { if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
array.push({ array.push({
@@ -166,7 +166,7 @@ export class TaskParameterSerializer {
return array; return array;
} }
public static readDefaultSecrets(): CloudRunnerSecret[] { public static readDefaultSecrets(): OrchestratorSecret[] {
let array = new Array(); let array = new Array();
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL'); array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL'); array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
@@ -179,13 +179,13 @@ export class TaskParameterSerializer {
} }
private static getValue(key: string) { private static getValue(key: string) {
return CloudRunnerQueryOverride.queryOverrides !== undefined && return OrchestratorQueryOverride.queryOverrides !== undefined &&
CloudRunnerQueryOverride.queryOverrides[key] !== undefined OrchestratorQueryOverride.queryOverrides[key] !== undefined
? CloudRunnerQueryOverride.queryOverrides[key] ? OrchestratorQueryOverride.queryOverrides[key]
: process.env[key]; : process.env[key];
} }
private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] { private static tryAddInput(array: OrchestratorSecret[], key: string): OrchestratorSecret[] {
const value = TaskParameterSerializer.getValue(key); const value = TaskParameterSerializer.getValue(key);
if (value !== undefined && value !== '' && value !== 'null') { if (value !== undefined && value !== '' && value !== 'null') {
array.push({ array.push({

View File

@@ -2,26 +2,26 @@ import { BuildParameters, Input } from '../../..';
import YAML from 'yaml'; import YAML from 'yaml';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger'; import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path'; import path from 'node:path';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import CloudRunnerLogger from '../core/cloud-runner-logger'; import OrchestratorLogger from '../core/orchestrator-logger';
import { CommandHook } from './command-hook'; import { CommandHook } from './command-hook';
// import CloudRunnerLogger from './cloud-runner-logger'; // import OrchestratorLogger from './orchestrator-logger';
export class CommandHookService { export class CommandHookService {
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string { public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CommandHookService.getHooks(buildParameters.commandHooks); const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`); OrchestratorLogger.log(`Applying hooks ${hooks.length}`);
return `echo "---" return `echo "---"
echo "start cloud runner init" echo "start orchestrator init"
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`} ${OrchestratorOptions.orchestratorDebug ? `printenv` : `#`}
echo "start of cloud runner job" echo "start of orchestrator job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} ${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands} ${commands}
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '} ${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
echo "end of cloud runner job" echo "end of orchestrator job"
echo "---${buildParameters.logId}"`; echo "---${buildParameters.logId}"`;
} }
@@ -51,7 +51,7 @@ echo "---${buildParameters.logId}"`;
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`); const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath); const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) { for (const file of files) {
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) { if (!OrchestratorOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
continue; continue;
} }
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`); const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
@@ -89,9 +89,9 @@ echo "---${buildParameters.logId}"`;
return []; return [];
} }
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) { // if (Orchestrator.buildParameters?.orchestratorIntegrationTests) {
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`); // OrchestratorLogger.log(`Parsing build hooks: ${steps}`);
// } // }
const isArray = hooks.replace(/\s/g, ``)[0] === `-`; const isArray = hooks.replace(/\s/g, ``)[0] === `-`;

View File

@@ -1,8 +1,8 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
export class CommandHook { export class CommandHook {
public commands: string[] = new Array<string>(); public commands: string[] = new Array<string>();
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>(); public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
public name!: string; public name!: string;
public hook!: string[]; public hook!: string[];
public step!: string[]; public step!: string[];

View File

@@ -1,13 +1,13 @@
import YAML from 'yaml'; import YAML from 'yaml';
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { CustomWorkflow } from '../../workflows/custom-workflow'; import { CustomWorkflow } from '../../workflows/custom-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger'; import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path'; import path from 'node:path';
import fs from 'node:fs'; import fs from 'node:fs';
import Input from '../../../input'; import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import { ContainerHook as ContainerHook } from './container-hook'; import { ContainerHook as ContainerHook } from './container-hook';
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters'; import { OrchestratorStepParameters } from '../../options/orchestrator-step-parameters';
export class ContainerHookService { export class ContainerHookService {
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] { static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
@@ -16,7 +16,7 @@ export class ContainerHookService {
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`); const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
const files = fs.readdirSync(gameCiCustomStepsPath); const files = fs.readdirSync(gameCiCustomStepsPath);
for (const file of files) { for (const file of files) {
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) { if (!OrchestratorOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`); // RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
continue; continue;
} }
@@ -49,13 +49,13 @@ export class ContainerHookService {
fi fi
ENDPOINT_ARGS="" ENDPOINT_ARGS=""
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${ aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${ } s3://${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} || true } || true
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${ rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} || true } || true
else else
echo "AWS CLI not available, skipping aws-s3-upload-build" echo "AWS CLI not available, skipping aws-s3-upload-build"
@@ -68,7 +68,7 @@ export class ContainerHookService {
- name: awsDefaultRegion - name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``} value: ${process.env.AWS_REGION || ``}
- name: AWS_S3_ENDPOINT - name: AWS_S3_ENDPOINT
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``} value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
- name: aws-s3-pull-build - name: aws-s3-pull-build
image: amazon/aws-cli image: amazon/aws-cli
commands: | commands: |
@@ -85,14 +85,14 @@ export class ContainerHookService {
fi fi
ENDPOINT_ARGS="" ENDPOINT_ARGS=""
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ || true
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build || true
aws s3 cp s3://${ aws s3 cp s3://${
CloudRunner.buildParameters.awsStackName Orchestrator.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${ }/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${ } /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} || true } || true
else else
echo "AWS CLI not available, skipping aws-s3-pull-build" echo "AWS CLI not available, skipping aws-s3-pull-build"
@@ -156,12 +156,12 @@ export class ContainerHookService {
ENDPOINT_ARGS="" ENDPOINT_ARGS=""
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${ aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
CloudRunner.buildParameters.awsStackName Orchestrator.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/lfs || true }/orchestrator-cache/$CACHE_KEY/lfs || true
rm -r /data/cache/$CACHE_KEY/lfs || true rm -r /data/cache/$CACHE_KEY/lfs || true
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${ aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
CloudRunner.buildParameters.awsStackName Orchestrator.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/Library || true }/orchestrator-cache/$CACHE_KEY/Library || true
rm -r /data/cache/$CACHE_KEY/Library || true rm -r /data/cache/$CACHE_KEY/Library || true
else else
echo "AWS CLI not available, skipping aws-s3-upload-cache" echo "AWS CLI not available, skipping aws-s3-upload-cache"
@@ -174,7 +174,7 @@ export class ContainerHookService {
- name: AWS_DEFAULT_REGION - name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``} value: ${process.env.AWS_REGION || ``}
- name: AWS_S3_ENDPOINT - name: AWS_S3_ENDPOINT
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``} value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
- name: aws-s3-pull-cache - name: aws-s3-pull-cache
image: amazon/aws-cli image: amazon/aws-cli
hook: before hook: before
@@ -193,11 +193,11 @@ export class ContainerHookService {
fi fi
ENDPOINT_ARGS="" ENDPOINT_ARGS=""
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ 2>/dev/null || true aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ 2>/dev/null || true
aws $ENDPOINT_ARGS s3 ls ${ aws $ENDPOINT_ARGS s3 ls ${
CloudRunner.buildParameters.awsStackName Orchestrator.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/ 2>/dev/null || true }/orchestrator-cache/$CACHE_KEY/ 2>/dev/null || true
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/" BUCKET1="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/Library/"
OBJECT1="" OBJECT1=""
LS_OUTPUT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 2>/dev/null || echo '')" LS_OUTPUT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 2>/dev/null || echo '')"
if [ -n "$LS_OUTPUT1" ] && [ "$LS_OUTPUT1" != "" ]; then if [ -n "$LS_OUTPUT1" ] && [ "$LS_OUTPUT1" != "" ]; then
@@ -206,7 +206,7 @@ export class ContainerHookService {
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ 2>/dev/null || true aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ 2>/dev/null || true
fi fi
fi fi
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/" BUCKET2="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/lfs/"
OBJECT2="" OBJECT2=""
LS_OUTPUT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 2>/dev/null || echo '')" LS_OUTPUT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 2>/dev/null || echo '')"
if [ -n "$LS_OUTPUT2" ] && [ "$LS_OUTPUT2" != "" ]; then if [ -n "$LS_OUTPUT2" ] && [ "$LS_OUTPUT2" != "" ]; then
@@ -223,29 +223,29 @@ export class ContainerHookService {
hook: after hook: after
commands: | commands: |
if command -v rclone > /dev/null 2>&1; then if command -v rclone > /dev/null 2>&1; then
rclone copy /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${ rclone copy /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} ${CloudRunner.buildParameters.rcloneRemote}/cloud-runner-cache/$CACHE_KEY/build/ || true } ${Orchestrator.buildParameters.rcloneRemote}/orchestrator-cache/$CACHE_KEY/build/ || true
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${ rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} || true } || true
else else
echo "rclone not available, skipping rclone-upload-build" echo "rclone not available, skipping rclone-upload-build"
fi fi
secrets: secrets:
- name: RCLONE_REMOTE - name: RCLONE_REMOTE
value: ${CloudRunner.buildParameters.rcloneRemote || ``} value: ${Orchestrator.buildParameters.rcloneRemote || ``}
- name: rclone-pull-build - name: rclone-pull-build
image: rclone/rclone image: rclone/rclone
commands: | commands: |
mkdir -p /data/cache/$CACHE_KEY/build/ mkdir -p /data/cache/$CACHE_KEY/build/
if command -v rclone > /dev/null 2>&1; then if command -v rclone > /dev/null 2>&1; then
rclone copy ${ rclone copy ${
CloudRunner.buildParameters.rcloneRemote Orchestrator.buildParameters.rcloneRemote
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${ }/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${ } /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : '' Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
} || true } || true
else else
echo "rclone not available, skipping rclone-pull-build" echo "rclone not available, skipping rclone-pull-build"
@@ -253,26 +253,26 @@ export class ContainerHookService {
secrets: secrets:
- name: BUILD_GUID_TARGET - name: BUILD_GUID_TARGET
- name: RCLONE_REMOTE - name: RCLONE_REMOTE
value: ${CloudRunner.buildParameters.rcloneRemote || ``} value: ${Orchestrator.buildParameters.rcloneRemote || ``}
- name: rclone-upload-cache - name: rclone-upload-cache
image: rclone/rclone image: rclone/rclone
hook: after hook: after
commands: | commands: |
if command -v rclone > /dev/null 2>&1; then if command -v rclone > /dev/null 2>&1; then
rclone copy /data/cache/$CACHE_KEY/lfs ${ rclone copy /data/cache/$CACHE_KEY/lfs ${
CloudRunner.buildParameters.rcloneRemote Orchestrator.buildParameters.rcloneRemote
}/cloud-runner-cache/$CACHE_KEY/lfs || true }/orchestrator-cache/$CACHE_KEY/lfs || true
rm -r /data/cache/$CACHE_KEY/lfs || true rm -r /data/cache/$CACHE_KEY/lfs || true
rclone copy /data/cache/$CACHE_KEY/Library ${ rclone copy /data/cache/$CACHE_KEY/Library ${
CloudRunner.buildParameters.rcloneRemote Orchestrator.buildParameters.rcloneRemote
}/cloud-runner-cache/$CACHE_KEY/Library || true }/orchestrator-cache/$CACHE_KEY/Library || true
rm -r /data/cache/$CACHE_KEY/Library || true rm -r /data/cache/$CACHE_KEY/Library || true
else else
echo "rclone not available, skipping rclone-upload-cache" echo "rclone not available, skipping rclone-upload-cache"
fi fi
secrets: secrets:
- name: RCLONE_REMOTE - name: RCLONE_REMOTE
value: ${CloudRunner.buildParameters.rcloneRemote || ``} value: ${Orchestrator.buildParameters.rcloneRemote || ``}
- name: rclone-pull-cache - name: rclone-pull-cache
image: rclone/rclone image: rclone/rclone
hook: before hook: before
@@ -281,24 +281,24 @@ export class ContainerHookService {
mkdir -p /data/cache/$CACHE_KEY/lfs/ mkdir -p /data/cache/$CACHE_KEY/lfs/
if command -v rclone > /dev/null 2>&1; then if command -v rclone > /dev/null 2>&1; then
rclone copy ${ rclone copy ${
CloudRunner.buildParameters.rcloneRemote Orchestrator.buildParameters.rcloneRemote
}/cloud-runner-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true }/orchestrator-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true
rclone copy ${ rclone copy ${
CloudRunner.buildParameters.rcloneRemote Orchestrator.buildParameters.rcloneRemote
}/cloud-runner-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true }/orchestrator-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true
else else
echo "rclone not available, skipping rclone-pull-cache" echo "rclone not available, skipping rclone-pull-cache"
fi fi
secrets: secrets:
- name: RCLONE_REMOTE - name: RCLONE_REMOTE
value: ${CloudRunner.buildParameters.rcloneRemote || ``} value: ${Orchestrator.buildParameters.rcloneRemote || ``}
- name: debug-cache - name: debug-cache
image: ubuntu image: ubuntu
hook: after hook: after
commands: | commands: |
apt-get update > /dev/null || true apt-get update > /dev/null || true
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null || true` : `#`} ${OrchestratorOptions.orchestratorDebug ? `apt-get install -y tree > /dev/null || true` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache || true` : `#`} ${OrchestratorOptions.orchestratorDebug ? `tree -L 3 /data/cache || true` : `#`}
secrets: secrets:
- name: awsAccessKeyId - name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``} value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@@ -307,11 +307,11 @@ export class ContainerHookService {
- name: awsDefaultRegion - name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``} value: ${process.env.AWS_REGION || ``}
- name: AWS_S3_ENDPOINT - name: AWS_S3_ENDPOINT
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`, value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`,
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle); ).filter((x) => OrchestratorOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
// In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks // In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks
const provider = CloudRunner.buildParameters?.providerStrategy; const provider = Orchestrator.buildParameters?.providerStrategy;
const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker'; const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker';
const hasAwsCreds = const hasAwsCreds =
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) || (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
@@ -320,7 +320,7 @@ export class ContainerHookService {
// Always include AWS hooks on the AWS provider (task role provides creds), // Always include AWS hooks on the AWS provider (task role provides creds),
// otherwise require explicit creds for other containerized providers. // otherwise require explicit creds for other containerized providers.
const shouldIncludeAwsHooks = const shouldIncludeAwsHooks =
isContainerized && !CloudRunner.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds)); isContainerized && !Orchestrator.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds));
const filteredBuiltIns = shouldIncludeAwsHooks const filteredBuiltIns = shouldIncludeAwsHooks
? builtInContainerHooks ? builtInContainerHooks
: builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli'); : builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli');
@@ -360,8 +360,8 @@ export class ContainerHookService {
} else { } else {
for (const secret of step.secrets) { for (const secret of step.secrets) {
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) { if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
if (CloudRunner.buildParameters?.cloudRunnerDebug) { if (Orchestrator.buildParameters?.orchestratorDebug) {
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`); // OrchestratorLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
} }
secret.ParameterValue = process.env[secret.ParameterKey] || ``; secret.ParameterValue = process.env[secret.ParameterKey] || ``;
} }
@@ -383,35 +383,35 @@ export class ContainerHookService {
return object; return object;
} }
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) { static async RunPostBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
let output = ``; let output = ``;
const steps: ContainerHook[] = [ const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks), ...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.postBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`after`), ...ContainerHookService.GetContainerHooksFromFiles(`after`),
]; ];
if (steps.length > 0) { if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob( output += await CustomWorkflow.runContainerJob(
steps, steps,
cloudRunnerStepState.environment, orchestratorStepState.environment,
cloudRunnerStepState.secrets, orchestratorStepState.secrets,
); );
} }
return output; return output;
} }
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) { static async RunPreBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
let output = ``; let output = ``;
const steps: ContainerHook[] = [ const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks), ...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.preBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`before`), ...ContainerHookService.GetContainerHooksFromFiles(`before`),
]; ];
if (steps.length > 0) { if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob( output += await CustomWorkflow.runContainerJob(
steps, steps,
cloudRunnerStepState.environment, orchestratorStepState.environment,
cloudRunnerStepState.secrets, orchestratorStepState.secrets,
); );
} }

View File

@@ -1,8 +1,8 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret'; import OrchestratorSecret from '../../options/orchestrator-secret';
export class ContainerHook { export class ContainerHook {
public commands!: string; public commands!: string;
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>(); public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
public name!: string; public name!: string;
public image: string = `ubuntu`; public image: string = `ubuntu`;
public hook!: string; public hook!: string;

View File

@@ -1,20 +1,20 @@
import path from 'node:path'; import path from 'node:path';
import { CloudRunnerFolders } from '../../options/cloud-runner-folders'; import { OrchestratorFolders } from '../../options/orchestrator-folders';
import { CloudRunnerSystem } from '../core/cloud-runner-system'; import { OrchestratorSystem } from '../core/orchestrator-system';
import fs from 'node:fs'; import fs from 'node:fs';
import { Cli } from '../../../cli/cli'; import { Cli } from '../../../cli/cli';
import { CliFunction } from '../../../cli/cli-functions-repository'; import { CliFunction } from '../../../cli/cli-functions-repository';
export class LfsHashing { export class LfsHashing {
public static async createLFSHashFiles() { public static async createLFSHashFiles() {
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`); await OrchestratorSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`); await OrchestratorSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
const lfsHashes = { const lfsHashes = {
lfsGuid: fs lfsGuid: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8') .readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
.replace(/\n/g, ``), .replace(/\n/g, ``),
lfsGuidSum: fs lfsGuidSum: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8') .readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
.replace(' .lfs-assets-guid', '') .replace(' .lfs-assets-guid', '')
.replace(/\n/g, ``), .replace(/\n/g, ``),
}; };
@@ -24,7 +24,7 @@ export class LfsHashing {
public static async hashAllFiles(folder: string) { public static async hashAllFiles(folder: string) {
const startPath = process.cwd(); const startPath = process.cwd();
process.chdir(folder); process.chdir(folder);
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`)) const result = await (await OrchestratorSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
.replace(/\n/g, '') .replace(/\n/g, '')
.split(` `)[0]; .split(` `)[0];
process.chdir(startPath); process.chdir(startPath);

View File

@@ -1,13 +1,13 @@
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { BuildParameters, ImageTag } from '../../..'; import { BuildParameters, ImageTag } from '../../..';
import UnityVersioning from '../../../unity-versioning'; import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli'; import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import setups from '../cloud-runner-suite.test'; import setups from '../orchestrator-suite.test';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
async function CreateParameters(overrides: any) { async function CreateParameters(overrides: any) {
if (overrides) { if (overrides) {
@@ -17,10 +17,10 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create(); return await BuildParameters.create();
} }
describe('Cloud Runner Caching', () => { describe('Orchestrator Caching', () => {
it('Responds', () => {}); it('Responds', () => {});
setups(); setups();
if (CloudRunnerOptions.cloudRunnerDebug) { if (OrchestratorOptions.orchestratorDebug) {
it('Run one build it should not use cache, run subsequent build which should use cache', async () => { it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
const overrides: any = { const overrides: any = {
versioning: 'None', versioning: 'None',
@@ -30,8 +30,8 @@ describe('Cloud Runner Caching', () => {
targetPlatform: 'StandaloneLinux64', targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`, cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `debug-cache`, containerHookFiles: `debug-cache`,
cloudRunnerBranch: `cloud-runner-develop`, orchestratorBranch: `orchestrator-develop`,
cloudRunnerDebug: true, orchestratorDebug: true,
}; };
// For AWS LocalStack tests, explicitly set provider strategy to 'aws' // For AWS LocalStack tests, explicitly set provider strategy to 'aws'
@@ -40,19 +40,19 @@ describe('Cloud Runner Caching', () => {
if ( if (
process.env.AWS_S3_ENDPOINT && process.env.AWS_S3_ENDPOINT &&
process.env.AWS_S3_ENDPOINT.includes('localhost') && process.env.AWS_S3_ENDPOINT.includes('localhost') &&
CloudRunnerOptions.providerStrategy !== 'k8s' OrchestratorOptions.providerStrategy !== 'k8s'
) { ) {
overrides.providerStrategy = 'aws'; overrides.providerStrategy = 'aws';
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`; overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
} }
if (CloudRunnerOptions.providerStrategy === `k8s`) { if (OrchestratorOptions.providerStrategy === `k8s`) {
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`; overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
} }
const buildParameter = await CreateParameters(overrides); const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath); expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString()); const resultsObject = await Orchestrator.run(buildParameter, baseImage.toString());
const results = resultsObject.BuildResults; const results = resultsObject.BuildResults;
const libraryString = 'Rebuilding Library because the asset database could not be found!'; const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library'; const cachePushFail = 'Did not push source folder to cache because it was empty Library';
@@ -62,26 +62,26 @@ describe('Cloud Runner Caching', () => {
// Keep minimal assertions to reduce brittleness // Keep minimal assertions to reduce brittleness
expect(results).not.toContain(cachePushFail); expect(results).not.toContain(cachePushFail);
CloudRunnerLogger.log(`run 1 succeeded`); OrchestratorLogger.log(`run 1 succeeded`);
if (CloudRunnerOptions.providerStrategy === `local-docker`) { if (OrchestratorOptions.providerStrategy === `local-docker`) {
await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`); await OrchestratorSystem.Run(`tree ./orchestrator-cache/cache`);
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`, `cp ./orchestrator-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
); );
await CloudRunnerSystem.Run(`mkdir results`); await OrchestratorSystem.Run(`mkdir results`);
await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`); await OrchestratorSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
await CloudRunnerSystem.Run(`tree -d ./results`); await OrchestratorSystem.Run(`tree -d ./results`);
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`); const cacheFolderExists = fs.existsSync(`orchestrator-cache/cache/${overrides.cacheKey}`);
expect(cacheFolderExists).toBeTruthy(); expect(cacheFolderExists).toBeTruthy();
} }
const buildParameter2 = await CreateParameters(overrides); const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey; buildParameter2.cacheKey = buildParameter.cacheKey;
const baseImage2 = new ImageTag(buildParameter2); const baseImage2 = new ImageTag(buildParameter2);
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString()); const results2Object = await Orchestrator.run(buildParameter2, baseImage2.toString());
const results2 = results2Object.BuildResults; const results2 = results2Object.BuildResults;
CloudRunnerLogger.log(`run 2 succeeded`); OrchestratorLogger.log(`run 2 succeeded`);
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey); const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes( const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
@@ -101,33 +101,33 @@ describe('Cloud Runner Caching', () => {
}, 1_000_000_000); }, 1_000_000_000);
afterAll(async () => { afterAll(async () => {
// Clean up cache files to prevent disk space issues // Clean up cache files to prevent disk space issues
if (CloudRunnerOptions.providerStrategy === `local-docker` || CloudRunnerOptions.providerStrategy === `aws`) { if (OrchestratorOptions.providerStrategy === `local-docker` || OrchestratorOptions.providerStrategy === `aws`) {
const cachePath = `./cloud-runner-cache`; const cachePath = `./orchestrator-cache`;
if (fs.existsSync(cachePath)) { if (fs.existsSync(cachePath)) {
try { try {
CloudRunnerLogger.log(`Cleaning up cache directory: ${cachePath}`); OrchestratorLogger.log(`Cleaning up cache directory: ${cachePath}`);
// Try to change ownership first (if running as root or with sudo) // Try to change ownership first (if running as root or with sudo)
// Then try multiple cleanup methods to handle permission issues // Then try multiple cleanup methods to handle permission issues
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${cachePath} 2>/dev/null || chown -R $(whoami) ${cachePath} 2>/dev/null || true`, `chmod -R u+w ${cachePath} 2>/dev/null || chown -R $(whoami) ${cachePath} 2>/dev/null || true`,
); );
// Try regular rm first // Try regular rm first
await CloudRunnerSystem.Run(`rm -rf ${cachePath}/* 2>/dev/null || true`); await OrchestratorSystem.Run(`rm -rf ${cachePath}/* 2>/dev/null || true`);
// If that fails, try with sudo if available // If that fails, try with sudo if available
await CloudRunnerSystem.Run(`sudo rm -rf ${cachePath}/* 2>/dev/null || true`); await OrchestratorSystem.Run(`sudo rm -rf ${cachePath}/* 2>/dev/null || true`);
// As last resort, try to remove files one by one, ignoring permission errors // As last resort, try to remove files one by one, ignoring permission errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cachePath} -type f -exec rm -f {} + 2>/dev/null || find ${cachePath} -type f -delete 2>/dev/null || true`, `find ${cachePath} -type f -exec rm -f {} + 2>/dev/null || find ${cachePath} -type f -delete 2>/dev/null || true`,
); );
// Remove empty directories // Remove empty directories
await CloudRunnerSystem.Run(`find ${cachePath} -type d -empty -delete 2>/dev/null || true`); await OrchestratorSystem.Run(`find ${cachePath} -type d -empty -delete 2>/dev/null || true`);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup cache: ${error.message}`); OrchestratorLogger.log(`Failed to cleanup cache: ${error.message}`);
// Don't throw - cleanup failures shouldn't fail the test suite // Don't throw - cleanup failures shouldn't fail the test suite
} }

View File

@@ -1,11 +1,11 @@
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { BuildParameters } from '../../..'; import { BuildParameters } from '../../..';
import UnityVersioning from '../../../unity-versioning'; import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli'; import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import setups from '../cloud-runner-suite.test'; import setups from '../orchestrator-suite.test';
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking'; import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
async function CreateParameters(overrides: any) { async function CreateParameters(overrides: any) {
@@ -16,10 +16,10 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create(); return await BuildParameters.create();
} }
describe('Cloud Runner Locking', () => { describe('Orchestrator Locking', () => {
setups(); setups();
it('Responds', () => {}); it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) { if (OrchestratorOptions.orchestratorDebug) {
it(`Simple Locking End2End Flow`, async () => { it(`Simple Locking End2End Flow`, async () => {
const overrides: any = { const overrides: any = {
versioning: 'None', versioning: 'None',
@@ -33,7 +33,7 @@ describe('Cloud Runner Locking', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`; const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4(); const runId = uuidv4();
CloudRunner.buildParameters = buildParameters; Orchestrator.buildParameters = buildParameters;
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters); await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy(); expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy(); expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
@@ -69,7 +69,7 @@ describe('Cloud Runner Locking', () => {
newWorkspaceName, newWorkspaceName,
buildParameters, buildParameters,
); );
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4)); OrchestratorLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length).toBe(1); expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters); await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters); const locks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
@@ -85,7 +85,7 @@ describe('Cloud Runner Locking', () => {
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false, (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false,
).toBeTruthy(); ).toBeTruthy();
await SharedWorkspaceLocking.CleanupWorkspace(newWorkspaceName, buildParameters); await SharedWorkspaceLocking.CleanupWorkspace(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(`Starting get or create`); OrchestratorLogger.log(`Starting get or create`);
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy(); expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
}, 350000); }, 350000);
} }

View File

@@ -1,21 +1,21 @@
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import { ImageTag } from '../../..'; import { ImageTag } from '../../..';
import UnityVersioning from '../../../unity-versioning'; import UnityVersioning from '../../../unity-versioning';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import setups from './../cloud-runner-suite.test'; import setups from './../orchestrator-suite.test';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import { CloudRunnerFolders } from '../../options/cloud-runner-folders'; import { OrchestratorFolders } from '../../options/orchestrator-folders';
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking'; import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
import { CreateParameters } from '../create-test-parameter'; import { CreateParameters } from '../create-test-parameter';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../../services/core/orchestrator-system';
describe('Cloud Runner Retain Workspace', () => { describe('Orchestrator Retain Workspace', () => {
it('Responds', () => {}); it('Responds', () => {});
setups(); setups();
if (CloudRunnerOptions.cloudRunnerDebug) { if (OrchestratorOptions.orchestratorDebug) {
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => { it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
const overrides = { const overrides = {
versioning: 'None', versioning: 'None',
@@ -24,13 +24,13 @@ describe('Cloud Runner Retain Workspace', () => {
targetPlatform: 'StandaloneLinux64', targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`, cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 1, maxRetainedWorkspaces: 1,
cloudRunnerDebug: true, orchestratorDebug: true,
}; };
const buildParameter = await CreateParameters(overrides); const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath); expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString()); const resultsObject = await Orchestrator.run(buildParameter, baseImage.toString());
const results = resultsObject.BuildResults; const results = resultsObject.BuildResults;
const libraryString = 'Rebuilding Library because the asset database could not be found!'; const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library'; const cachePushFail = 'Did not push source folder to cache because it was empty Library';
@@ -40,44 +40,44 @@ describe('Cloud Runner Retain Workspace', () => {
// Keep minimal assertions to reduce brittleness // Keep minimal assertions to reduce brittleness
expect(results).not.toContain(cachePushFail); expect(results).not.toContain(cachePushFail);
if (CloudRunnerOptions.providerStrategy === `local-docker`) { if (OrchestratorOptions.providerStrategy === `local-docker`) {
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`); const cacheFolderExists = fs.existsSync(`orchestrator-cache/cache/${overrides.cacheKey}`);
expect(cacheFolderExists).toBeTruthy(); expect(cacheFolderExists).toBeTruthy();
await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache`); await OrchestratorSystem.Run(`tree -d ./orchestrator-cache`);
} }
CloudRunnerLogger.log(`run 1 succeeded`); OrchestratorLogger.log(`run 1 succeeded`);
// Clean up k3d node between builds to free space, but preserve Unity image // Clean up k3d node between builds to free space, but preserve Unity image
if (CloudRunnerOptions.providerStrategy === 'k8s') { if (OrchestratorOptions.providerStrategy === 'k8s') {
try { try {
CloudRunnerLogger.log('Cleaning up k3d node between builds (preserving Unity image)...'); OrchestratorLogger.log('Cleaning up k3d node between builds (preserving Unity image)...');
const K3D_NODE_CONTAINERS = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0']; const K3D_NODE_CONTAINERS = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
for (const NODE of K3D_NODE_CONTAINERS) { for (const NODE of K3D_NODE_CONTAINERS) {
// Remove stopped containers only - DO NOT touch images // Remove stopped containers only - DO NOT touch images
// Removing images risks removing the Unity image which causes "no space left" errors // Removing images risks removing the Unity image which causes "no space left" errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`docker exec ${NODE} sh -c "crictl rm --all 2>/dev/null || true" || true`, `docker exec ${NODE} sh -c "crictl rm --all 2>/dev/null || true" || true`,
true, true,
true, true,
); );
} }
CloudRunnerLogger.log('Cleanup between builds completed (containers removed, images preserved)'); OrchestratorLogger.log('Cleanup between builds completed (containers removed, images preserved)');
} catch (cleanupError) { } catch (cleanupError) {
CloudRunnerLogger.logWarning(`Failed to cleanup between builds: ${cleanupError}`); OrchestratorLogger.logWarning(`Failed to cleanup between builds: ${cleanupError}`);
// Continue anyway // Continue anyway
} }
} }
// await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache/${}`); // await OrchestratorSystem.Run(`tree -d ./orchestrator-cache/${}`);
const buildParameter2 = await CreateParameters(overrides); const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey; buildParameter2.cacheKey = buildParameter.cacheKey;
const baseImage2 = new ImageTag(buildParameter2); const baseImage2 = new ImageTag(buildParameter2);
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString()); const results2Object = await Orchestrator.run(buildParameter2, baseImage2.toString());
const results2 = results2Object.BuildResults; const results2 = results2Object.BuildResults;
CloudRunnerLogger.log(`run 2 succeeded`); OrchestratorLogger.log(`run 2 succeeded`);
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey); const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid); const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
@@ -101,70 +101,70 @@ describe('Cloud Runner Retain Workspace', () => {
expect(splitResults[splitResults.length - 1]).not.toContain(libraryString); expect(splitResults[splitResults.length - 1]).not.toContain(libraryString);
}, 1_000_000_000); }, 1_000_000_000);
afterAll(async () => { afterAll(async () => {
await SharedWorkspaceLocking.CleanupWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters); await SharedWorkspaceLocking.CleanupWorkspace(Orchestrator.lockedWorkspace || ``, Orchestrator.buildParameters);
if ( if (
fs.existsSync(`./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`) fs.existsSync(`./orchestrator-cache/${path.basename(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`)
) { ) {
CloudRunnerLogger.log( OrchestratorLogger.log(
`Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`, `Cleaning up ./orchestrator-cache/${path.basename(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,
); );
try { try {
const workspaceCachePath = `./cloud-runner-cache/${path.basename( const workspaceCachePath = `./orchestrator-cache/${path.basename(
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute,
)}`; )}`;
// Try to fix permissions first to avoid permission denied errors // Try to fix permissions first to avoid permission denied errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${workspaceCachePath} 2>/dev/null || chown -R $(whoami) ${workspaceCachePath} 2>/dev/null || true`, `chmod -R u+w ${workspaceCachePath} 2>/dev/null || chown -R $(whoami) ${workspaceCachePath} 2>/dev/null || true`,
); );
// Try regular rm first // Try regular rm first
await CloudRunnerSystem.Run(`rm -rf ${workspaceCachePath} 2>/dev/null || true`); await OrchestratorSystem.Run(`rm -rf ${workspaceCachePath} 2>/dev/null || true`);
// If that fails, try with sudo if available // If that fails, try with sudo if available
await CloudRunnerSystem.Run(`sudo rm -rf ${workspaceCachePath} 2>/dev/null || true`); await OrchestratorSystem.Run(`sudo rm -rf ${workspaceCachePath} 2>/dev/null || true`);
// As last resort, try to remove files one by one, ignoring permission errors // As last resort, try to remove files one by one, ignoring permission errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${workspaceCachePath} -type f -exec rm -f {} + 2>/dev/null || find ${workspaceCachePath} -type f -delete 2>/dev/null || true`, `find ${workspaceCachePath} -type f -exec rm -f {} + 2>/dev/null || find ${workspaceCachePath} -type f -delete 2>/dev/null || true`,
); );
// Remove empty directories // Remove empty directories
await CloudRunnerSystem.Run(`find ${workspaceCachePath} -type d -empty -delete 2>/dev/null || true`); await OrchestratorSystem.Run(`find ${workspaceCachePath} -type d -empty -delete 2>/dev/null || true`);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup workspace: ${error.message}`); OrchestratorLogger.log(`Failed to cleanup workspace: ${error.message}`);
// Don't throw - cleanup failures shouldn't fail the test suite // Don't throw - cleanup failures shouldn't fail the test suite
} }
} }
// Clean up cache files to prevent disk space issues // Clean up cache files to prevent disk space issues
const cachePath = `./cloud-runner-cache`; const cachePath = `./orchestrator-cache`;
if (fs.existsSync(cachePath)) { if (fs.existsSync(cachePath)) {
try { try {
CloudRunnerLogger.log(`Cleaning up cache directory: ${cachePath}`); OrchestratorLogger.log(`Cleaning up cache directory: ${cachePath}`);
// Try to change ownership first (if running as root or with sudo) // Try to change ownership first (if running as root or with sudo)
// Then try multiple cleanup methods to handle permission issues // Then try multiple cleanup methods to handle permission issues
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`chmod -R u+w ${cachePath} 2>/dev/null || chown -R $(whoami) ${cachePath} 2>/dev/null || true`, `chmod -R u+w ${cachePath} 2>/dev/null || chown -R $(whoami) ${cachePath} 2>/dev/null || true`,
); );
// Try regular rm first // Try regular rm first
await CloudRunnerSystem.Run(`rm -rf ${cachePath}/* 2>/dev/null || true`); await OrchestratorSystem.Run(`rm -rf ${cachePath}/* 2>/dev/null || true`);
// If that fails, try with sudo if available // If that fails, try with sudo if available
await CloudRunnerSystem.Run(`sudo rm -rf ${cachePath}/* 2>/dev/null || true`); await OrchestratorSystem.Run(`sudo rm -rf ${cachePath}/* 2>/dev/null || true`);
// As last resort, try to remove files one by one, ignoring permission errors // As last resort, try to remove files one by one, ignoring permission errors
await CloudRunnerSystem.Run( await OrchestratorSystem.Run(
`find ${cachePath} -type f -exec rm -f {} + 2>/dev/null || find ${cachePath} -type f -delete 2>/dev/null || true`, `find ${cachePath} -type f -exec rm -f {} + 2>/dev/null || find ${cachePath} -type f -delete 2>/dev/null || true`,
); );
// Remove empty directories // Remove empty directories
await CloudRunnerSystem.Run(`find ${cachePath} -type d -empty -delete 2>/dev/null || true`); await OrchestratorSystem.Run(`find ${cachePath} -type d -empty -delete 2>/dev/null || true`);
} catch (error: any) { } catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup cache: ${error.message}`); OrchestratorLogger.log(`Failed to cleanup cache: ${error.message}`);
// Don't throw - cleanup failures shouldn't fail the test suite // Don't throw - cleanup failures shouldn't fail the test suite
} }

View File

@@ -1,10 +1,10 @@
import CloudRunner from '../../cloud-runner'; import Orchestrator from '../../orchestrator';
import UnityVersioning from '../../../unity-versioning'; import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli'; import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger'; import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options'; import OrchestratorOptions from '../../options/orchestrator-options';
import setups from '../cloud-runner-suite.test'; import setups from '../orchestrator-suite.test';
import BuildParameters from '../../../build-parameters'; import BuildParameters from '../../../build-parameters';
import ImageTag from '../../../image-tag'; import ImageTag from '../../../image-tag';
@@ -16,15 +16,15 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create(); return await BuildParameters.create();
} }
describe('Cloud Runner Kubernetes', () => { describe('Orchestrator Kubernetes', () => {
it('Responds', () => {}); it('Responds', () => {});
setups(); setups();
if (CloudRunnerOptions.cloudRunnerDebug) { if (OrchestratorOptions.orchestratorDebug) {
const enableK8sE2E = process.env.ENABLE_K8S_E2E === 'true'; const enableK8sE2E = process.env.ENABLE_K8S_E2E === 'true';
const testBody = async () => { const testBody = async () => {
if (CloudRunnerOptions.providerStrategy !== `k8s`) { if (OrchestratorOptions.providerStrategy !== `k8s`) {
return; return;
} }
process.env.USE_IL2CPP = 'false'; process.env.USE_IL2CPP = 'false';
@@ -36,13 +36,13 @@ describe('Cloud Runner Kubernetes', () => {
cacheKey: `test-case-${uuidv4()}`, cacheKey: `test-case-${uuidv4()}`,
providerStrategy: 'k8s', providerStrategy: 'k8s',
buildPlatform: 'linux', buildPlatform: 'linux',
cloudRunnerDebug: true, orchestratorDebug: true,
}; };
const buildParameter = await CreateParameters(overrides); const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath); expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString()); const resultsObject = await Orchestrator.run(buildParameter, baseImage.toString());
const results = resultsObject.BuildResults; const results = resultsObject.BuildResults;
const libraryString = 'Rebuilding Library because the asset database could not be found!'; const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library'; const cachePushFail = 'Did not push source folder to cache because it was empty Library';
@@ -70,13 +70,13 @@ describe('Cloud Runner Kubernetes', () => {
// If we hit the aggressive fallback path and couldn't retrieve any logs from the pod, // If we hit the aggressive fallback path and couldn't retrieve any logs from the pod,
// don't assert on specific Unity log contents just assert that we got the fallback message. // don't assert on specific Unity log contents just assert that we got the fallback message.
// This makes the test resilient to cluster-level evictions / PreStop hook failures while still // This makes the test resilient to cluster-level evictions / PreStop hook failures while still
// ensuring Cloud Runner surfaces a useful message in BuildResults. // ensuring Orchestrator surfaces a useful message in BuildResults.
// However, if we got logs but they're incomplete (missing "Collected Logs"), the test should fail // However, if we got logs but they're incomplete (missing "Collected Logs"), the test should fail
// as this indicates the build didn't complete successfully (pod was evicted/killed). // as this indicates the build didn't complete successfully (pod was evicted/killed).
if (results.includes(fallbackLogsUnavailableMessage)) { if (results.includes(fallbackLogsUnavailableMessage)) {
// Complete failure - no logs at all (acceptable for eviction scenarios) // Complete failure - no logs at all (acceptable for eviction scenarios)
expect(results).toContain(fallbackLogsUnavailableMessage); expect(results).toContain(fallbackLogsUnavailableMessage);
CloudRunnerLogger.log('Test passed with fallback message (pod was evicted before any logs were written)'); OrchestratorLogger.log('Test passed with fallback message (pod was evicted before any logs were written)');
} else if (results.includes(incompleteLogsMessage)) { } else if (results.includes(incompleteLogsMessage)) {
// Incomplete logs - we got some output but missing "Collected Logs" (build didn't complete) // Incomplete logs - we got some output but missing "Collected Logs" (build didn't complete)
// This should fail the test as the build didn't succeed // This should fail the test as the build didn't succeed
@@ -93,14 +93,14 @@ describe('Cloud Runner Kubernetes', () => {
expect(results).not.toContain(cachePushFail); expect(results).not.toContain(cachePushFail);
} }
CloudRunnerLogger.log(`run 1 succeeded`); OrchestratorLogger.log(`run 1 succeeded`);
}; };
if (enableK8sE2E) { if (enableK8sE2E) {
it('Run one build it using K8s without error', testBody, 1_000_000_000); it('Run one build it using K8s without error', testBody, 1_000_000_000);
} else { } else {
it.skip('Run one build it using K8s without error - disabled (no outbound network)', () => { it.skip('Run one build it using K8s without error - disabled (no outbound network)', () => {
CloudRunnerLogger.log('Skipping K8s e2e (ENABLE_K8S_E2E not true)'); OrchestratorLogger.log('Skipping K8s e2e (ENABLE_K8S_E2E not true)');
}); });
} }
} }

View File

@@ -1,9 +1,9 @@
import { BuildParameters, ImageTag } from '../..'; import { BuildParameters, ImageTag } from '../..';
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import UnityVersioning from '../../unity-versioning'; import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli'; import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../options/cloud-runner-options'; import OrchestratorOptions from '../options/orchestrator-options';
import setups from './cloud-runner-suite.test'; import setups from './orchestrator-suite.test';
import { OptionValues } from 'commander'; import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) { async function CreateParameters(overrides: OptionValues | undefined) {
@@ -11,18 +11,18 @@ async function CreateParameters(overrides: OptionValues | undefined) {
return BuildParameters.create(); return BuildParameters.create();
} }
describe('Cloud Runner Async Workflows', () => { describe('Orchestrator Async Workflows', () => {
setups(); setups();
it('Responds', () => {}); it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) { if (OrchestratorOptions.orchestratorDebug && OrchestratorOptions.providerStrategy !== `local-docker`) {
it('Async Workflows', async () => { it('Async Workflows', async () => {
// Setup parameters // Setup parameters
const buildParameter = await CreateParameters({ const buildParameter = await CreateParameters({
versioning: 'None', versioning: 'None',
projectPath: 'test-project', projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'), unityVersion: UnityVersioning.read('test-project'),
asyncCloudRunner: `true`, asyncOrchestrator: `true`,
githubChecks: `true`, githubChecks: `true`,
providerStrategy: 'k8s', providerStrategy: 'k8s',
buildPlatform: 'linux', buildPlatform: 'linux',
@@ -31,7 +31,7 @@ describe('Cloud Runner Async Workflows', () => {
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
// Run the job // Run the job
await CloudRunner.run(buildParameter, baseImage.toString()); await Orchestrator.run(buildParameter, baseImage.toString());
// wait for 15 seconds // wait for 15 seconds
await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12)); await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12));

View File

@@ -3,15 +3,15 @@ import path from 'node:path';
import BuildParameters from '../../build-parameters'; import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli'; import { Cli } from '../../cli/cli';
import UnityVersioning from '../../unity-versioning'; import UnityVersioning from '../../unity-versioning';
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system'; import { OrchestratorSystem } from '../services/core/orchestrator-system';
import { Caching } from '../remote-client/caching'; import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github'; import GitHub from '../../github';
import CloudRunnerOptions from '../options/cloud-runner-options'; import OrchestratorOptions from '../options/orchestrator-options';
describe('Cloud Runner (Remote Client) Caching', () => { describe('Orchestrator (Remote Client) Caching', () => {
it('responds', () => {}); it('responds', () => {});
if (CloudRunnerOptions.providerStrategy === `local-docker`) { if (OrchestratorOptions.providerStrategy === `local-docker`) {
it('Simple caching works', async () => { it('Simple caching works', async () => {
Cli.options = { Cli.options = {
versioning: 'None', versioning: 'None',
@@ -22,7 +22,7 @@ describe('Cloud Runner (Remote Client) Caching', () => {
}; };
GitHub.githubInputEnabled = false; GitHub.githubInputEnabled = false;
const buildParameter = await BuildParameters.create(); const buildParameter = await BuildParameters.create();
CloudRunner.buildParameters = buildParameter; Orchestrator.buildParameters = buildParameter;
// Create test folder // Create test folder
const testFolder = path.resolve(__dirname, Cli.options.cacheKey); const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
@@ -43,7 +43,7 @@ describe('Cloud Runner (Remote Client) Caching', () => {
testFolder.replace(/\\/g, `/`), testFolder.replace(/\\/g, `/`),
`${Cli.options.cacheKey}`, `${Cli.options.cacheKey}`,
); );
await CloudRunnerSystem.Run(`du -h ${__dirname}`); await OrchestratorSystem.Run(`du -h ${__dirname}`);
// Compare validity to original hash // Compare validity to original hash
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain( expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(

View File

@@ -1,12 +1,12 @@
import { BuildParameters, CloudRunner, ImageTag, Input } from '../..'; import { BuildParameters, Orchestrator, ImageTag, Input } from '../..';
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer'; import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning'; import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli'; import { Cli } from '../../cli/cli';
import GitHub from '../../github'; import GitHub from '../../github';
import setups from './cloud-runner-suite.test'; import setups from './orchestrator-suite.test';
import { CloudRunnerStatics } from '../options/cloud-runner-statics'; import { OrchestratorStatics } from '../options/orchestrator-statics';
import CloudRunnerOptions from '../options/cloud-runner-options'; import OrchestratorOptions from '../options/orchestrator-options';
import CloudRunnerLogger from '../services/core/cloud-runner-logger'; import OrchestratorLogger from '../services/core/orchestrator-logger';
async function CreateParameters(overrides: any) { async function CreateParameters(overrides: any) {
if (overrides) { if (overrides) {
@@ -21,14 +21,14 @@ async function CreateParameters(overrides: any) {
return results; return results;
} }
describe('Cloud Runner Sync Environments', () => { describe('Orchestrator Sync Environments', () => {
setups(); setups();
const testSecretName = 'testSecretName'; const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue'; const testSecretValue = 'testSecretValue';
it('Responds', () => {}); it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) { if (OrchestratorOptions.orchestratorDebug) {
it('All build parameters sent to cloud runner as env vars', async () => { it('All build parameters sent to orchestrator as env vars', async () => {
// Setup parameters // Setup parameters
const buildParameter = await CreateParameters({ const buildParameter = await CreateParameters({
versioning: 'None', versioning: 'None',
@@ -43,7 +43,7 @@ describe('Cloud Runner Sync Environments', () => {
- name: '${testSecretName}' - name: '${testSecretName}'
value: '${testSecretValue}' value: '${testSecretValue}'
`, `,
cloudRunnerDebug: true, orchestratorDebug: true,
}); });
const baseImage = new ImageTag(buildParameter); const baseImage = new ImageTag(buildParameter);
if (baseImage.toString().includes('undefined')) { if (baseImage.toString().includes('undefined')) {
@@ -51,12 +51,12 @@ describe('Cloud Runner Sync Environments', () => {
} }
// Run the job // Run the job
const file = (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults; const file = (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
// Assert results // Assert results
// expect(file).toContain(JSON.stringify(buildParameter)); // expect(file).toContain(JSON.stringify(buildParameter));
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`); expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
const environmentVariables = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter); const environmentVariables = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => { const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
return { return {
name: x.EnvironmentVariable, name: x.EnvironmentVariable,
@@ -87,7 +87,8 @@ describe('Cloud Runner Sync Environments', () => {
if ( if (
endpointEnvironmentNames.has(x.name) && endpointEnvironmentNames.has(x.name) &&
(x.value.startsWith('http://localhost') || x.value.startsWith('http://127.0.0.1')) && (x.value.startsWith('http://localhost') || x.value.startsWith('http://127.0.0.1')) &&
(CloudRunnerOptions.providerStrategy === 'local-docker' || CloudRunnerOptions.providerStrategy === 'aws') (OrchestratorOptions.providerStrategy === 'local-docker' ||
OrchestratorOptions.providerStrategy === 'aws')
) { ) {
x.value = x.value x.value = x.value
.replace('http://localhost', 'http://host.docker.internal') .replace('http://localhost', 'http://host.docker.internal')
@@ -102,10 +103,10 @@ describe('Cloud Runner Sync Environments', () => {
}); });
const newLinePurgedFile = file const newLinePurgedFile = file
.replace(/\s+/g, '') .replace(/\s+/g, '')
.replace(new RegExp(`\\[${CloudRunnerStatics.logPrefix}\\]`, 'g'), ''); .replace(new RegExp(`\\[${OrchestratorStatics.logPrefix}\\]`, 'g'), '');
for (const element of combined) { for (const element of combined) {
expect(newLinePurgedFile).toContain(`${element.name}`); expect(newLinePurgedFile).toContain(`${element.name}`);
CloudRunnerLogger.log(`Contains ${element.name}`); OrchestratorLogger.log(`Contains ${element.name}`);
const fullNameEqualValue = `${element.name}=${element.value}`; const fullNameEqualValue = `${element.name}=${element.value}`;
expect(newLinePurgedFile).toContain(fullNameEqualValue); expect(newLinePurgedFile).toContain(fullNameEqualValue);
} }
@@ -113,11 +114,11 @@ describe('Cloud Runner Sync Environments', () => {
} }
}); });
describe('Cloud Runner Environment Serializer', () => { describe('Orchestrator Environment Serializer', () => {
setups(); setups();
const testSecretName = 'testSecretName'; const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue'; const testSecretValue = 'testSecretValue';
it('Cloud Runner Parameter Serialization', async () => { it('Orchestrator Parameter Serialization', async () => {
// Setup parameters // Setup parameters
const buildParameter = await CreateParameters({ const buildParameter = await CreateParameters({
versioning: 'None', versioning: 'None',
@@ -133,9 +134,9 @@ describe('Cloud Runner Environment Serializer', () => {
`, `,
}); });
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter); const result = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy(); expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter); const result2 = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy(); expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
}); });
}); });

View File

@@ -1,9 +1,9 @@
import CloudRunner from '../cloud-runner'; import Orchestrator from '../orchestrator';
import UnityVersioning from '../../unity-versioning'; import UnityVersioning from '../../unity-versioning';
import setups from './cloud-runner-suite.test'; import setups from './orchestrator-suite.test';
import GitHub from '../../github'; import GitHub from '../../github';
import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/cloud-runner-test-helpers'; import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/orchestrator-test-helpers';
describe('Cloud Runner Github Checks', () => { describe('Orchestrator Github Checks', () => {
setups(); setups();
it('Responds', () => {}); it('Responds', () => {});
@@ -33,11 +33,11 @@ describe('Cloud Runner Github Checks', () => {
versioning: 'None', versioning: 'None',
projectPath: 'test-project', projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'), unityVersion: UnityVersioning.read('test-project'),
asyncCloudRunner: `true`, asyncOrchestrator: `true`,
githubChecks: `true`, githubChecks: `true`,
}); });
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`); Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`); await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`);
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`); await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`);
}, },
@@ -51,12 +51,12 @@ describe('Cloud Runner Github Checks', () => {
versioning: 'None', versioning: 'None',
projectPath: 'test-project', projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'), unityVersion: UnityVersioning.read('test-project'),
asyncCloudRunner: `true`, asyncOrchestrator: `true`,
githubChecks: `true`, githubChecks: `true`,
}); });
GitHub.forceAsyncTest = true; GitHub.forceAsyncTest = true;
await CloudRunner.setup(buildParameter); await Orchestrator.setup(buildParameter);
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`); Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`); await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`);
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`); await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`);
GitHub.forceAsyncTest = false; GitHub.forceAsyncTest = false;

Some files were not shown because too many files have changed in this diff Show More