name: Orchestrator Integration Tests # ============================================================================== # Exhaustive integration tests — runs on a daily cron and manual dispatch. # Slow (~1-2h wall-clock): k8s, AWS, local-docker, rclone via MiniStack + k3d. # # Mirrors the full orchestrator-integrity.yml test suite from the orchestrator # repo, run from unity-builder's perspective to catch cross-repo regressions. # # For fast per-PR checks, see validate-orchestrator.yml. # ============================================================================== on: workflow_dispatch: workflow_call: # Allow integrity-check.yml and other workflows to invoke these tests schedule: - cron: '0 3 * * *' # Daily at 3 AM UTC permissions: contents: read checks: write statuses: write # Note: no concurrency block here — when invoked via workflow_call, the caller # (integrity-check.yml) manages concurrency. For direct dispatch/cron, runs are # naturally serialized by GitHub's single-concurrency-per-ref default. env: AWS_STACK_NAME: game-ci-team-pipelines DEBUG: true PROJECT_PATH: test-project USE_IL2CPP: false # ============================================================================== # 4 parallel jobs on separate runners (fresh 14GB disk each). # Matches the orchestrator-integrity.yml architecture. # ============================================================================== jobs: # ============================================================================ # PLUGIN INTERFACE (fast gate — fails fast before slow jobs waste time) # ============================================================================ plugin-interface: name: Plugin Interface Tests runs-on: ubuntu-latest steps: - name: Checkout unity-builder uses: actions/checkout@v4 - name: Checkout orchestrator (matching branch) uses: actions/checkout@v4 with: repository: game-ci/orchestrator ref: ${{ github.head_ref || github.ref_name }} path: orchestrator-standalone continue-on-error: true id: orchestrator-branch - name: Fallback to orchestrator main branch if: steps.orchestrator-branch.outcome == 'failure' uses: actions/checkout@v4 with: repository: game-ci/orchestrator path: orchestrator-standalone - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: yarn - name: Install unity-builder dependencies run: yarn install --frozen-lockfile - name: Build unity-builder run: | echo "Building unity-builder TypeScript..." npx tsc echo "✓ unity-builder compiles successfully" - name: Run plugin interface unit tests run: | echo "Running orchestrator-plugin unit tests..." npx jest orchestrator-plugin --verbose --detectOpenHandles --forceExit - name: Build and pack orchestrator working-directory: orchestrator-standalone run: | yarn install --frozen-lockfile echo "Building orchestrator..." npx tsc echo "✓ orchestrator compiles successfully" echo "Packing orchestrator as tarball..." npm pack - name: Install orchestrator into unity-builder run: | echo "Installing orchestrator into unity-builder workspace..." npm install ./orchestrator-standalone/game-ci-orchestrator-*.tgz --no-save --legacy-peer-deps - name: Verify plugin loader returns exports with orchestrator installed run: | node -e " const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin'); (async () => { const plugin = await loadOrchestratorPlugin(); if (plugin === undefined) { console.error('ERROR: loadOrchestratorPlugin should return defined plugin when package is installed'); process.exit(1); } const lifecycleMethods = [ 'initialize', 'canHandleBuild', 'handleBuild', 'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild', ]; for (const method of lifecycleMethods) { if (typeof plugin[method] !== 'function') { console.error('ERROR: plugin.' + method + ' should be a function, got ' + typeof plugin[method]); process.exit(1); } } console.log('✓ loadOrchestratorPlugin() returns plugin with all ' + lifecycleMethods.length + ' lifecycle methods'); })(); " - name: Verify type declarations match exports run: | node -e " const orch = require('@game-ci/orchestrator'); const expected = ['Orchestrator','BuildReliabilityService','TestWorkflowService','HotRunnerService','OutputService','OutputTypeRegistry','ArtifactUploadHandler','IncrementalSyncService','ChildWorkspaceService','LocalCacheService','SubmoduleProfileService','LfsAgentService','GitHooksService']; const missing = expected.filter(e => orch[e] === undefined); if (missing.length > 0) { console.error('Missing exports:', missing.join(', ')); process.exit(1); } console.log('✓ All ' + expected.length + ' exports present'); " # ============================================================================ # K8S INTEGRATION TESTS (k3d + MiniStack) # ============================================================================ k8s-integration: name: K8s Integration Tests runs-on: ubuntu-latest env: K3D_NODE_CONTAINERS: 'k3d-unity-builder-agent-0' AWS_FORCE_PROVIDER: aws-local RESOURCE_TRACKING: 'true' K8S_LOCALSTACK_HOST: localstack-main steps: - name: Checkout orchestrator (matching branch) uses: actions/checkout@v4 with: repository: game-ci/orchestrator ref: ${{ github.head_ref || github.ref_name }} lfs: false continue-on-error: true id: orch-branch - name: Fallback to orchestrator main branch if: steps.orch-branch.outcome == 'failure' uses: actions/checkout@v4 with: repository: game-ci/orchestrator lfs: false - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: yarn - name: Set up kubectl uses: azure/setup-kubectl@v4 with: version: 'v1.34.1' - name: Install k3d run: | curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash k3d version | cat - name: Define cleanup functions run: | cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF' light_cleanup() { echo "--- Light cleanup ---" rm -rf ./orchestrator-cache/* || true docker system prune -f || true df -h } k8s_resource_cleanup() { echo "--- K8s resource cleanup ---" kubectl delete jobs --all --ignore-not-found=true -n default || true kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do kubectl delete "$pod" --ignore-not-found=true || true done || true kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do kubectl delete "$pvc" --ignore-not-found=true || true done || true kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do kubectl delete "$secret" --ignore-not-found=true || true done || true } k3d_node_cleanup() { echo "--- K3d node image cleanup (preserving Unity images) ---" K3D_NODE_CONTAINERS="${K3D_NODE_CONTAINERS:-k3d-unity-builder-agent-0 k3d-unity-builder-server-0}" for NODE in $K3D_NODE_CONTAINERS; do docker exec "$NODE" sh -c "crictl rm --all 2>/dev/null || true" || true docker exec "$NODE" sh -c "for img in \$(crictl images -q 2>/dev/null); do repo=\$(crictl inspecti \$img --format '{{.repo}}' 2>/dev/null || echo ''); if echo \"\$repo\" | grep -qvE 'unityci/editor|unity'; then crictl rmi \$img 2>/dev/null || true; fi; done" || true docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true done || true } full_k8s_cleanup() { k8s_resource_cleanup k3d_node_cleanup light_cleanup } CLEANUP_EOF echo "Cleanup functions defined at /tmp/cleanup-functions.sh" - name: Initial disk space cleanup run: | echo "Initial disk space cleanup..." df -h k3d cluster delete unity-builder || true docker stop localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true docker system prune -af --volumes || true docker network rm orchestrator-net 2>/dev/null || true docker network create orchestrator-net || true echo "Disk usage after cleanup:" df -h - name: Start MiniStack run: | echo "Starting MiniStack..." docker run -d \ --name localstack-main \ --network orchestrator-net \ --add-host=host.docker.internal:host-gateway \ -p 4566:4566 \ ministackorg/ministack:latest || true echo "Waiting for MiniStack to be ready..." MAX_ATTEMPTS=60 READY=false for i in $(seq 1 $MAX_ATTEMPTS); do if ! docker ps | grep -q localstack-main; then sleep 2; continue; fi HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "") if [ -z "$HEALTH" ] || ! echo "$HEALTH" | grep -q "services"; then sleep 2; continue; fi if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack is ready (attempt $i/$MAX_ATTEMPTS)" READY=true break fi sleep 2 done if [ "$READY" != "true" ]; then echo "ERROR: MiniStack did not become ready" docker logs localstack-main --tail 100 || true exit 1 fi - name: Install AWS CLI tools run: | if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi pip install awscli-local || true - name: Create S3 bucket for tests run: | for i in {1..10}; do if curl -s http://localhost:4566/_localstack/health > /dev/null 2>&1; then break; fi sleep 1 done for i in {1..5}; do if command -v awslocal > /dev/null 2>&1; then awslocal s3 mb s3://$AWS_STACK_NAME 2>&1 && break else aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1 && break fi sleep 2 done - run: yarn install --frozen-lockfile - name: Build orchestrator run: | echo "Building orchestrator TypeScript..." yarn build echo "✓ orchestrator build successful" # --- Fast unit tests (fast-fail gate) --- - name: Run orchestrator unit tests (fast, no infra) timeout-minutes: 2 run: >- yarn run test --testPathPattern="orchestrator-guid|orchestrator-folders|task-parameter-serializer|follow-log-stream-service|runner-availability-service|provider-url-parser|provider-loader|provider-git-manager|orchestrator-image|orchestrator-hooks|orchestrator-github-checks|middleware-service" --verbose --detectOpenHandles --forceExit --runInBand # --- K8s cluster setup --- - name: Clean up disk space before K8s tests run: | rm -rf ./orchestrator-cache/* || true sudo apt-get clean || true docker system prune -f || true df -h - name: Create k3s cluster (k3d) timeout-minutes: 5 run: | LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "") echo "MiniStack container IP: $LOCALSTACK_IP" k3d cluster create unity-builder \ --agents 1 \ --network orchestrator-net \ --wait kubectl config current-context | cat echo "LOCALSTACK_IP=$LOCALSTACK_IP" >> $GITHUB_ENV - name: Verify cluster readiness and MiniStack connectivity timeout-minutes: 2 run: | for i in {1..60}; do if kubectl get nodes 2>/dev/null | grep -q Ready; then echo "Cluster is ready"; break; fi echo "Waiting for cluster... ($i/60)" sleep 5 done kubectl get nodes LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "") kubectl run test-localstack --image=curlimages/curl --rm -i --restart=Never --timeout=30s -- \ curl -v --max-time 10 http://${LOCALSTACK_IP}:4566/_localstack/health 2>&1 | head -30 || \ echo "Cluster connectivity test - MiniStack may not be accessible from k3d" - name: Clean up K8s resources before tests run: | source /tmp/cleanup-functions.sh k8s_resource_cleanup for i in {1..30}; do PVC_COUNT=$(kubectl get pvc -n default 2>/dev/null | grep "unity-builder-pvc-" | wc -l || echo "0") if [ "$PVC_COUNT" -eq 0 ]; then echo "All PVCs deleted"; break; fi sleep 1 done docker system prune -f || true # --- K8s Test 1: orchestrator-image --- - name: Run orchestrator-image test (K8s) timeout-minutes: 10 run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None KUBE_STORAGE_CLASS: local-path PROVIDER_STRATEGY: k8s KUBE_VOLUME_SIZE: 2Gi containerCpu: '512' containerMemory: '512' GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-image (K8s) if: always() run: | source /tmp/cleanup-functions.sh full_k8s_cleanup # --- K8s Test 2: orchestrator-kubernetes --- - name: Run orchestrator-kubernetes test timeout-minutes: 30 run: yarn run test "orchestrator-kubernetes" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None KUBE_STORAGE_CLASS: local-path PROVIDER_STRATEGY: k8s KUBE_VOLUME_SIZE: 2Gi containerCpu: '1000' containerMemory: '1024' AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_S3_ENDPOINT: http://localhost:4566 AWS_ENDPOINT: http://localhost:4566 INPUT_AWSS3ENDPOINT: http://localhost:4566 INPUT_AWSENDPOINT: http://localhost:4566 AWS_S3_FORCE_PATH_STYLE: 'true' AWS_EC2_METADATA_DISABLED: 'true' GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-kubernetes if: always() run: | source /tmp/cleanup-functions.sh full_k8s_cleanup # --- K8s Test 3: orchestrator-s3-steps --- - name: Run orchestrator-s3-steps test (K8s) timeout-minutes: 30 run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None KUBE_STORAGE_CLASS: local-path PROVIDER_STRATEGY: k8s KUBE_VOLUME_SIZE: 2Gi containerCpu: '1000' containerMemory: '1024' AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_S3_ENDPOINT: http://localhost:4566 AWS_ENDPOINT: http://localhost:4566 INPUT_AWSS3ENDPOINT: http://localhost:4566 INPUT_AWSENDPOINT: http://localhost:4566 AWS_S3_FORCE_PATH_STYLE: 'true' AWS_EC2_METADATA_DISABLED: 'true' GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-s3-steps (K8s) if: always() run: | source /tmp/cleanup-functions.sh full_k8s_cleanup # --- K8s Test 4: orchestrator-end2end-caching --- - name: Run orchestrator-end2end-caching test (K8s) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None KUBE_STORAGE_CLASS: local-path PROVIDER_STRATEGY: k8s KUBE_VOLUME_SIZE: 2Gi containerCpu: '1000' containerMemory: '1024' AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_S3_ENDPOINT: http://localhost:4566 AWS_ENDPOINT: http://localhost:4566 INPUT_AWSS3ENDPOINT: http://localhost:4566 INPUT_AWSENDPOINT: http://localhost:4566 AWS_S3_FORCE_PATH_STYLE: 'true' AWS_EC2_METADATA_DISABLED: 'true' GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-caching (K8s) if: always() run: | source /tmp/cleanup-functions.sh full_k8s_cleanup # --- K8s Test 5: orchestrator-end2end-retaining --- - name: Extra disk cleanup before retaining test run: | source /tmp/cleanup-functions.sh full_k8s_cleanup docker system prune -af --volumes || true df -h - name: Run orchestrator-end2end-retaining test (K8s) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-retaining" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None KUBE_STORAGE_CLASS: local-path PROVIDER_STRATEGY: k8s KUBE_VOLUME_SIZE: 2Gi containerCpu: '1000' containerMemory: '1024' AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_S3_ENDPOINT: http://localhost:4566 AWS_ENDPOINT: http://localhost:4566 INPUT_AWSS3ENDPOINT: http://localhost:4566 INPUT_AWSENDPOINT: http://localhost:4566 AWS_S3_FORCE_PATH_STYLE: 'true' AWS_EC2_METADATA_DISABLED: 'true' GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-retaining (K8s) if: always() run: | source /tmp/cleanup-functions.sh full_k8s_cleanup # --- K8s teardown --- - name: Delete k3d cluster and final cleanup if: always() run: | k3d cluster delete unity-builder || true docker stop localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true docker system prune -af --volumes || true df -h # ============================================================================ # AWS/LOCALSTACK INTEGRATION TESTS # ============================================================================ aws-integration: name: AWS Integration Tests runs-on: ubuntu-latest env: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_ENDPOINT: http://localhost:4566 AWS_ENDPOINT_URL: http://localhost:4566 steps: - name: Checkout orchestrator (matching branch) uses: actions/checkout@v4 with: repository: game-ci/orchestrator ref: ${{ github.head_ref || github.ref_name }} lfs: false continue-on-error: true id: orch-branch - name: Fallback to orchestrator main branch if: steps.orch-branch.outcome == 'failure' uses: actions/checkout@v4 with: repository: game-ci/orchestrator lfs: false - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: yarn - name: Define cleanup functions run: | cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF' light_cleanup() { echo "--- Light cleanup ---" rm -rf ./orchestrator-cache/* || true docker system prune -f || true df -h } heavy_cleanup() { echo "--- Heavy cleanup ---" rm -rf ./orchestrator-cache/* || true docker system prune -af --volumes || true df -h } CLEANUP_EOF - name: Initial disk space cleanup run: | df -h docker system prune -af --volumes || true df -h - name: Start MiniStack run: | docker run -d \ --name localstack-main \ -p 4566:4566 \ ministackorg/ministack:latest || true MAX_ATTEMPTS=60 for i in $(seq 1 $MAX_ATTEMPTS); do HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "") if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi sleep 2 done - name: Install AWS CLI tools run: | if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi pip install awscli-local || true - name: Create S3 bucket for tests run: | for i in {1..5}; do if command -v awslocal > /dev/null 2>&1; then awslocal s3 mb s3://$AWS_STACK_NAME 2>&1 && break else aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1 && break fi sleep 2 done - run: yarn install --frozen-lockfile - name: Build orchestrator run: | echo "Building orchestrator TypeScript..." yarn build echo "✓ orchestrator build successful" # --- AWS Test 1: orchestrator-image --- - name: Run orchestrator-image test (AWS) timeout-minutes: 10 run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-image (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- AWS Test 2: orchestrator-environment --- - name: Run orchestrator-environment test (AWS) timeout-minutes: 30 run: yarn run test "orchestrator-environment" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-environment (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- AWS Test 3: orchestrator-s3-steps --- - name: Run orchestrator-s3-steps test (AWS) timeout-minutes: 30 run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-s3-steps (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- AWS Test 4: orchestrator-hooks --- - name: Run orchestrator-hooks test (AWS) timeout-minutes: 30 run: yarn run test "orchestrator-hooks" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-hooks (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- AWS Test 5: orchestrator-caching --- - name: Run orchestrator-caching test (AWS) timeout-minutes: 60 run: yarn run test "orchestrator-caching" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-caching (AWS) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- AWS Test 6: orchestrator-locking-core --- - name: Run orchestrator-locking-core test (AWS) timeout-minutes: 60 run: yarn run test "orchestrator-locking-core" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-locking-core (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- AWS Test 7: orchestrator-locking-get-locked --- - name: Run orchestrator-locking-get-locked test (AWS) timeout-minutes: 60 run: yarn run test "orchestrator-locking-get-locked" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-locking-get-locked (AWS) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # End-to-end tests (continue-on-error) — run LAST to prevent # workspace corruption from affecting mandatory tests above. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # --- AWS Test 8: orchestrator-end2end-caching --- - name: Run orchestrator-end2end-caching test (AWS) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-caching (AWS) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- AWS Test 9: orchestrator-end2end-retaining --- - name: Run orchestrator-end2end-retaining test (AWS) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-retaining" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-retaining (AWS) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- AWS Test 10: orchestrator-end2end-locking --- - name: Run orchestrator-end2end-locking test (AWS) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-locking" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneWindows64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: aws GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-locking (AWS) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- Final cleanup --- - name: Final cleanup if: always() run: | docker stop localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true docker system prune -af --volumes || true df -h # ============================================================================ # LOCAL-DOCKER INTEGRATION TESTS # ============================================================================ local-docker-integration: name: Local Docker Integration Tests runs-on: ubuntu-latest env: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_ENDPOINT: http://localhost:4566 AWS_ENDPOINT_URL: http://localhost:4566 steps: - name: Checkout orchestrator (matching branch) uses: actions/checkout@v4 with: repository: game-ci/orchestrator ref: ${{ github.head_ref || github.ref_name }} lfs: false continue-on-error: true id: orch-branch - name: Fallback to orchestrator main branch if: steps.orch-branch.outcome == 'failure' uses: actions/checkout@v4 with: repository: game-ci/orchestrator lfs: false - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: yarn - name: Define cleanup functions run: | cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF' light_cleanup() { echo "--- Light cleanup ---" rm -rf ./orchestrator-cache/* || true docker system prune -f || true df -h } heavy_cleanup() { echo "--- Heavy cleanup ---" rm -rf ./orchestrator-cache/* || true docker system prune -af --volumes || true df -h } CLEANUP_EOF - name: Initial disk space cleanup run: | df -h docker system prune -af --volumes || true df -h - name: Start MiniStack (for S3-dependent tests) run: | docker run -d \ --name localstack-main \ -p 4566:4566 \ ministackorg/ministack:latest || true MAX_ATTEMPTS=60 for i in $(seq 1 $MAX_ATTEMPTS); do HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "") if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi sleep 2 done - name: Install AWS CLI tools run: | if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi pip install awscli-local || true - name: Create S3 bucket for tests run: | for i in {1..5}; do if command -v awslocal > /dev/null 2>&1; then awslocal s3 mb s3://$AWS_STACK_NAME 2>&1 && break else aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1 && break fi sleep 2 done - run: yarn install --frozen-lockfile - name: Build orchestrator run: | yarn build echo "✓ orchestrator build successful" # --- Local Docker Test 1: orchestrator-image --- - name: Run orchestrator-image test (local-docker) timeout-minutes: 10 run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-image (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 2: orchestrator-hooks --- - name: Run orchestrator-hooks test (local-docker) timeout-minutes: 30 run: yarn run test "orchestrator-hooks" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-hooks (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 3: orchestrator-local-persistence --- - name: Run orchestrator-local-persistence test (local-docker) timeout-minutes: 30 run: yarn run test "orchestrator-local-persistence" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-local-persistence (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 4: orchestrator-caching --- - name: Run orchestrator-caching test (local-docker) timeout-minutes: 30 run: yarn run test "orchestrator-caching" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-caching (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 5: orchestrator-github-checks --- - name: Run orchestrator-github-checks test (local-docker) timeout-minutes: 30 run: yarn run test "orchestrator-github-checks" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-github-checks (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 6: orchestrator-locking-core (with S3) --- - name: Run orchestrator-locking-core test (local-docker + S3) timeout-minutes: 30 run: yarn run test "orchestrator-locking-core" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-locking-core (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 7: orchestrator-locking-get-locked (with S3) --- - name: Run orchestrator-locking-get-locked test (local-docker + S3) timeout-minutes: 30 run: yarn run test "orchestrator-locking-get-locked" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-locking-get-locked (local-docker) if: always() run: source /tmp/cleanup-functions.sh && light_cleanup # --- Local Docker Test 8: orchestrator-s3-steps (with S3) --- - name: Run orchestrator-s3-steps test (local-docker + S3) timeout-minutes: 30 run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-s3-steps (local-docker) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- Local Docker Test 9: orchestrator-end2end-caching (with S3) --- - name: Run orchestrator-end2end-caching test (local-docker + S3) timeout-minutes: 60 continue-on-error: true run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} - name: Cleanup after orchestrator-end2end-caching (local-docker) if: always() run: source /tmp/cleanup-functions.sh && heavy_cleanup # --- Final cleanup --- - name: Final cleanup if: always() run: | docker stop localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true docker system prune -af --volumes || true df -h # ============================================================================ # RCLONE INTEGRATION TESTS # ============================================================================ rclone-integration: name: Rclone Integration Tests runs-on: ubuntu-latest env: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_ENDPOINT: http://localhost:4566 AWS_ENDPOINT_URL: http://localhost:4566 steps: - name: Checkout orchestrator (matching branch) uses: actions/checkout@v4 with: repository: game-ci/orchestrator ref: ${{ github.head_ref || github.ref_name }} lfs: false continue-on-error: true id: orch-branch - name: Fallback to orchestrator main branch if: steps.orch-branch.outcome == 'failure' uses: actions/checkout@v4 with: repository: game-ci/orchestrator lfs: false - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: yarn - name: Initial disk space cleanup run: | docker system prune -af --volumes || true df -h - name: Start MiniStack run: | docker run -d \ --name localstack-main \ -p 4566:4566 \ ministackorg/ministack:latest || true MAX_ATTEMPTS=60 for i in $(seq 1 $MAX_ATTEMPTS); do HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "") if echo "$HEALTH" | grep -q '"s3"'; then echo "MiniStack ready ($i/$MAX_ATTEMPTS)"; break; fi sleep 2 done - name: Install rclone run: | curl https://rclone.org/install.sh | sudo bash || true rclone version || echo "rclone not available" - name: Install AWS CLI tools run: | if ! command -v aws > /dev/null 2>&1; then pip install awscli || true; fi pip install awscli-local || true - name: Create S3 bucket for tests run: | for i in {1..5}; do if command -v awslocal > /dev/null 2>&1; then awslocal s3 mb s3://$AWS_STACK_NAME 2>&1 && break else aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1 && break fi sleep 2 done - run: yarn install --frozen-lockfile - name: Build orchestrator run: | yarn build echo "✓ orchestrator build successful" # --- Rclone Test --- - name: Run orchestrator-rclone-steps test timeout-minutes: 30 run: yarn run test "orchestrator-rclone-steps" --detectOpenHandles --forceExit --runInBand env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} TARGET_PLATFORM: StandaloneLinux64 orchestratorTests: true versioning: None PROVIDER_STRATEGY: local-docker GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} # --- Final cleanup --- - name: Final cleanup if: always() run: | docker stop localstack-main 2>/dev/null || true docker rm localstack-main 2>/dev/null || true docker system prune -af --volumes || true df -h