name: Validate Orchestrator Integration

on:
  workflow_dispatch:
  push:
    branches: [main, 'release/**', 'feature/**', 'refactor/**']
    paths:
      - 'src/model/orchestrator-plugin.ts'
      - 'src/model/build-parameters.ts'
      - 'src/model/input.ts'
      - 'src/model/github.ts'
      - 'src/types/game-ci-orchestrator.d.ts'
      - 'action.yml'
      - '.github/workflows/validate-orchestrator-integration.yml'
  pull_request:
    branches: [main, 'release/**']
    paths:
      - 'src/model/orchestrator-plugin.ts'
      - 'src/model/build-parameters.ts'
      - 'src/model/input.ts'
      - 'src/model/github.ts'
      - 'src/types/game-ci-orchestrator.d.ts'
      - 'action.yml'
      - '.github/workflows/validate-orchestrator-integration.yml'

permissions:
  contents: read
  checks: write
  statuses: write

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  AWS_STACK_NAME: game-ci-team-pipelines
  DEBUG: true
  PROJECT_PATH: test-project
  USE_IL2CPP: false

# ==============================================================================
# Cross-repo integration testing
# ==============================================================================
# Validates that the orchestrator package works correctly when installed as a
# plugin into unity-builder. Each job runs on its own runner with a fresh 14GB
# disk to avoid disk exhaustion (a known issue with localstack + k3d).
#
# Job groups:
#   plugin-interface  - Unit tests + installed plugin smoke tests (no infra)
#   k8s-integration   - k3d cluster + LocalStack, 3 representative tests
#   aws-integration   - LocalStack only (no k3d), 3 representative tests
# ==============================================================================

jobs:
  # ============================================================================
  # PLUGIN INTERFACE VALIDATION
  # ============================================================================
  plugin-interface:
    name: Plugin Interface Tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout unity-builder
        uses: actions/checkout@v4

      - name: Checkout orchestrator
        uses: actions/checkout@v4
        with:
          repository: game-ci/orchestrator
          path: orchestrator-standalone

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: yarn

      - name: Install unity-builder dependencies
        run: yarn install --frozen-lockfile

      - name: Build unity-builder
        run: |
          echo "Building unity-builder TypeScript..."
          npx tsc
          echo "✓ unity-builder compiles successfully"

      - name: Run plugin interface unit tests
        run: |
          echo "Running orchestrator-plugin unit tests..."
          npx jest orchestrator-plugin --verbose --detectOpenHandles --forceExit

      - name: Verify plugin loader returns undefined without orchestrator
        run: |
          echo "Checking plugin loader handles missing @game-ci/orchestrator..."
          node -e "
            const { loadOrchestrator, loadEnterpriseServices } = require('./lib/model/orchestrator-plugin');
            (async () => {
              const orch = await loadOrchestrator();
              if (orch !== undefined) {
                console.error('ERROR: loadOrchestrator should return undefined when package not installed');
                process.exit(1);
              }
              console.log('✓ loadOrchestrator() returns undefined when package not installed');

              const services = await loadEnterpriseServices();
              if (services !== undefined) {
                console.error('ERROR: loadEnterpriseServices should return undefined when package not installed');
                process.exit(1);
              }
              console.log('✓ loadEnterpriseServices() returns undefined when package not installed');
            })();
          "

      - name: Build and pack orchestrator
        working-directory: orchestrator-standalone
        run: |
          yarn install --frozen-lockfile
          echo "Building orchestrator..."
          npx tsc
          echo "✓ orchestrator compiles successfully"
          echo "Packing orchestrator as tarball..."
          npm pack

      - name: Install orchestrator into unity-builder
        run: |
          echo "Installing orchestrator into unity-builder workspace..."
          npm install ./orchestrator-standalone/game-ci-orchestrator-*.tgz --no-save --legacy-peer-deps

      - name: Verify plugin loader returns exports with orchestrator installed
        run: |
          echo "Checking plugin loader returns defined exports..."
          node -e "
            const { loadOrchestrator, loadEnterpriseServices } = require('./lib/model/orchestrator-plugin');
            (async () => {
              const orch = await loadOrchestrator();
              if (orch === undefined) {
                console.error('ERROR: loadOrchestrator should return defined exports when package is installed');
                process.exit(1);
              }
              if (typeof orch.run !== 'function') {
                console.error('ERROR: loadOrchestrator().run should be a function');
                process.exit(1);
              }
              console.log('✓ loadOrchestrator() returns defined exports with orchestrator installed');

              const services = await loadEnterpriseServices();
              if (services === undefined) {
                console.error('ERROR: loadEnterpriseServices should return defined exports when package is installed');
                process.exit(1);
              }
              const expectedServices = [
                'BuildReliabilityService', 'TestWorkflowService', 'HotRunnerService',
                'OutputService', 'OutputTypeRegistry', 'ArtifactUploadHandler',
                'IncrementalSyncService',
              ];
              for (const svc of expectedServices) {
                if (services[svc] === undefined) {
                  console.error('ERROR: ' + svc + ' should be defined');
                  process.exit(1);
                }
              }
              console.log('✓ loadEnterpriseServices() returns all ' + expectedServices.length + ' services');

              const lazyLoaders = [
                'loadChildWorkspaceService', 'loadLocalCacheService',
                'loadSubmoduleProfileService', 'loadLfsAgentService', 'loadGitHooksService',
              ];
              for (const loader of lazyLoaders) {
                if (typeof services[loader] !== 'function') {
                  console.error('ERROR: ' + loader + ' should be a function');
                  process.exit(1);
                }
                const loaded = await services[loader]();
                if (loaded === undefined) {
                  console.error('ERROR: ' + loader + '() should return defined service');
                  process.exit(1);
                }
              }
              console.log('✓ All ' + lazyLoaders.length + ' lazy loaders return defined services');
            })();
          "

      - name: Verify type declarations match orchestrator exports
        run: |
          echo "Checking type declarations align with orchestrator exports..."
          node -e "
            const orch = require('@game-ci/orchestrator');
            const expectedExports = [
              'Orchestrator', 'BuildReliabilityService', 'TestWorkflowService',
              'HotRunnerService', 'OutputService', 'OutputTypeRegistry',
              'ArtifactUploadHandler', 'IncrementalSyncService',
              'ChildWorkspaceService', 'LocalCacheService', 'SubmoduleProfileService',
              'LfsAgentService', 'GitHooksService',
            ];
            const missing = expectedExports.filter(e => orch[e] === undefined);
            if (missing.length > 0) {
              console.error('ERROR: Missing exports from @game-ci/orchestrator:', missing.join(', '));
              process.exit(1);
            }
            console.log('✓ All ' + expectedExports.length + ' declared exports present in orchestrator package');
          "

  # ============================================================================
  # K8S INTEGRATION TESTS (k3d + LocalStack)
  # ============================================================================
  k8s-integration:
    name: K8s Integration Tests
    runs-on: ubuntu-latest
    env:
      K3D_NODE_CONTAINERS: 'k3d-unity-builder-agent-0'
      AWS_FORCE_PROVIDER: aws-local
      RESOURCE_TRACKING: 'true'
      K8S_LOCALSTACK_HOST: localstack-main
    steps:
      - name: Checkout orchestrator
        uses: actions/checkout@v4
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: yarn

      - name: Set up kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: 'v1.34.1'

      - name: Install k3d
        run: |
          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
          k3d version | cat

      - name: Define cleanup functions
        run: |
          cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF'
          light_cleanup() {
            echo "--- Light cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -f || true
            df -h
          }

          k8s_resource_cleanup() {
            echo "--- K8s resource cleanup ---"
            kubectl delete jobs --all --ignore-not-found=true -n default || true
            kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
              kubectl delete "$pod" --ignore-not-found=true || true
            done || true
            kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
              kubectl delete "$pvc" --ignore-not-found=true || true
            done || true
            kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
              kubectl delete "$secret" --ignore-not-found=true || true
            done || true
          }

          k3d_node_cleanup() {
            echo "--- K3d node image cleanup (preserving Unity images) ---"
            K3D_NODE_CONTAINERS="${K3D_NODE_CONTAINERS:-k3d-unity-builder-agent-0 k3d-unity-builder-server-0}"
            for NODE in $K3D_NODE_CONTAINERS; do
              docker exec "$NODE" sh -c "crictl rm --all 2>/dev/null || true" || true
              docker exec "$NODE" sh -c "for img in \$(crictl images -q 2>/dev/null); do repo=\$(crictl inspecti \$img --format '{{.repo}}' 2>/dev/null || echo ''); if echo \"\$repo\" | grep -qvE 'unityci/editor|unity'; then crictl rmi \$img 2>/dev/null || true; fi; done" || true
              docker exec "$NODE" sh -c "crictl rmi --prune 2>/dev/null || true" || true
            done || true
          }

          full_k8s_cleanup() {
            k8s_resource_cleanup
            k3d_node_cleanup
            light_cleanup
          }
          CLEANUP_EOF
          echo "Cleanup functions defined at /tmp/cleanup-functions.sh"

      - name: Initial disk space cleanup
        run: |
          echo "Initial disk space cleanup..."
          df -h
          k3d cluster delete unity-builder || true
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          docker network rm orchestrator-net 2>/dev/null || true
          docker network create orchestrator-net || true
          echo "Disk usage after cleanup:"
          df -h

      - name: Start LocalStack
        run: |
          echo "Starting LocalStack..."
          HOST_IP=$(ip route | grep default | awk '{print $3}')
          echo "Host gateway IP: $HOST_IP"
          docker run -d \
            --name localstack-main \
            --network orchestrator-net \
            --add-host=host.docker.internal:host-gateway \
            -p 4566:4566 \
            -e SERVICES=s3,cloudformation,ecs,kinesis,cloudwatch,logs,efs,ec2,iam,elasticfilesystem,secretsmanager,lambda,events,sts \
            -e DEBUG=0 \
            -e HOSTNAME_EXTERNAL=localstack-main \
            localstack/localstack:latest || true
          echo "Waiting for LocalStack to be ready..."
          MAX_ATTEMPTS=60
          READY=false
          for i in $(seq 1 $MAX_ATTEMPTS); do
            if ! docker ps | grep -q localstack-main; then
              echo "LocalStack container not running (attempt $i/$MAX_ATTEMPTS)"
              sleep 2
              continue
            fi
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if [ -z "$HEALTH" ] || ! echo "$HEALTH" | grep -q "services"; then
              echo "LocalStack health endpoint not ready (attempt $i/$MAX_ATTEMPTS)"
              sleep 2
              continue
            fi
            if echo "$HEALTH" | grep -q '"s3"'; then
              echo "LocalStack is ready with S3 service (attempt $i/$MAX_ATTEMPTS)"
              READY=true
              break
            fi
            echo "Waiting for LocalStack S3 service... ($i/$MAX_ATTEMPTS)"
            sleep 2
          done
          if [ "$READY" != "true" ]; then
            echo "ERROR: LocalStack did not become ready after $MAX_ATTEMPTS attempts"
            docker ps -a | grep localstack || echo "No LocalStack container found"
            docker logs localstack-main --tail 100 || true
            exit 1
          fi

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then
            pip install awscli || true
          fi
          pip install awscli-local || true
          aws --version || echo "AWS CLI not available"

      - name: Create S3 bucket for tests
        run: |
          echo "Verifying LocalStack connectivity..."
          for i in {1..10}; do
            if curl -s http://localhost:4566/_localstack/health > /dev/null 2>&1; then
              echo "LocalStack is accessible"
              break
            fi
            echo "Waiting for LocalStack... ($i/10)"
            sleep 1
          done
          MAX_RETRIES=5
          RETRY_COUNT=0
          BUCKET_CREATED=false
          while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$BUCKET_CREATED" != "true" ]; do
            RETRY_COUNT=$((RETRY_COUNT + 1))
            echo "Attempting to create S3 bucket (attempt $RETRY_COUNT/$MAX_RETRIES)..."
            if command -v awslocal > /dev/null 2>&1; then
              if awslocal s3 mb s3://$AWS_STACK_NAME 2>&1; then
                echo "Bucket created successfully with awslocal"
                BUCKET_CREATED=true
              else
                echo "Bucket creation failed, will retry..."
                sleep 2
              fi
            elif command -v aws > /dev/null 2>&1; then
              if aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1; then
                echo "Bucket created successfully with aws CLI"
                BUCKET_CREATED=true
              else
                echo "Bucket creation failed, will retry..."
                sleep 2
              fi
            else
              echo "Neither awslocal nor aws CLI available"
              exit 1
            fi
          done
          if [ "$BUCKET_CREATED" != "true" ]; then
            echo "ERROR: Failed to create S3 bucket after $MAX_RETRIES attempts"
            docker logs localstack-main --tail 50 || true
            exit 1
          fi

      - run: yarn install --frozen-lockfile

      - name: Build orchestrator
        run: |
          echo "Building orchestrator TypeScript..."
          yarn build
          echo "✓ orchestrator build successful"

      # --- Fast unit tests (fast-fail gate before heavy infra tests) ---
      - name: Run orchestrator unit tests (fast, no infra)
        timeout-minutes: 2
        run: >-
          yarn run test
          --testPathPattern="orchestrator-guid|orchestrator-folders|task-parameter-serializer|follow-log-stream-service|runner-availability-service|provider-url-parser|provider-loader|provider-git-manager|orchestrator-image|orchestrator-hooks|orchestrator-github-checks"
          --verbose --detectOpenHandles --forceExit --runInBand

      # --- K8s cluster setup ---
      - name: Clean up disk space before K8s tests
        run: |
          echo "Cleaning up disk space before K8s tests..."
          rm -rf ./orchestrator-cache/* || true
          sudo apt-get clean || true
          docker system prune -f || true
          df -h

      - name: Create k3s cluster (k3d)
        timeout-minutes: 5
        run: |
          LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "")
          echo "LocalStack container IP: $LOCALSTACK_IP"
          k3d cluster create unity-builder \
            --agents 1 \
            --network orchestrator-net \
            --wait
          kubectl config current-context | cat
          echo "LOCALSTACK_IP=$LOCALSTACK_IP" >> $GITHUB_ENV

      - name: Verify cluster readiness and LocalStack connectivity
        timeout-minutes: 2
        run: |
          for i in {1..60}; do
            if kubectl get nodes 2>/dev/null | grep -q Ready; then
              echo "Cluster is ready"
              break
            fi
            echo "Waiting for cluster... ($i/60)"
            sleep 5
          done
          kubectl get nodes
          kubectl get storageclass
          LOCALSTACK_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' localstack-main 2>/dev/null || echo "")
          echo "LocalStack container IP: $LOCALSTACK_IP"
          echo "Testing LocalStack connectivity from k3d cluster..."
          curl -s --max-time 5 http://localhost:4566/_localstack/health | head -5 || echo "Host connectivity failed"
          docker run --rm --network orchestrator-net curlimages/curl \
            curl -s --max-time 5 http://localstack-main:4566/_localstack/health 2>&1 | head -5 || echo "Container network test failed"
          kubectl run test-localstack --image=curlimages/curl --rm -i --restart=Never --timeout=30s -- \
            curl -v --max-time 10 http://${LOCALSTACK_IP}:4566/_localstack/health 2>&1 | head -30 || \
            echo "Cluster connectivity test - if this fails, LocalStack may not be accessible from k3d"

      - name: Clean up K8s resources before tests
        run: |
          source /tmp/cleanup-functions.sh
          k8s_resource_cleanup
          for i in {1..30}; do
            PVC_COUNT=$(kubectl get pvc -n default 2>/dev/null | grep "unity-builder-pvc-" | wc -l || echo "0")
            if [ "$PVC_COUNT" -eq 0 ]; then
              echo "All PVCs deleted"
              break
            fi
            echo "Waiting for PVCs to be deleted... ($i/30) - Found $PVC_COUNT PVCs"
            sleep 1
          done
          kubectl get pv 2>/dev/null | grep -E "(Released|Failed)" | awk '{print $1}' | while read pv; do
            if [ -n "$pv" ] && [ "$pv" != "NAME" ]; then
              kubectl delete pv "$pv" --ignore-not-found=true || true
            fi
          done || true
          sleep 3
          docker system prune -f || true

      # --- K8s Test 1: orchestrator-image ---
      - name: Run orchestrator-image test (K8s)
        timeout-minutes: 10
        run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '512'
          containerMemory: '512'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-image (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 2: orchestrator-kubernetes ---
      - name: Run orchestrator-kubernetes test
        timeout-minutes: 30
        run: yarn run test "orchestrator-kubernetes" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-kubernetes
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s Test 3: orchestrator-s3-steps ---
      - name: Run orchestrator-s3-steps test (K8s)
        timeout-minutes: 30
        run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneLinux64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: k8s
          KUBE_VOLUME_SIZE: 2Gi
          containerCpu: '1000'
          containerMemory: '1024'
          AWS_ACCESS_KEY_ID: test
          AWS_SECRET_ACCESS_KEY: test
          AWS_S3_ENDPOINT: http://localhost:4566
          AWS_ENDPOINT: http://localhost:4566
          INPUT_AWSS3ENDPOINT: http://localhost:4566
          INPUT_AWSENDPOINT: http://localhost:4566
          AWS_S3_FORCE_PATH_STYLE: 'true'
          AWS_EC2_METADATA_DISABLED: 'true'
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-s3-steps (K8s)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          full_k8s_cleanup

      # --- K8s teardown ---
      - name: Delete k3d cluster and final cleanup
        if: always()
        run: |
          echo "Deleting k3d cluster..."
          k3d cluster delete unity-builder || true
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          echo "Final disk usage:"
          df -h

  # ============================================================================
  # AWS/LOCALSTACK INTEGRATION TESTS
  # ============================================================================
  aws-integration:
    name: AWS Integration Tests
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ENDPOINT_URL: http://localhost:4566
    steps:
      - name: Checkout orchestrator
        uses: actions/checkout@v4
        with:
          repository: game-ci/orchestrator
          lfs: false

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: yarn

      - name: Define cleanup functions
        run: |
          cat > /tmp/cleanup-functions.sh << 'CLEANUP_EOF'
          light_cleanup() {
            echo "--- Light cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -f || true
            df -h
          }

          heavy_cleanup() {
            echo "--- Heavy cleanup ---"
            rm -rf ./orchestrator-cache/* || true
            docker system prune -af --volumes || true
            df -h
          }
          CLEANUP_EOF
          echo "Cleanup functions defined at /tmp/cleanup-functions.sh"

      - name: Initial disk space cleanup
        run: |
          echo "Initial disk space cleanup..."
          df -h
          docker system prune -af --volumes || true
          echo "Disk usage after cleanup:"
          df -h

      - name: Start LocalStack
        run: |
          echo "Starting LocalStack..."
          docker run -d \
            --name localstack-main \
            -p 4566:4566 \
            -e SERVICES=s3,cloudformation,ecs,kinesis,cloudwatch,logs,efs,ec2,iam,elasticfilesystem,secretsmanager,lambda,events,sts \
            -e DEBUG=0 \
            localstack/localstack:latest || true
          echo "Waiting for LocalStack to be ready..."
          MAX_ATTEMPTS=60
          READY=false
          for i in $(seq 1 $MAX_ATTEMPTS); do
            if ! docker ps | grep -q localstack-main; then
              echo "LocalStack container not running (attempt $i/$MAX_ATTEMPTS)"
              sleep 2
              continue
            fi
            HEALTH=$(curl -s http://localhost:4566/_localstack/health 2>/dev/null || echo "")
            if [ -z "$HEALTH" ] || ! echo "$HEALTH" | grep -q "services"; then
              sleep 2
              continue
            fi
            if echo "$HEALTH" | grep -q '"s3"'; then
              echo "LocalStack is ready with S3 service (attempt $i/$MAX_ATTEMPTS)"
              READY=true
              break
            fi
            sleep 2
          done
          if [ "$READY" != "true" ]; then
            echo "ERROR: LocalStack did not become ready"
            docker logs localstack-main --tail 100 || true
            exit 1
          fi

      - name: Install AWS CLI tools
        run: |
          if ! command -v aws > /dev/null 2>&1; then
            pip install awscli || true
          fi
          pip install awscli-local || true

      - name: Create S3 bucket for tests
        run: |
          for i in {1..10}; do
            if curl -s http://localhost:4566/_localstack/health > /dev/null 2>&1; then break; fi
            sleep 1
          done
          MAX_RETRIES=5
          RETRY_COUNT=0
          BUCKET_CREATED=false
          while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$BUCKET_CREATED" != "true" ]; do
            RETRY_COUNT=$((RETRY_COUNT + 1))
            if command -v awslocal > /dev/null 2>&1; then
              if awslocal s3 mb s3://$AWS_STACK_NAME 2>&1; then BUCKET_CREATED=true; else sleep 2; fi
            elif command -v aws > /dev/null 2>&1; then
              if aws --endpoint-url=http://localhost:4566 s3 mb s3://$AWS_STACK_NAME 2>&1; then BUCKET_CREATED=true; else sleep 2; fi
            else
              echo "Neither awslocal nor aws CLI available"; exit 1
            fi
          done
          if [ "$BUCKET_CREATED" != "true" ]; then
            echo "ERROR: Failed to create S3 bucket"
            docker logs localstack-main --tail 50 || true
            exit 1
          fi

      - run: yarn install --frozen-lockfile

      - name: Build orchestrator
        run: |
          echo "Building orchestrator TypeScript..."
          yarn build
          echo "✓ orchestrator build successful"

      # --- AWS Test 1: orchestrator-image ---
      - name: Run orchestrator-image test (AWS)
        timeout-minutes: 10
        run: yarn run test "orchestrator-image" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-image (AWS)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          light_cleanup

      # --- AWS Test 2: orchestrator-s3-steps ---
      - name: Run orchestrator-s3-steps test (AWS)
        timeout-minutes: 30
        run: yarn run test "orchestrator-s3-steps" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-s3-steps (AWS)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          light_cleanup

      # --- AWS Test 3: orchestrator-end2end-caching ---
      - name: Run orchestrator-end2end-caching test (AWS)
        timeout-minutes: 60
        run: yarn run test "orchestrator-end2end-caching" --detectOpenHandles --forceExit --runInBand
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
          TARGET_PLATFORM: StandaloneWindows64
          orchestratorTests: true
          versioning: None
          KUBE_STORAGE_CLASS: local-path
          PROVIDER_STRATEGY: aws
          GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
      - name: Cleanup after orchestrator-end2end-caching (AWS)
        if: always()
        run: |
          source /tmp/cleanup-functions.sh
          light_cleanup

      # --- Final cleanup ---
      - name: Final cleanup
        if: always()
        run: |
          rm -rf ./orchestrator-cache/* || true
          docker stop localstack-main 2>/dev/null || true
          docker rm localstack-main 2>/dev/null || true
          docker system prune -af --volumes || true
          df -h