mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-14 20:16:48 -07:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 327c2c99ee | |||
| 703e4212dd | |||
| 79ae55802d | |||
| 4d7e8717e9 | |||
| 120c3c5b24 | |||
| ff56194b30 | |||
| 12f287168d | |||
| 9d475434d3 | |||
| f3849ee1c9 | |||
| 0c82a58873 | |||
| 1d4ee0697f | |||
| 3a2abf9037 | |||
| cfdebb67c1 | |||
| ab64768ceb | |||
| 00fa0d3772 | |||
| d587557287 | |||
| 6e0bf17345 | |||
| 2822af505e | |||
| 8ec161b981 | |||
| 88a89c94a0 | |||
| f7f3f70c57 |
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"files.autoSave": "on",
|
||||
"files.autoSaveWhen": "on",
|
||||
"files.autoSaveDelay": 1000,
|
||||
|
||||
"editor.formatOnSave": false,
|
||||
"editor.formatOnPaste": false,
|
||||
"editor.formatOnType": false,
|
||||
|
||||
"editor.codeActionsOnSave": {},
|
||||
|
||||
"git.autorefresh": false,
|
||||
"git.confirmSync": false,
|
||||
"git.autofetch": false,
|
||||
|
||||
"editor.defaultFormatter": null
|
||||
}
|
||||
|
||||
+2
-1
@@ -14,7 +14,8 @@
|
||||
"env": {
|
||||
"node": true,
|
||||
"es6": true,
|
||||
"jest/globals": true
|
||||
"jest/globals": true,
|
||||
"es2020": true
|
||||
},
|
||||
"rules": {
|
||||
// Error out for code formatting errors
|
||||
|
||||
@@ -12,6 +12,7 @@ jobs:
|
||||
buildForAllPlatformsMacOS:
|
||||
name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
|
||||
runs-on: macos-latest
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -24,6 +25,13 @@ jobs:
|
||||
targetPlatform:
|
||||
- StandaloneOSX # Build a MacOS executable
|
||||
- iOS # Build an iOS executable
|
||||
include:
|
||||
# Additionally test enableGpu build for a standalone windows target
|
||||
- unityVersion: 6000.0.36f1
|
||||
targetPlatform: StandaloneOSX
|
||||
- unityVersion: 6000.0.36f1
|
||||
targetPlatform: StandaloneOSX
|
||||
buildProfile: 'Assets/Settings/Build Profiles/Sample macOS Build Profile.asset'
|
||||
|
||||
steps:
|
||||
###########################
|
||||
@@ -65,6 +73,7 @@ jobs:
|
||||
projectPath: ${{ matrix.projectPath }}
|
||||
unityVersion: ${{ matrix.unityVersion }}
|
||||
targetPlatform: ${{ matrix.targetPlatform }}
|
||||
buildProfile: ${{ matrix.buildProfile }}
|
||||
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
||||
# We use dirty build because we are replacing the default project settings file above
|
||||
allowDirtyBuild: true
|
||||
@@ -74,6 +83,6 @@ jobs:
|
||||
###########################
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})
|
||||
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
||||
path: build
|
||||
retention-days: 14
|
||||
|
||||
@@ -34,7 +34,12 @@ jobs:
|
||||
unityVersion: 2023.2.2f1
|
||||
targetPlatform: StandaloneWindows64
|
||||
enableGpu: true
|
||||
|
||||
- unityVersion: 6000.0.36f1
|
||||
targetPlatform: StandaloneWindows64
|
||||
- unityVersion: 6000.0.36f1
|
||||
targetPlatform: StandaloneWindows64
|
||||
buildProfile: 'Assets/Settings/Build Profiles/Sample Windows Build Profile.asset'
|
||||
|
||||
steps:
|
||||
###########################
|
||||
# Checkout #
|
||||
@@ -79,6 +84,7 @@ jobs:
|
||||
projectPath: ${{ matrix.projectPath }}
|
||||
unityVersion: ${{ matrix.unityVersion }}
|
||||
targetPlatform: ${{ matrix.targetPlatform }}
|
||||
buildProfile: ${{ matrix.buildProfile }}
|
||||
enableGpu: ${{ matrix.enableGpu }}
|
||||
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
||||
allowDirtyBuild: true
|
||||
@@ -140,6 +146,6 @@ jobs:
|
||||
###########################
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}
|
||||
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
||||
path: build
|
||||
retention-days: 14
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
name: Cleanup (cron)
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 10 * * SUN' # every sunday at 10:30
|
||||
|
||||
jobs:
|
||||
deleteArtifacts:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete old artifacts
|
||||
uses: kolpav/purge-artifacts-action@v1
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
expire-in: 21 days
|
||||
cleanupCloudRunner:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
if: github.event.event_type != 'pull_request_target'
|
||||
with:
|
||||
lfs: true
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '18'
|
||||
- run: yarn
|
||||
- run: yarn run cli --help
|
||||
env:
|
||||
AWS_REGION: eu-west-2
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: eu-west-2
|
||||
- run: yarn run cli -m list-resources
|
||||
env:
|
||||
AWS_REGION: eu-west-2
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: eu-west-2
|
||||
@@ -1,83 +0,0 @@
|
||||
name: cloud-runner-integrity-localstack
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runGithubIntegrationTests:
|
||||
description: 'Run GitHub Checks integration tests'
|
||||
required: false
|
||||
default: 'false'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
checks: write
|
||||
contents: read
|
||||
actions: write
|
||||
packages: read
|
||||
pull-requests: write
|
||||
statuses: write
|
||||
id-token: write
|
||||
|
||||
env:
|
||||
AWS_REGION: us-east-1
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
AWS_STACK_NAME: game-ci-local
|
||||
AWS_ENDPOINT: http://localhost:4566
|
||||
AWS_ENDPOINT_URL: http://localhost:4566
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||
DEBUG: true
|
||||
PROJECT_PATH: test-project
|
||||
USE_IL2CPP: false
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Cloud Runner Tests (LocalStack)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
localstack:
|
||||
image: localstack/localstack
|
||||
ports:
|
||||
- 4566:4566
|
||||
env:
|
||||
SERVICES: cloudformation,ecs,kinesis,cloudwatch,s3,logs
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test:
|
||||
- 'cloud-runner-end2end-locking'
|
||||
- 'cloud-runner-end2end-caching'
|
||||
- 'cloud-runner-end2end-retaining'
|
||||
- 'cloud-runner-caching'
|
||||
- 'cloud-runner-environment'
|
||||
- 'cloud-runner-image'
|
||||
- 'cloud-runner-hooks'
|
||||
- 'cloud-runner-local-persistence'
|
||||
- 'cloud-runner-locking-core'
|
||||
- 'cloud-runner-locking-get-locked'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
lfs: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'yarn'
|
||||
- run: yarn install --frozen-lockfile
|
||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||
PROJECT_PATH: test-project
|
||||
TARGET_PLATFORM: StandaloneWindows64
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
KUBE_STORAGE_CLASS: local-path
|
||||
PROVIDER_STRATEGY: aws
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
@@ -1,196 +0,0 @@
|
||||
name: cloud-runner-integrity
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runGithubIntegrationTests:
|
||||
description: 'Run GitHub Checks integration tests'
|
||||
required: false
|
||||
default: 'false'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
checks: write
|
||||
contents: read
|
||||
actions: write
|
||||
packages: read
|
||||
pull-requests: write
|
||||
statuses: write
|
||||
id-token: write
|
||||
|
||||
env:
|
||||
AWS_REGION: eu-west-2
|
||||
AWS_DEFAULT_REGION: eu-west-2
|
||||
AWS_STACK_NAME: game-ci-team-pipelines
|
||||
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||
DEBUG: true
|
||||
PROJECT_PATH: test-project
|
||||
USE_IL2CPP: false
|
||||
|
||||
jobs:
|
||||
k8s:
|
||||
name: Cloud Runner Tests (K8s)
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# K8s runs (k3s)
|
||||
- test: 'cloud-runner-end2end-caching'
|
||||
provider: k8s
|
||||
- test: 'cloud-runner-end2end-retaining'
|
||||
provider: k8s
|
||||
- test: 'cloud-runner-hooks'
|
||||
provider: k8s
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
lfs: false
|
||||
# Set up Kubernetes (k3s via k3d) only for k8s matrix entries
|
||||
- name: Set up kubectl
|
||||
if: ${{ matrix.provider == 'k8s' }}
|
||||
uses: azure/setup-kubectl@v4
|
||||
with:
|
||||
version: 'v1.29.0'
|
||||
- name: Install k3d
|
||||
if: ${{ matrix.provider == 'k8s' }}
|
||||
run: |
|
||||
curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
|
||||
k3d version | cat
|
||||
- name: Create k3s cluster (k3d)
|
||||
if: ${{ matrix.provider == 'k8s' }}
|
||||
run: |
|
||||
k3d cluster create unity-builder --agents 1 --wait
|
||||
kubectl config current-context | cat
|
||||
- name: Verify cluster readiness
|
||||
if: ${{ matrix.provider == 'k8s' }}
|
||||
run: |
|
||||
for i in {1..60}; do kubectl get nodes && break || sleep 5; done
|
||||
kubectl get storageclass
|
||||
- name: Start LocalStack (S3)
|
||||
uses: localstack/setup-localstack@v0.2.3
|
||||
with:
|
||||
install-awslocal: true
|
||||
- name: Create S3 bucket for tests (host LocalStack)
|
||||
run: |
|
||||
awslocal s3 mb s3://$AWS_STACK_NAME || true
|
||||
awslocal s3 ls
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'yarn'
|
||||
- run: yarn install --frozen-lockfile
|
||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||
PROJECT_PATH: test-project
|
||||
TARGET_PLATFORM: StandaloneWindows64
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
KUBE_STORAGE_CLASS: ${{ matrix.provider == 'k8s' && 'local-path' || '' }}
|
||||
PROVIDER_STRATEGY: ${{ matrix.provider }}
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
AWS_S3_ENDPOINT: http://localhost:4566
|
||||
AWS_ENDPOINT: http://localhost:4566
|
||||
INPUT_AWSS3ENDPOINT: http://localhost:4566
|
||||
INPUT_AWSENDPOINT: http://localhost:4566
|
||||
AWS_S3_FORCE_PATH_STYLE: 'true'
|
||||
AWS_EC2_METADATA_DISABLED: 'true'
|
||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
localstack:
|
||||
name: Cloud Runner Tests (LocalStack)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
localstack:
|
||||
image: localstack/localstack
|
||||
ports:
|
||||
- 4566:4566
|
||||
env:
|
||||
SERVICES: cloudformation,ecs,kinesis,cloudwatch,s3,logs
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test:
|
||||
- 'cloud-runner-end2end-locking'
|
||||
- 'cloud-runner-end2end-caching'
|
||||
- 'cloud-runner-end2end-retaining'
|
||||
- 'cloud-runner-caching'
|
||||
- 'cloud-runner-environment'
|
||||
- 'cloud-runner-image'
|
||||
- 'cloud-runner-hooks'
|
||||
- 'cloud-runner-local-persistence'
|
||||
- 'cloud-runner-locking-core'
|
||||
- 'cloud-runner-locking-get-locked'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
lfs: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'yarn'
|
||||
- run: yarn install --frozen-lockfile
|
||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||
PROJECT_PATH: test-project
|
||||
TARGET_PLATFORM: StandaloneWindows64
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
KUBE_STORAGE_CLASS: local-path
|
||||
PROVIDER_STRATEGY: aws
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
AWS_ENDPOINT: http://localhost:4566
|
||||
AWS_ENDPOINT_URL: http://localhost:4566
|
||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
aws:
|
||||
name: Cloud Runner Tests (AWS)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [k8s, localstack]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test:
|
||||
- 'cloud-runner-end2end-caching'
|
||||
- 'cloud-runner-end2end-retaining'
|
||||
- 'cloud-runner-hooks'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
lfs: false
|
||||
- name: Configure AWS Credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'yarn'
|
||||
- run: yarn install --frozen-lockfile
|
||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||
PROJECT_PATH: test-project
|
||||
TARGET_PLATFORM: StandaloneWindows64
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
PROVIDER_STRATEGY: aws
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||
@@ -4,6 +4,11 @@ on:
|
||||
push: { branches: [main] }
|
||||
pull_request: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
statuses: write
|
||||
|
||||
env:
|
||||
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
|
||||
|
||||
@@ -27,8 +32,7 @@ jobs:
|
||||
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
|
||||
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
|
||||
|
||||
cloud-runner:
|
||||
name: Cloud Runner Integrity
|
||||
uses: ./.github/workflows/cloud-runner-integrity.yml
|
||||
orchestrator:
|
||||
name: Orchestrator Integrity
|
||||
uses: ./.github/workflows/orchestrator-integrity.yml
|
||||
secrets: inherit
|
||||
|
||||
|
||||
+15
-13
@@ -18,15 +18,16 @@ env:
|
||||
GKE_CLUSTER: 'game-ci-github-pipelines'
|
||||
GCP_LOGGING: true
|
||||
GCP_PROJECT: unitykubernetesbuilder
|
||||
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
|
||||
AWS_REGION: eu-west-2
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: eu-west-2
|
||||
AWS_STACK_NAME: game-ci-github-pipelines
|
||||
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||
CLOUD_RUNNER_DEBUG: true
|
||||
CLOUD_RUNNER_DEBUG_TREE: true
|
||||
GCP_LOG_FILE: ${{ github.workspace }}/orchestrator-logs.txt
|
||||
# Commented out: Using LocalStack tests instead of real AWS
|
||||
# AWS_REGION: eu-west-2
|
||||
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
# AWS_DEFAULT_REGION: eu-west-2
|
||||
# AWS_STACK_NAME: game-ci-github-pipelines
|
||||
ORCHESTRATOR_BRANCH: ${{ github.ref }}
|
||||
ORCHESTRATOR_DEBUG: true
|
||||
ORCHESTRATOR_DEBUG_TREE: true
|
||||
DEBUG: true
|
||||
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
||||
PROJECT_PATH: test-project
|
||||
@@ -46,13 +47,14 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TARGET_PLATFORM: StandaloneWindows64
|
||||
cloudRunnerTests: true
|
||||
orchestratorTests: true
|
||||
versioning: None
|
||||
CLOUD_RUNNER_CLUSTER: local-docker
|
||||
AWS_STACK_NAME: game-ci-github-pipelines
|
||||
ORCHESTRATOR_CLUSTER: local-docker
|
||||
# Commented out: Using LocalStack tests instead of real AWS
|
||||
# AWS_STACK_NAME: game-ci-github-pipelines
|
||||
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
|
||||
run: |
|
||||
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
|
||||
git clone -b main https://github.com/game-ci/unity-builder
|
||||
cd unity-builder
|
||||
yarn
|
||||
ls
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,3 +5,5 @@ lib/
|
||||
.vsconfig
|
||||
yarn-error.log
|
||||
.orig
|
||||
$LOG_FILE
|
||||
temp/
|
||||
|
||||
+96
-22
@@ -104,11 +104,11 @@ inputs:
|
||||
gitPrivateToken:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[CloudRunner] Github private token to pull from github'
|
||||
description: '[Orchestrator] Github private token to pull from github'
|
||||
githubOwner:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[CloudRunner] GitHub owner name or organization/team name'
|
||||
description: '[Orchestrator] GitHub owner name or organization/team name'
|
||||
runAsHostUser:
|
||||
required: false
|
||||
default: 'false'
|
||||
@@ -149,97 +149,101 @@ inputs:
|
||||
allowDirtyBuild:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.'
|
||||
description: '[Orchestrator] Allows the branch of the build to be dirty, and still generate the build.'
|
||||
postBuildSteps:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array),
|
||||
'[Orchestrator] run a post build job in yaml format with the keys image, secrets (name, value object array),
|
||||
command string'
|
||||
preBuildSteps:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
|
||||
'[Orchestrator] Run a pre build job after the repository setup but before the build job (in yaml format with the
|
||||
keys image, secrets (name, value object array), command line string)'
|
||||
containerHookFiles:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must
|
||||
'[Orchestrator] Specify the names (by file name) of custom steps to run before or after orchestrator jobs, must
|
||||
match a yaml step file inside your repo in the folder .game-ci/steps/'
|
||||
customHookFiles:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
|
||||
'[Orchestrator] Specify the names (by file name) of custom hooks to run before or after orchestrator jobs, must
|
||||
match a yaml step file inside your repo in the folder .game-ci/hooks/'
|
||||
customCommandHooks:
|
||||
required: false
|
||||
default: ''
|
||||
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
|
||||
description: '[Orchestrator] Specify custom commands and trigger hooks (injects commands into jobs)'
|
||||
customJob:
|
||||
required: false
|
||||
default: ''
|
||||
description:
|
||||
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
|
||||
keys image, secrets (name, value object array), command line string)'
|
||||
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
|
||||
the keys image, secrets (name, value object array), command line string)'
|
||||
awsStackName:
|
||||
default: 'game-ci'
|
||||
required: false
|
||||
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
|
||||
description: '[Orchestrator] The Cloud Formation stack name that must be setup before using this option.'
|
||||
providerStrategy:
|
||||
default: 'local'
|
||||
required: false
|
||||
description:
|
||||
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
|
||||
'[Orchestrator] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
|
||||
be configured.'
|
||||
resourceTracking:
|
||||
default: 'false'
|
||||
required: false
|
||||
description: '[Orchestrator] Enable resource tracking logs for disk usage and allocation summaries.'
|
||||
containerCpu:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[CloudRunner] Amount of CPU time to assign the remote build container'
|
||||
description: '[Orchestrator] Amount of CPU time to assign the remote build container'
|
||||
containerMemory:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[CloudRunner] Amount of memory to assign the remote build container'
|
||||
description: '[Orchestrator] Amount of memory to assign the remote build container'
|
||||
readInputFromOverrideList:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"'
|
||||
description: '[Orchestrator] Comma separated list of input value names to read from "input override command"'
|
||||
readInputOverrideCommand:
|
||||
default: ''
|
||||
required: false
|
||||
description:
|
||||
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
|
||||
'[Orchestrator] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
|
||||
provider secret managers'
|
||||
kubeConfig:
|
||||
default: ''
|
||||
required: false
|
||||
description:
|
||||
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
|
||||
'[Orchestrator] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
|
||||
completion.'
|
||||
kubeVolume:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.'
|
||||
description: '[Orchestrator] Supply a Persistent Volume Claim name to use for the Unity build.'
|
||||
kubeStorageClass:
|
||||
default: ''
|
||||
required: false
|
||||
description:
|
||||
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
|
||||
'[Orchestrator] Kubernetes storage class to use for orchestrator jobs, leave empty to install rook cluster.'
|
||||
kubeVolumeSize:
|
||||
default: '5Gi'
|
||||
required: false
|
||||
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume'
|
||||
description: '[Orchestrator] Amount of disc space to assign the Kubernetes Persistent Volume'
|
||||
cacheKey:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[CloudRunner] Cache key to indicate bucket for cache'
|
||||
description: '[Orchestrator] Cache key to indicate bucket for cache'
|
||||
watchToEnd:
|
||||
default: 'true'
|
||||
required: false
|
||||
description:
|
||||
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
|
||||
'[Orchestrator] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
|
||||
imports or self-hosted ephemeral runners.'
|
||||
cacheUnityInstallationOnMac:
|
||||
default: 'false'
|
||||
@@ -265,6 +269,76 @@ inputs:
|
||||
default: 'false'
|
||||
required: false
|
||||
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
|
||||
cloneDepth:
|
||||
default: '50'
|
||||
required: false
|
||||
description: '[Orchestrator] Specifies the depth of the git clone for the repository. Use 0 for full clone.'
|
||||
orchestratorRepoName:
|
||||
default: 'game-ci/unity-builder'
|
||||
required: false
|
||||
description:
|
||||
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
|
||||
fixes.'
|
||||
remotePowershellHost:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Remote PowerShell host (hostname or IP) for the remote-powershell provider'
|
||||
remotePowershellCredential:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Remote PowerShell credential (username:password or certificate path)'
|
||||
remotePowershellTransport:
|
||||
default: 'wsman'
|
||||
required: false
|
||||
description: '[Orchestrator] Remote PowerShell transport protocol (wsman or ssh)'
|
||||
githubActionsRepo:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Target repository (owner/repo) for the github-actions provider'
|
||||
githubActionsWorkflow:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Workflow filename or ID to dispatch for the github-actions provider'
|
||||
githubActionsToken:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] PAT with actions:write scope for the github-actions provider'
|
||||
githubActionsRef:
|
||||
default: 'main'
|
||||
required: false
|
||||
description: '[Orchestrator] Branch/ref to run the workflow on for the github-actions provider'
|
||||
gitlabProjectId:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] GitLab project ID or URL-encoded path for the gitlab-ci provider'
|
||||
gitlabTriggerToken:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Pipeline trigger token for the gitlab-ci provider'
|
||||
gitlabApiUrl:
|
||||
default: 'https://gitlab.com'
|
||||
required: false
|
||||
description: '[Orchestrator] GitLab API URL (for self-hosted instances) for the gitlab-ci provider'
|
||||
gitlabRef:
|
||||
default: 'main'
|
||||
required: false
|
||||
description: '[Orchestrator] Branch/ref to trigger the pipeline on for the gitlab-ci provider'
|
||||
ansibleInventory:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Path to Ansible inventory file or dynamic inventory script'
|
||||
ansiblePlaybook:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Path to Ansible playbook for Unity builds'
|
||||
ansibleExtraVars:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Additional Ansible variables as JSON'
|
||||
ansibleVaultPassword:
|
||||
default: ''
|
||||
required: false
|
||||
description: '[Orchestrator] Path to Ansible vault password file'
|
||||
|
||||
outputs:
|
||||
volume:
|
||||
|
||||
@@ -56,14 +56,19 @@ namespace UnityBuilderAction
|
||||
// of either `UnityEditor.BuildPlayerOptions` or `UnityEditor.BuildPlayerWithProfileOptions`
|
||||
dynamic buildPlayerOptions;
|
||||
|
||||
if (options["customBuildProfile"] != "") {
|
||||
if (options.TryGetValue("activeBuildProfile", out var buildProfilePath)) {
|
||||
if (string.IsNullOrEmpty(buildProfilePath)) {
|
||||
throw new Exception("`-activeBuildProfile` is set but with an empty value; this shouldn't happen");
|
||||
}
|
||||
|
||||
#if UNITY_6000_0_OR_NEWER
|
||||
// Load build profile from Assets folder
|
||||
BuildProfile buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(options["customBuildProfile"]);
|
||||
var buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(buildProfilePath)
|
||||
?? throw new Exception("Build profile file not found at path: " + buildProfilePath);
|
||||
|
||||
// Set it as active
|
||||
BuildProfile.SetActiveBuildProfile(buildProfile);
|
||||
// no need to set active profile, as already set by `-activeBuildProfile` CLI argument
|
||||
// BuildProfile.SetActiveBuildProfile(buildProfile);
|
||||
Debug.Log($"build profile: {buildProfile.name}");
|
||||
|
||||
// Define BuildPlayerWithProfileOptions
|
||||
buildPlayerOptions = new BuildPlayerWithProfileOptions {
|
||||
@@ -71,12 +76,16 @@ namespace UnityBuilderAction
|
||||
locationPathName = options["customBuildPath"],
|
||||
options = buildOptions,
|
||||
};
|
||||
#else
|
||||
#else // UNITY_6000_0_OR_NEWER
|
||||
throw new Exception("Build profiles are not supported by this version of Unity (" + Application.unityVersion +")");
|
||||
#endif
|
||||
#endif // UNITY_6000_0_OR_NEWER
|
||||
|
||||
} else {
|
||||
|
||||
#if BUILD_PROFILE_LOADED
|
||||
throw new Exception("Build profile's define symbol present; shouldn't happen");
|
||||
#endif // BUILD_PROFILE_LOADED
|
||||
|
||||
// Gather values from project
|
||||
var scenes = EditorBuildSettings.scenes.Where(scene => scene.enabled).Select(s => s.path).ToArray();
|
||||
|
||||
|
||||
+2
@@ -115,6 +115,7 @@ namespace UnityBuilderAction.Input
|
||||
}
|
||||
}
|
||||
|
||||
#if UNITY_6000_0_OR_NEWER
|
||||
private static void SetDebugSymbols(string enumValueName)
|
||||
{
|
||||
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
|
||||
@@ -144,5 +145,6 @@ namespace UnityBuilderAction.Input
|
||||
}
|
||||
levelProp.SetValue(null, enumValue);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -21,6 +21,19 @@ namespace UnityBuilderAction.Input
|
||||
EditorApplication.Exit(110);
|
||||
}
|
||||
|
||||
#if UNITY_6000_0_OR_NEWER
|
||||
var buildProfileSupport = true;
|
||||
#else
|
||||
var buildProfileSupport = false;
|
||||
#endif // UNITY_6000_0_OR_NEWER
|
||||
|
||||
string buildProfile;
|
||||
if (buildProfileSupport && validatedOptions.TryGetValue("activeBuildProfile", out buildProfile)) {
|
||||
if (validatedOptions.ContainsKey("buildTarget")) {
|
||||
Console.WriteLine("Extra argument -buildTarget");
|
||||
EditorApplication.Exit(122);
|
||||
}
|
||||
} else {
|
||||
string buildTarget;
|
||||
if (!validatedOptions.TryGetValue("buildTarget", out buildTarget)) {
|
||||
Console.WriteLine("Missing argument -buildTarget");
|
||||
@@ -31,6 +44,7 @@ namespace UnityBuilderAction.Input
|
||||
Console.WriteLine(buildTarget + " is not a defined " + typeof(BuildTarget).Name);
|
||||
EditorApplication.Exit(121);
|
||||
}
|
||||
}
|
||||
|
||||
string customBuildPath;
|
||||
if (!validatedOptions.TryGetValue("customBuildPath", out customBuildPath)) {
|
||||
|
||||
+30446
-30628
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+102
-95
@@ -6865,76 +6865,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
@deno/shim-deno
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Copyright 2021-2022 the Deno authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
@deno/shim-deno-test
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Copyright 2021-2022 the Deno authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
@fastify/busboy
|
||||
MIT
|
||||
Copyright Brian White. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to
|
||||
deal in the Software without restriction, including without limitation the
|
||||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
IN THE SOFTWARE.
|
||||
|
||||
@kubernetes/client-node
|
||||
Apache-2.0
|
||||
Apache License
|
||||
@@ -17122,6 +17052,31 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
IN THE SOFTWARE.
|
||||
|
||||
|
||||
cross-fetch
|
||||
MIT
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017 Leonardo Quixadá
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
cross-spawn
|
||||
MIT
|
||||
The MIT License (MIT)
|
||||
@@ -17614,6 +17569,56 @@ The above copyright notice and this permission notice shall be included in all c
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
graphql
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Copyright (c) GraphQL Contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
graphql-request
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Jason Kuhrt
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
har-schema
|
||||
ISC
|
||||
Copyright (c) 2015, Ahmad Nassri <ahmad@ahmadnassri.com>
|
||||
@@ -19407,6 +19412,33 @@ The above copyright notice and this permission notice shall be included in all c
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
shell-quote
|
||||
MIT
|
||||
The MIT License
|
||||
|
||||
Copyright (c) 2013 James Halliday (mail@substack.net)
|
||||
|
||||
Permission is hereby granted, free of charge,
|
||||
to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to
|
||||
deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify,
|
||||
merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom
|
||||
the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
|
||||
ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
shelljs
|
||||
BSD-3-Clause
|
||||
Copyright (c) 2012, Artur Adib <arturadib@gmail.com>
|
||||
@@ -19741,31 +19773,6 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
undici
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Copyright (c) Matteo Collina and Undici contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
unity-changeset
|
||||
MIT
|
||||
MIT License
|
||||
|
||||
Vendored
+61
-13
@@ -4,21 +4,69 @@
|
||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||
pushd "$ACTIVATE_LICENSE_PATH"
|
||||
|
||||
echo "Requesting activation"
|
||||
if [[ -n "$UNITY_SERIAL" && -n "$UNITY_EMAIL" && -n "$UNITY_PASSWORD" ]]; then
|
||||
#
|
||||
# SERIAL LICENSE MODE
|
||||
#
|
||||
# This will activate unity, using the serial activation process.
|
||||
#
|
||||
|
||||
# Activate license
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||
-logFile - \
|
||||
-batchmode \
|
||||
-nographics \
|
||||
-quit \
|
||||
-serial "$UNITY_SERIAL" \
|
||||
-username "$UNITY_EMAIL" \
|
||||
-password "$UNITY_PASSWORD" \
|
||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||
echo "Requesting activation"
|
||||
|
||||
# Store the exit code from the verify command
|
||||
UNITY_EXIT_CODE=$?
|
||||
# Activate license
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||
-logFile - \
|
||||
-batchmode \
|
||||
-nographics \
|
||||
-quit \
|
||||
-serial "$UNITY_SERIAL" \
|
||||
-username "$UNITY_EMAIL" \
|
||||
-password "$UNITY_PASSWORD" \
|
||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||
|
||||
# Store the exit code from the verify command
|
||||
UNITY_EXIT_CODE=$?
|
||||
|
||||
elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
||||
#
|
||||
# Custom Unity License Server
|
||||
#
|
||||
echo "Adding licensing server config"
|
||||
mkdir -p "$UNITY_LICENSE_PATH/config/"
|
||||
cp "$ACTION_FOLDER/unity-config/services-config.json" "$UNITY_LICENSE_PATH/config/services-config.json"
|
||||
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
||||
--acquire-floating > license.txt
|
||||
|
||||
# Store the exit code from the verify command
|
||||
UNITY_EXIT_CODE=$?
|
||||
|
||||
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
||||
PARSEDFILE=$(grep -oE '\"[^"]*\"' < license.txt | tr -d '"')
|
||||
export FLOATING_LICENSE
|
||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||
|
||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||
fi
|
||||
else
|
||||
#
|
||||
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
||||
#
|
||||
# This will exit since no activation strategies could be matched.
|
||||
#
|
||||
echo "License activation strategy could not be determined."
|
||||
echo ""
|
||||
echo "Visit https://game.ci/docs/github/activation for more"
|
||||
echo "details on how to set up one of the possible activation strategies."
|
||||
|
||||
echo "::error ::No valid license activation strategy could be determined. Make sure to provide UNITY_EMAIL, UNITY_PASSWORD, and either a UNITY_SERIAL \
|
||||
or UNITY_LICENSE. Otherwise please use UNITY_LICENSING_SERVER. See more info at https://game.ci/docs/github/activation"
|
||||
|
||||
# Immediately exit as no UNITY_EXIT_CODE can be derived.
|
||||
exit 1;
|
||||
|
||||
fi
|
||||
|
||||
#
|
||||
# Display information about the result
|
||||
|
||||
Vendored
+2
-3
@@ -149,14 +149,13 @@ echo ""
|
||||
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
||||
-batchmode \
|
||||
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
|
||||
-username "$UNITY_EMAIL" \
|
||||
-password "$UNITY_PASSWORD" \
|
||||
-customBuildName "$BUILD_NAME" \
|
||||
-projectPath "$UNITY_PROJECT_PATH" \
|
||||
-buildTarget "$BUILD_TARGET" \
|
||||
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET") \
|
||||
-customBuildTarget "$BUILD_TARGET" \
|
||||
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
||||
-customBuildProfile "$BUILD_PROFILE" \
|
||||
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
|
||||
-executeMethod "$BUILD_METHOD" \
|
||||
-buildVersion "$VERSION" \
|
||||
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
||||
|
||||
+23
-9
@@ -4,15 +4,29 @@
|
||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||
pushd "$ACTIVATE_LICENSE_PATH"
|
||||
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||
-logFile - \
|
||||
-batchmode \
|
||||
-nographics \
|
||||
-quit \
|
||||
-username "$UNITY_EMAIL" \
|
||||
-password "$UNITY_PASSWORD" \
|
||||
-returnlicense \
|
||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||
if [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
||||
#
|
||||
# Return any floating license used.
|
||||
#
|
||||
echo "Returning floating license: \"$FLOATING_LICENSE\""
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
||||
--return-floating "$FLOATING_LICENSE"
|
||||
elif [[ -n "$UNITY_SERIAL" ]]; then
|
||||
#
|
||||
# SERIAL LICENSE MODE
|
||||
#
|
||||
# This will return the license that is currently in use.
|
||||
#
|
||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||
-logFile - \
|
||||
-batchmode \
|
||||
-nographics \
|
||||
-quit \
|
||||
-username "$UNITY_EMAIL" \
|
||||
-password "$UNITY_PASSWORD" \
|
||||
-returnlicense \
|
||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||
fi
|
||||
|
||||
# Return to previous working directory
|
||||
popd
|
||||
|
||||
+9
-5
@@ -68,14 +68,18 @@ elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
||||
echo "Adding licensing server config"
|
||||
|
||||
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
|
||||
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
||||
export FLOATING_LICENSE
|
||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||
|
||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||
# Store the exit code from the verify command
|
||||
UNITY_EXIT_CODE=$?
|
||||
|
||||
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
||||
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
||||
export FLOATING_LICENSE
|
||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||
|
||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||
fi
|
||||
else
|
||||
#
|
||||
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
||||
|
||||
Vendored
+2
-1
@@ -125,10 +125,11 @@ unity-editor \
|
||||
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
||||
-customBuildName "$BUILD_NAME" \
|
||||
-projectPath "$UNITY_PROJECT_PATH" \
|
||||
-buildTarget "$BUILD_TARGET" \
|
||||
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET" ) \
|
||||
-customBuildTarget "$BUILD_TARGET" \
|
||||
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
||||
-customBuildProfile "$BUILD_PROFILE" \
|
||||
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
|
||||
-executeMethod "$BUILD_METHOD" \
|
||||
-buildVersion "$VERSION" \
|
||||
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
||||
|
||||
Vendored
+7
-1
@@ -166,7 +166,6 @@ $unityArgs = @(
|
||||
"-customBuildName", "`"$Env:BUILD_NAME`"",
|
||||
"-projectPath", "`"$Env:UNITY_PROJECT_PATH`"",
|
||||
"-executeMethod", "`"$Env:BUILD_METHOD`"",
|
||||
"-buildTarget", "`"$Env:BUILD_TARGET`"",
|
||||
"-customBuildTarget", "`"$Env:BUILD_TARGET`"",
|
||||
"-customBuildPath", "`"$Env:CUSTOM_BUILD_PATH`"",
|
||||
"-customBuildProfile", "`"$Env:BUILD_PROFILE`"",
|
||||
@@ -181,6 +180,13 @@ $unityArgs = @(
|
||||
"-logfile", "-"
|
||||
) + $customParametersArray
|
||||
|
||||
if (-not $Env:BUILD_PROFILE) {
|
||||
$unityArgs += @("-buildTarget", "`"$Env:BUILD_TARGET`"")
|
||||
}
|
||||
if ($Env:BUILD_PROFILE) {
|
||||
$unityArgs += @("-activeBuildProfile", "`"$Env:BUILD_PROFILE`"")
|
||||
}
|
||||
|
||||
# Remove null items as that will fail the Start-Process call
|
||||
$unityArgs = $unityArgs | Where-Object { $_ -ne $null }
|
||||
|
||||
|
||||
+3
@@ -18,6 +18,9 @@ regsvr32 C:\ProgramData\Microsoft\VisualStudio\Setup\x64\Microsoft.VisualStudio.
|
||||
# Kill the regsvr process
|
||||
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
|
||||
|
||||
# Install Visual C++ 2013 Redistributables
|
||||
. "c:\steps\install_vcredist13.ps1"
|
||||
|
||||
# Setup Git Credentials
|
||||
. "c:\steps\set_gitcredential.ps1"
|
||||
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# For some reason, Unity is failing in github actions windows runners
|
||||
# due to missing Visual C++ 2013 redistributables.
|
||||
# This script downloads and installs the required redistributables.
|
||||
Write-Output ""
|
||||
Write-Output "#########################################################"
|
||||
Write-Output "# Installing Visual C++ Redistributables (2013) #"
|
||||
Write-Output "#########################################################"
|
||||
Write-Output ""
|
||||
|
||||
|
||||
choco install vcredist2013 -y --no-progress
|
||||
+9
-8
@@ -12,18 +12,18 @@
|
||||
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
||||
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
||||
"cli": "yarn ts-node src/index.ts -m cli",
|
||||
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
|
||||
"gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"gcp-secrets-tests": "cross-env providerStrategy=aws orchestratorTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"orchestrator\"",
|
||||
"gcp-secrets-cli": "cross-env orchestratorTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"aws-secrets-cli": "cross-env orchestratorTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
|
||||
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
|
||||
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
||||
"test-cli": "cross-env orchestratorTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
||||
"test": "jest",
|
||||
"test:ci": "jest --config=jest.ci.config.js --runInBand",
|
||||
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
|
||||
"test-i": "cross-env orchestratorTests=true yarn test -i -t \"orchestrator\"",
|
||||
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
|
||||
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
|
||||
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
|
||||
"test-i-aws": "cross-env orchestratorTests=true providerStrategy=aws yarn test -i -t \"orchestrator\"",
|
||||
"test-i-k8s": "cross-env orchestratorTests=true providerStrategy=k8s yarn test -i -t \"orchestrator\""
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.x"
|
||||
@@ -50,8 +50,9 @@
|
||||
"nanoid": "^3.3.1",
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"semver": "^7.5.2",
|
||||
"shell-quote": "^1.8.3",
|
||||
"ts-md5": "^1.3.1",
|
||||
"unity-changeset": "^2.0.0",
|
||||
"unity-changeset": "^3.1.0",
|
||||
"uuid": "^9.0.0",
|
||||
"yaml": "^2.2.2"
|
||||
},
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
import * as core from '@actions/core';
|
||||
import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model';
|
||||
import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
|
||||
import { Cli } from './model/cli/cli';
|
||||
import MacBuilder from './model/mac-builder';
|
||||
import PlatformSetup from './model/platform-setup';
|
||||
@@ -33,7 +33,7 @@ async function runMain() {
|
||||
...buildParameters,
|
||||
});
|
||||
} else {
|
||||
await CloudRunner.run(buildParameters, baseImage.toString());
|
||||
await Orchestrator.run(buildParameters, baseImage.toString());
|
||||
exitCode = 0;
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -1,13 +1,13 @@
|
||||
// Integration test for exercising real GitHub check creation and updates.
|
||||
import CloudRunner from '../model/cloud-runner/cloud-runner';
|
||||
import Orchestrator from '../model/orchestrator/orchestrator';
|
||||
import UnityVersioning from '../model/unity-versioning';
|
||||
import GitHub from '../model/github';
|
||||
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/cloud-runner-test-helpers';
|
||||
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/orchestrator-test-helpers';
|
||||
|
||||
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
|
||||
const describeOrSkip = runIntegration ? describe : describe.skip;
|
||||
|
||||
describeOrSkip('Cloud Runner Github Checks Integration', () => {
|
||||
describeOrSkip('Orchestrator Github Checks Integration', () => {
|
||||
it(
|
||||
'creates and updates a real GitHub check',
|
||||
async () => {
|
||||
@@ -15,10 +15,10 @@ describeOrSkip('Cloud Runner Github Checks Integration', () => {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
asyncCloudRunner: `true`,
|
||||
asyncOrchestrator: `true`,
|
||||
githubChecks: `true`,
|
||||
});
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
const checkId = await GitHub.createGitHubCheck(`integration create`);
|
||||
expect(checkId).not.toEqual('');
|
||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);
|
||||
@@ -1,7 +1,7 @@
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import AndroidVersioning from './android-versioning';
|
||||
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
|
||||
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
|
||||
import OrchestratorConstants from './orchestrator/options/orchestrator-constants';
|
||||
import OrchestratorBuildGuid from './orchestrator/options/orchestrator-guid';
|
||||
import Input from './input';
|
||||
import Platform from './platform';
|
||||
import UnityVersioning from './unity-versioning';
|
||||
@@ -10,8 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
|
||||
import { GithubCliReader } from './input-readers/github-cli';
|
||||
import { Cli } from './cli/cli';
|
||||
import GitHub from './github';
|
||||
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
|
||||
import CloudRunner from './cloud-runner/cloud-runner';
|
||||
import OrchestratorOptions from './orchestrator/options/orchestrator-options';
|
||||
import Orchestrator from './orchestrator/orchestrator';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
class BuildParameters {
|
||||
@@ -67,6 +67,7 @@ class BuildParameters {
|
||||
public kubeConfig!: string;
|
||||
public containerMemory!: string;
|
||||
public containerCpu!: string;
|
||||
public containerNamespace!: string;
|
||||
public kubeVolumeSize!: string;
|
||||
public kubeVolume!: string;
|
||||
public kubeStorageClass!: string;
|
||||
@@ -83,11 +84,13 @@ class BuildParameters {
|
||||
public runNumber!: string;
|
||||
public branch!: string;
|
||||
public githubRepo!: string;
|
||||
public orchestratorRepoName!: string;
|
||||
public cloneDepth!: number;
|
||||
public gitSha!: string;
|
||||
public logId!: string;
|
||||
public buildGuid!: string;
|
||||
public cloudRunnerBranch!: string;
|
||||
public cloudRunnerDebug!: boolean | undefined;
|
||||
public orchestratorBranch!: string;
|
||||
public orchestratorDebug!: boolean | undefined;
|
||||
public buildPlatform!: string | undefined;
|
||||
public isCliMode!: boolean;
|
||||
public maxRetainedWorkspaces!: number;
|
||||
@@ -104,8 +107,31 @@ class BuildParameters {
|
||||
public unityHubVersionOnMac!: string;
|
||||
public dockerWorkspacePath!: string;
|
||||
|
||||
// Remote PowerShell provider
|
||||
public remotePowershellHost!: string;
|
||||
public remotePowershellCredential!: string;
|
||||
public remotePowershellTransport!: string;
|
||||
|
||||
// GitHub Actions provider
|
||||
public githubActionsRepo!: string;
|
||||
public githubActionsWorkflow!: string;
|
||||
public githubActionsToken!: string;
|
||||
public githubActionsRef!: string;
|
||||
|
||||
// GitLab CI provider
|
||||
public gitlabProjectId!: string;
|
||||
public gitlabTriggerToken!: string;
|
||||
public gitlabApiUrl!: string;
|
||||
public gitlabRef!: string;
|
||||
|
||||
// Ansible provider
|
||||
public ansibleInventory!: string;
|
||||
public ansiblePlaybook!: string;
|
||||
public ansibleExtraVars!: string;
|
||||
public ansibleVaultPassword!: string;
|
||||
|
||||
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
||||
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
|
||||
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
|
||||
}
|
||||
|
||||
static async create(): Promise<BuildParameters> {
|
||||
@@ -190,52 +216,78 @@ class BuildParameters {
|
||||
dockerIsolationMode: Input.dockerIsolationMode,
|
||||
containerRegistryRepository: Input.containerRegistryRepository,
|
||||
containerRegistryImageVersion: Input.containerRegistryImageVersion,
|
||||
providerStrategy: CloudRunnerOptions.providerStrategy,
|
||||
buildPlatform: CloudRunnerOptions.buildPlatform,
|
||||
kubeConfig: CloudRunnerOptions.kubeConfig,
|
||||
containerMemory: CloudRunnerOptions.containerMemory,
|
||||
containerCpu: CloudRunnerOptions.containerCpu,
|
||||
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
|
||||
kubeVolume: CloudRunnerOptions.kubeVolume,
|
||||
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
|
||||
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
|
||||
customJob: CloudRunnerOptions.customJob,
|
||||
providerStrategy: OrchestratorOptions.providerStrategy,
|
||||
buildPlatform: OrchestratorOptions.buildPlatform,
|
||||
kubeConfig: OrchestratorOptions.kubeConfig,
|
||||
containerMemory: OrchestratorOptions.containerMemory,
|
||||
containerCpu: OrchestratorOptions.containerCpu,
|
||||
containerNamespace: OrchestratorOptions.containerNamespace,
|
||||
kubeVolumeSize: OrchestratorOptions.kubeVolumeSize,
|
||||
kubeVolume: OrchestratorOptions.kubeVolume,
|
||||
postBuildContainerHooks: OrchestratorOptions.postBuildContainerHooks,
|
||||
preBuildContainerHooks: OrchestratorOptions.preBuildContainerHooks,
|
||||
customJob: OrchestratorOptions.customJob,
|
||||
runNumber: Input.runNumber,
|
||||
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
||||
cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0],
|
||||
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
|
||||
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
|
||||
orchestratorBranch: OrchestratorOptions.orchestratorBranch.split('/').reverse()[0],
|
||||
orchestratorDebug: OrchestratorOptions.orchestratorDebug,
|
||||
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || OrchestratorOptions.orchestratorRepoName,
|
||||
orchestratorRepoName: OrchestratorOptions.orchestratorRepoName,
|
||||
cloneDepth: Number.parseInt(OrchestratorOptions.cloneDepth),
|
||||
isCliMode: Cli.isCliMode,
|
||||
awsStackName: CloudRunnerOptions.awsStackName,
|
||||
awsEndpoint: CloudRunnerOptions.awsEndpoint,
|
||||
awsCloudFormationEndpoint: CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||
awsEcsEndpoint: CloudRunnerOptions.awsEcsEndpoint,
|
||||
awsKinesisEndpoint: CloudRunnerOptions.awsKinesisEndpoint,
|
||||
awsCloudWatchLogsEndpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||
awsS3Endpoint: CloudRunnerOptions.awsS3Endpoint,
|
||||
storageProvider: CloudRunnerOptions.storageProvider,
|
||||
rcloneRemote: CloudRunnerOptions.rcloneRemote,
|
||||
awsStackName: OrchestratorOptions.awsStackName,
|
||||
awsEndpoint: OrchestratorOptions.awsEndpoint,
|
||||
awsCloudFormationEndpoint: OrchestratorOptions.awsCloudFormationEndpoint,
|
||||
awsEcsEndpoint: OrchestratorOptions.awsEcsEndpoint,
|
||||
awsKinesisEndpoint: OrchestratorOptions.awsKinesisEndpoint,
|
||||
awsCloudWatchLogsEndpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
||||
awsS3Endpoint: OrchestratorOptions.awsS3Endpoint,
|
||||
storageProvider: OrchestratorOptions.storageProvider,
|
||||
rcloneRemote: OrchestratorOptions.rcloneRemote,
|
||||
gitSha: Input.gitSha,
|
||||
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
|
||||
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
||||
commandHooks: CloudRunnerOptions.commandHooks,
|
||||
inputPullCommand: CloudRunnerOptions.inputPullCommand,
|
||||
pullInputList: CloudRunnerOptions.pullInputList,
|
||||
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
|
||||
cacheKey: CloudRunnerOptions.cacheKey,
|
||||
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
|
||||
useLargePackages: CloudRunnerOptions.useLargePackages,
|
||||
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
|
||||
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
|
||||
githubChecks: CloudRunnerOptions.githubChecks,
|
||||
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
|
||||
githubCheckId: CloudRunnerOptions.githubCheckId,
|
||||
finalHooks: CloudRunnerOptions.finalHooks,
|
||||
skipLfs: CloudRunnerOptions.skipLfs,
|
||||
skipCache: CloudRunnerOptions.skipCache,
|
||||
logId: customAlphabet(OrchestratorConstants.alphabet, 9)(),
|
||||
buildGuid: OrchestratorBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
||||
commandHooks: OrchestratorOptions.commandHooks,
|
||||
inputPullCommand: OrchestratorOptions.inputPullCommand,
|
||||
pullInputList: OrchestratorOptions.pullInputList,
|
||||
kubeStorageClass: OrchestratorOptions.kubeStorageClass,
|
||||
cacheKey: OrchestratorOptions.cacheKey,
|
||||
maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces),
|
||||
useLargePackages: OrchestratorOptions.useLargePackages,
|
||||
useCompressionStrategy: OrchestratorOptions.useCompressionStrategy,
|
||||
garbageMaxAge: OrchestratorOptions.garbageMaxAge,
|
||||
githubChecks: OrchestratorOptions.githubChecks,
|
||||
asyncWorkflow: OrchestratorOptions.asyncOrchestrator,
|
||||
githubCheckId: OrchestratorOptions.githubCheckId,
|
||||
finalHooks: OrchestratorOptions.finalHooks,
|
||||
skipLfs: OrchestratorOptions.skipLfs,
|
||||
skipCache: OrchestratorOptions.skipCache,
|
||||
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
||||
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
||||
dockerWorkspacePath: Input.dockerWorkspacePath,
|
||||
|
||||
// Remote PowerShell provider
|
||||
remotePowershellHost: Input.remotePowershellHost,
|
||||
remotePowershellCredential: Input.remotePowershellCredential,
|
||||
remotePowershellTransport: Input.remotePowershellTransport,
|
||||
|
||||
// GitHub Actions provider
|
||||
githubActionsRepo: Input.githubActionsRepo,
|
||||
githubActionsWorkflow: Input.githubActionsWorkflow,
|
||||
githubActionsToken: Input.githubActionsToken,
|
||||
githubActionsRef: Input.githubActionsRef,
|
||||
|
||||
// GitLab CI provider
|
||||
gitlabProjectId: Input.gitlabProjectId,
|
||||
gitlabTriggerToken: Input.gitlabTriggerToken,
|
||||
gitlabApiUrl: Input.gitlabApiUrl,
|
||||
gitlabRef: Input.gitlabRef,
|
||||
|
||||
// Ansible provider
|
||||
ansibleInventory: Input.ansibleInventory,
|
||||
ansiblePlaybook: Input.ansiblePlaybook,
|
||||
ansibleExtraVars: Input.ansibleExtraVars,
|
||||
ansibleVaultPassword: Input.ansibleVaultPassword,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+34
-34
@@ -1,14 +1,14 @@
|
||||
import { Command } from 'commander-ts';
|
||||
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
|
||||
import { BuildParameters, Orchestrator, ImageTag, Input } from '..';
|
||||
import * as core from '@actions/core';
|
||||
import { ActionYamlReader } from '../input-readers/action-yaml';
|
||||
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
|
||||
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
|
||||
import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
|
||||
import OrchestratorQueryOverride from '../orchestrator/options/orchestrator-query-override';
|
||||
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
||||
import { Caching } from '../cloud-runner/remote-client/caching';
|
||||
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
|
||||
import { RemoteClient } from '../cloud-runner/remote-client';
|
||||
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
|
||||
import { Caching } from '../orchestrator/remote-client/caching';
|
||||
import { LfsHashing } from '../orchestrator/services/utility/lfs-hashing';
|
||||
import { RemoteClient } from '../orchestrator/remote-client';
|
||||
import OrchestratorOptionsReader from '../orchestrator/options/orchestrator-options-reader';
|
||||
import GitHub from '../github';
|
||||
import { OptionValues } from 'commander';
|
||||
import { InputKey } from '../input';
|
||||
@@ -36,7 +36,7 @@ export class Cli {
|
||||
const program = new Command();
|
||||
program.version('0.0.1');
|
||||
|
||||
const properties = CloudRunnerOptionsReader.GetProperties();
|
||||
const properties = OrchestratorOptionsReader.GetProperties();
|
||||
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
||||
for (const element of properties) {
|
||||
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
||||
@@ -62,23 +62,23 @@ export class Cli {
|
||||
static async RunCli(): Promise<void> {
|
||||
GitHub.githubInputEnabled = false;
|
||||
if (Cli.options!['populateOverride'] === `true`) {
|
||||
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
|
||||
await OrchestratorQueryOverride.PopulateQueryOverrideInput();
|
||||
}
|
||||
if (Cli.options!['logInput']) {
|
||||
Cli.logInput();
|
||||
}
|
||||
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
|
||||
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
|
||||
OrchestratorLogger.log(`Entrypoint: ${results.key}`);
|
||||
Cli.options!.versioning = 'None';
|
||||
|
||||
CloudRunner.buildParameters = await BuildParameters.create();
|
||||
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
|
||||
CloudRunnerLogger.log(`Build Params:
|
||||
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
|
||||
Orchestrator.buildParameters = await BuildParameters.create();
|
||||
Orchestrator.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
|
||||
OrchestratorLogger.log(`Build Params:
|
||||
${JSON.stringify(Orchestrator.buildParameters, undefined, 4)}
|
||||
`);
|
||||
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
|
||||
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
|
||||
await CloudRunner.setup(CloudRunner.buildParameters);
|
||||
Orchestrator.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
|
||||
OrchestratorLogger.log(`Locked Workspace: ${Orchestrator.lockedWorkspace}`);
|
||||
await Orchestrator.setup(Orchestrator.buildParameters);
|
||||
|
||||
return await results.target[results.propertyKey](Cli.options);
|
||||
}
|
||||
@@ -87,7 +87,7 @@ export class Cli {
|
||||
private static logInput() {
|
||||
core.info(`\n`);
|
||||
core.info(`INPUT:`);
|
||||
const properties = CloudRunnerOptionsReader.GetProperties();
|
||||
const properties = OrchestratorOptionsReader.GetProperties();
|
||||
for (const element of properties) {
|
||||
if (
|
||||
element in Input &&
|
||||
@@ -104,28 +104,28 @@ export class Cli {
|
||||
core.info(`\n`);
|
||||
}
|
||||
|
||||
@CliFunction(`cli-build`, `runs a cloud runner build`)
|
||||
@CliFunction(`cli-build`, `runs a orchestrator build`)
|
||||
public static async CLIBuild(): Promise<string> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
|
||||
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
||||
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
|
||||
}
|
||||
|
||||
@CliFunction(`async-workflow`, `runs a cloud runner build`)
|
||||
@CliFunction(`async-workflow`, `runs a orchestrator build`)
|
||||
public static async asyncronousWorkflow(): Promise<string> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
|
||||
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
||||
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
|
||||
}
|
||||
|
||||
@CliFunction(`checks-update`, `runs a cloud runner build`)
|
||||
@CliFunction(`checks-update`, `runs a orchestrator build`)
|
||||
public static async checksUpdate() {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
|
||||
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
|
||||
if (input.mode === `create`) {
|
||||
@@ -139,18 +139,18 @@ export class Cli {
|
||||
public static async GarbageCollect(): Promise<string> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
|
||||
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false);
|
||||
return await Orchestrator.Provider.garbageCollect(``, false, 0, false, false);
|
||||
}
|
||||
|
||||
@CliFunction(`list-resources`, `lists active resources`)
|
||||
public static async ListResources(): Promise<string[]> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
await CloudRunner.setup(buildParameter);
|
||||
const result = await CloudRunner.Provider.listResources();
|
||||
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4));
|
||||
await Orchestrator.setup(buildParameter);
|
||||
const result = await Orchestrator.Provider.listResources();
|
||||
OrchestratorLogger.log(JSON.stringify(result, undefined, 4));
|
||||
|
||||
return result.map((x) => x.Name);
|
||||
}
|
||||
@@ -159,17 +159,17 @@ export class Cli {
|
||||
public static async ListWorfklow(): Promise<string[]> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
|
||||
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name);
|
||||
return (await Orchestrator.Provider.listWorkflow()).map((x) => x.Name);
|
||||
}
|
||||
|
||||
@CliFunction(`watch`, `follows logs of a running workflow`)
|
||||
public static async Watch(): Promise<string> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
await CloudRunner.setup(buildParameter);
|
||||
await Orchestrator.setup(buildParameter);
|
||||
|
||||
return await CloudRunner.Provider.watchWorkflow();
|
||||
return await Orchestrator.Provider.watchWorkflow();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
import AwsBuildPlatform from './providers/aws';
|
||||
import { BuildParameters, Input } from '..';
|
||||
import Kubernetes from './providers/k8s';
|
||||
import CloudRunnerLogger from './services/core/cloud-runner-logger';
|
||||
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
|
||||
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
|
||||
import { CloudRunnerError } from './error/cloud-runner-error';
|
||||
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunnerSecret from './options/cloud-runner-secret';
|
||||
import { ProviderInterface } from './providers/provider-interface';
|
||||
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
|
||||
import TestCloudRunner from './providers/test';
|
||||
import LocalCloudRunner from './providers/local';
|
||||
import LocalDockerCloudRunner from './providers/docker';
|
||||
import loadProvider from './providers/provider-loader';
|
||||
import GitHub from '../github';
|
||||
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
|
||||
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
|
||||
import CloudRunnerResult from './services/core/cloud-runner-result';
|
||||
import CloudRunnerOptions from './options/cloud-runner-options';
|
||||
|
||||
class CloudRunner {
|
||||
public static Provider: ProviderInterface;
|
||||
public static buildParameters: BuildParameters;
|
||||
private static defaultSecrets: CloudRunnerSecret[];
|
||||
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
|
||||
static lockedWorkspace: string = ``;
|
||||
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
|
||||
public static get isCloudRunnerEnvironment() {
|
||||
return process.env[`GITHUB_ACTIONS`] !== `true`;
|
||||
}
|
||||
public static get isCloudRunnerAsyncEnvironment() {
|
||||
return process.env[`ASYNC_WORKFLOW`] === `true`;
|
||||
}
|
||||
public static async setup(buildParameters: BuildParameters) {
|
||||
CloudRunnerLogger.setup();
|
||||
CloudRunnerLogger.log(`Setting up cloud runner`);
|
||||
CloudRunner.buildParameters = buildParameters;
|
||||
if (CloudRunner.buildParameters.githubCheckId === ``) {
|
||||
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
|
||||
}
|
||||
await CloudRunner.setupSelectedBuildPlatform();
|
||||
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
|
||||
CloudRunner.cloudRunnerEnvironmentVariables =
|
||||
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters);
|
||||
if (GitHub.githubInputEnabled) {
|
||||
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
|
||||
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
|
||||
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
|
||||
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
|
||||
}
|
||||
for (const element of buildParameterPropertyNames) {
|
||||
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
|
||||
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
|
||||
}
|
||||
core.setOutput(
|
||||
Input.ToEnvVarFormat(`buildArtifact`),
|
||||
`build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
);
|
||||
}
|
||||
FollowLogStreamService.Reset();
|
||||
}
|
||||
|
||||
private static async setupSelectedBuildPlatform() {
|
||||
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
|
||||
|
||||
// Detect LocalStack endpoints and reroute AWS provider to local-docker for CI tests that only need S3
|
||||
const endpointsToCheck = [
|
||||
process.env.AWS_ENDPOINT,
|
||||
process.env.AWS_S3_ENDPOINT,
|
||||
process.env.AWS_CLOUD_FORMATION_ENDPOINT,
|
||||
process.env.AWS_ECS_ENDPOINT,
|
||||
process.env.AWS_KINESIS_ENDPOINT,
|
||||
process.env.AWS_CLOUD_WATCH_LOGS_ENDPOINT,
|
||||
CloudRunnerOptions.awsEndpoint,
|
||||
CloudRunnerOptions.awsS3Endpoint,
|
||||
CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||
CloudRunnerOptions.awsEcsEndpoint,
|
||||
CloudRunnerOptions.awsKinesisEndpoint,
|
||||
CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||
]
|
||||
.filter((x) => typeof x === 'string')
|
||||
.join(' ');
|
||||
const isLocalStack = /localstack|localhost|127\.0\.0\.1/i.test(endpointsToCheck);
|
||||
let provider = CloudRunner.buildParameters.providerStrategy;
|
||||
if (provider === 'aws' && isLocalStack) {
|
||||
CloudRunnerLogger.log('LocalStack endpoints detected; routing provider to local-docker for this run');
|
||||
provider = 'local-docker';
|
||||
}
|
||||
|
||||
switch (provider) {
|
||||
case 'k8s':
|
||||
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
|
||||
break;
|
||||
case 'aws':
|
||||
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
|
||||
break;
|
||||
case 'test':
|
||||
CloudRunner.Provider = new TestCloudRunner();
|
||||
break;
|
||||
case 'local-docker':
|
||||
CloudRunner.Provider = new LocalDockerCloudRunner();
|
||||
break;
|
||||
case 'local-system':
|
||||
CloudRunner.Provider = new LocalCloudRunner();
|
||||
break;
|
||||
case 'local':
|
||||
CloudRunner.Provider = new LocalCloudRunner();
|
||||
break;
|
||||
default:
|
||||
// Try to load provider using the dynamic loader for unknown providers
|
||||
try {
|
||||
CloudRunner.Provider = await loadProvider(provider, CloudRunner.buildParameters);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Failed to load provider '${provider}' using dynamic loader: ${error.message}`);
|
||||
CloudRunnerLogger.log('Falling back to local provider...');
|
||||
CloudRunner.Provider = new LocalCloudRunner();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static async run(buildParameters: BuildParameters, baseImage: string) {
|
||||
if (baseImage.includes(`undefined`)) {
|
||||
throw new Error(`baseImage is undefined`);
|
||||
}
|
||||
await CloudRunner.setup(buildParameters);
|
||||
await CloudRunner.Provider.setupWorkflow(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
CloudRunner.buildParameters.branch,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
try {
|
||||
if (buildParameters.maxRetainedWorkspaces > 0) {
|
||||
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
|
||||
|
||||
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
|
||||
CloudRunner.lockedWorkspace,
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
);
|
||||
|
||||
if (result) {
|
||||
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`);
|
||||
CloudRunner.cloudRunnerEnvironmentVariables = [
|
||||
...CloudRunner.cloudRunnerEnvironmentVariables,
|
||||
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
|
||||
];
|
||||
} else {
|
||||
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
|
||||
buildParameters.maxRetainedWorkspaces = 0;
|
||||
CloudRunner.lockedWorkspace = ``;
|
||||
}
|
||||
}
|
||||
await CloudRunner.updateStatusWithBuildParameters();
|
||||
const output = await new WorkflowCompositionRoot().run(
|
||||
new CloudRunnerStepParameters(
|
||||
baseImage,
|
||||
CloudRunner.cloudRunnerEnvironmentVariables,
|
||||
CloudRunner.defaultSecrets,
|
||||
),
|
||||
);
|
||||
await CloudRunner.Provider.cleanupWorkflow(
|
||||
CloudRunner.buildParameters,
|
||||
CloudRunner.buildParameters.branch,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) {
|
||||
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
|
||||
}
|
||||
|
||||
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
||||
const workspace = CloudRunner.lockedWorkspace || ``;
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(
|
||||
workspace,
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
);
|
||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
|
||||
if (isLocked) {
|
||||
throw new Error(
|
||||
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
|
||||
workspace,
|
||||
buildParameters,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
CloudRunner.lockedWorkspace = ``;
|
||||
}
|
||||
|
||||
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
|
||||
|
||||
if (buildParameters.constantGarbageCollection) {
|
||||
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
|
||||
}
|
||||
|
||||
return new CloudRunnerResult(buildParameters, output, true, true, false);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
|
||||
await GitHub.updateGitHubCheck(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
`Failed - Error ${error?.message || error}`,
|
||||
`failure`,
|
||||
`completed`,
|
||||
);
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async updateStatusWithBuildParameters() {
|
||||
const content = { ...CloudRunner.buildParameters };
|
||||
content.gitPrivateToken = ``;
|
||||
content.unitySerial = ``;
|
||||
content.unityEmail = ``;
|
||||
content.unityPassword = ``;
|
||||
const jsonContent = JSON.stringify(content, undefined, 4);
|
||||
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
|
||||
}
|
||||
}
|
||||
export default CloudRunner;
|
||||
@@ -1,15 +0,0 @@
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
|
||||
export class CloudRunnerError {
|
||||
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
|
||||
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
|
||||
core.setFailed('Cloud Runner failed');
|
||||
if (CloudRunner.Provider !== undefined) {
|
||||
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
class CloudRunnerConstants {
|
||||
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
|
||||
}
|
||||
export default CloudRunnerConstants;
|
||||
@@ -1,5 +0,0 @@
|
||||
class CloudRunnerEnvironmentVariable {
|
||||
public name!: string;
|
||||
public value!: string;
|
||||
}
|
||||
export default CloudRunnerEnvironmentVariable;
|
||||
@@ -1,90 +0,0 @@
|
||||
import path from 'node:path';
|
||||
import CloudRunnerOptions from './cloud-runner-options';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
|
||||
export class CloudRunnerFolders {
|
||||
public static readonly repositoryFolder = 'repo';
|
||||
|
||||
public static ToLinuxFolder(folder: string) {
|
||||
return folder.replace(/\\/g, `/`);
|
||||
}
|
||||
|
||||
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
|
||||
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
|
||||
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
|
||||
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
|
||||
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
||||
}
|
||||
|
||||
public static get cacheFolderForAllFull(): string {
|
||||
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
|
||||
}
|
||||
|
||||
public static get cacheFolderForCacheKeyFull(): string {
|
||||
return path.join(
|
||||
'/',
|
||||
CloudRunnerFolders.buildVolumeFolder,
|
||||
CloudRunnerFolders.cacheFolder,
|
||||
CloudRunner.buildParameters.cacheKey,
|
||||
);
|
||||
}
|
||||
|
||||
public static get builderPathAbsolute(): string {
|
||||
return path.join(
|
||||
CloudRunnerOptions.useSharedBuilder
|
||||
? `/${CloudRunnerFolders.buildVolumeFolder}`
|
||||
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||
`builder`,
|
||||
);
|
||||
}
|
||||
|
||||
public static get repoPathAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
|
||||
}
|
||||
|
||||
public static get projectPathAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
|
||||
}
|
||||
|
||||
public static get libraryFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
|
||||
}
|
||||
|
||||
public static get projectBuildFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
|
||||
}
|
||||
|
||||
public static get lfsFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
|
||||
}
|
||||
|
||||
public static get purgeRemoteCaching(): boolean {
|
||||
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
||||
}
|
||||
|
||||
public static get lfsCacheFolderFull() {
|
||||
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
|
||||
}
|
||||
|
||||
public static get libraryCacheFolderFull() {
|
||||
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
|
||||
}
|
||||
|
||||
public static get unityBuilderRepoUrl(): string {
|
||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
|
||||
}
|
||||
|
||||
public static get targetBuildRepoUrl(): string {
|
||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
|
||||
}
|
||||
|
||||
public static get buildVolumeFolder() {
|
||||
return 'data';
|
||||
}
|
||||
|
||||
public static get cacheFolder() {
|
||||
return 'cache';
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import Input from '../../input';
|
||||
import CloudRunnerOptions from './cloud-runner-options';
|
||||
|
||||
class CloudRunnerOptionsReader {
|
||||
static GetProperties() {
|
||||
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
|
||||
}
|
||||
}
|
||||
|
||||
export default CloudRunnerOptionsReader;
|
||||
@@ -1,319 +0,0 @@
|
||||
import { Cli } from '../../cli/cli';
|
||||
import CloudRunnerQueryOverride from './cloud-runner-query-override';
|
||||
import GitHub from '../../github';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
class CloudRunnerOptions {
|
||||
// ### ### ###
|
||||
// Input Handling
|
||||
// ### ### ###
|
||||
public static getInput(query: string): string | undefined {
|
||||
if (GitHub.githubInputEnabled) {
|
||||
const coreInput = core.getInput(query);
|
||||
if (coreInput && coreInput !== '') {
|
||||
return coreInput;
|
||||
}
|
||||
}
|
||||
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
|
||||
|
||||
// Query input sources
|
||||
if (Cli.query(query, alternativeQuery)) {
|
||||
return Cli.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
|
||||
return CloudRunnerQueryOverride.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (process.env[query] !== undefined) {
|
||||
return process.env[query];
|
||||
}
|
||||
|
||||
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
|
||||
return process.env[alternativeQuery];
|
||||
}
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string): string {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
}
|
||||
|
||||
return input
|
||||
.replace(/([A-Z])/g, ' $1')
|
||||
.trim()
|
||||
.toUpperCase()
|
||||
.replace(/ /g, '_');
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Provider parameters
|
||||
// ### ### ###
|
||||
|
||||
static get region(): string {
|
||||
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// GitHub parameters
|
||||
// ### ### ###
|
||||
static get githubChecks(): boolean {
|
||||
const value = CloudRunnerOptions.getInput('githubChecks');
|
||||
|
||||
return value === `true` || false;
|
||||
}
|
||||
static get githubCheckId(): string {
|
||||
return CloudRunnerOptions.getInput('githubCheckId') || ``;
|
||||
}
|
||||
|
||||
static get githubOwner(): string {
|
||||
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
|
||||
}
|
||||
|
||||
static get githubRepoName(): string {
|
||||
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
|
||||
}
|
||||
|
||||
static get finalHooks(): string[] {
|
||||
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Git syncronization parameters
|
||||
// ### ### ###
|
||||
|
||||
static get githubRepo(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
|
||||
}
|
||||
static get branch(): string {
|
||||
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
|
||||
return (
|
||||
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
|
||||
);
|
||||
} else if (CloudRunnerOptions.getInput('branch')) {
|
||||
return CloudRunnerOptions.getInput('branch') || ``;
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Cloud Runner parameters
|
||||
// ### ### ###
|
||||
|
||||
static get buildPlatform(): string {
|
||||
const input = CloudRunnerOptions.getInput('buildPlatform');
|
||||
if (input && input !== '') {
|
||||
return input;
|
||||
}
|
||||
if (CloudRunnerOptions.providerStrategy !== 'local') {
|
||||
return 'linux';
|
||||
}
|
||||
|
||||
return process.platform;
|
||||
}
|
||||
|
||||
static get cloudRunnerBranch(): string {
|
||||
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
|
||||
}
|
||||
|
||||
static get providerStrategy(): string {
|
||||
const provider =
|
||||
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
|
||||
if (Cli.isCliMode) {
|
||||
return provider || 'aws';
|
||||
}
|
||||
|
||||
return provider || 'local';
|
||||
}
|
||||
|
||||
static get containerCpu(): string {
|
||||
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
|
||||
}
|
||||
|
||||
static get containerMemory(): string {
|
||||
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
|
||||
}
|
||||
|
||||
static get customJob(): string {
|
||||
return CloudRunnerOptions.getInput('customJob') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Custom commands from files parameters
|
||||
// ### ### ###
|
||||
|
||||
static get containerHookFiles(): string[] {
|
||||
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
static get commandHookFiles(): string[] {
|
||||
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Custom commands from yaml parameters
|
||||
// ### ### ###
|
||||
|
||||
static get commandHooks(): string {
|
||||
return CloudRunnerOptions.getInput('commandHooks') || '';
|
||||
}
|
||||
|
||||
static get postBuildContainerHooks(): string {
|
||||
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
|
||||
}
|
||||
|
||||
static get preBuildContainerHooks(): string {
|
||||
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Input override handling
|
||||
// ### ### ###
|
||||
|
||||
static get pullInputList(): string[] {
|
||||
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
static get inputPullCommand(): string {
|
||||
const value = CloudRunnerOptions.getInput('inputPullCommand');
|
||||
|
||||
if (value === 'gcp-secret-manager') {
|
||||
return 'gcloud secrets versions access 1 --secret="{0}"';
|
||||
} else if (value === 'aws-secret-manager') {
|
||||
return 'aws secretsmanager get-secret-value --secret-id {0}';
|
||||
}
|
||||
|
||||
return value || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Aws
|
||||
// ### ### ###
|
||||
|
||||
static get awsStackName() {
|
||||
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
|
||||
}
|
||||
|
||||
static get awsEndpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsEndpoint');
|
||||
}
|
||||
|
||||
static get awsCloudFormationEndpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsCloudFormationEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsEcsEndpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsEcsEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsKinesisEndpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsKinesisEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsCloudWatchLogsEndpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsCloudWatchLogsEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsS3Endpoint(): string | undefined {
|
||||
return CloudRunnerOptions.getInput('awsS3Endpoint') || CloudRunnerOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Storage
|
||||
// ### ### ###
|
||||
|
||||
static get storageProvider(): string {
|
||||
return CloudRunnerOptions.getInput('storageProvider') || 's3';
|
||||
}
|
||||
|
||||
static get rcloneRemote(): string {
|
||||
return CloudRunnerOptions.getInput('rcloneRemote') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// K8s
|
||||
// ### ### ###
|
||||
|
||||
static get kubeConfig(): string {
|
||||
return CloudRunnerOptions.getInput('kubeConfig') || '';
|
||||
}
|
||||
|
||||
static get kubeVolume(): string {
|
||||
return CloudRunnerOptions.getInput('kubeVolume') || '';
|
||||
}
|
||||
|
||||
static get kubeVolumeSize(): string {
|
||||
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
|
||||
}
|
||||
|
||||
static get kubeStorageClass(): string {
|
||||
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Caching
|
||||
// ### ### ###
|
||||
|
||||
static get cacheKey(): string {
|
||||
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Utility Parameters
|
||||
// ### ### ###
|
||||
|
||||
static get cloudRunnerDebug(): boolean {
|
||||
return (
|
||||
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
|
||||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
|
||||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
|
||||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
|
||||
false
|
||||
);
|
||||
}
|
||||
static get skipLfs(): boolean {
|
||||
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
|
||||
}
|
||||
static get skipCache(): boolean {
|
||||
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
|
||||
}
|
||||
|
||||
public static get asyncCloudRunner(): boolean {
|
||||
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
|
||||
}
|
||||
|
||||
public static get useLargePackages(): boolean {
|
||||
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
|
||||
}
|
||||
|
||||
public static get useSharedBuilder(): boolean {
|
||||
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
|
||||
}
|
||||
|
||||
public static get useCompressionStrategy(): boolean {
|
||||
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
|
||||
}
|
||||
|
||||
public static get useCleanupCron(): boolean {
|
||||
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Retained Workspace
|
||||
// ### ### ###
|
||||
|
||||
public static get maxRetainedWorkspaces(): string {
|
||||
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Garbage Collection
|
||||
// ### ### ###
|
||||
|
||||
static get garbageMaxAge(): number {
|
||||
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
|
||||
}
|
||||
}
|
||||
|
||||
export default CloudRunnerOptions;
|
||||
@@ -1,67 +0,0 @@
|
||||
import Input from '../../input';
|
||||
import { GenericInputReader } from '../../input-readers/generic-input-reader';
|
||||
import CloudRunnerOptions from './cloud-runner-options';
|
||||
|
||||
const formatFunction = (value: string, arguments_: any[]) => {
|
||||
for (const element of arguments_) {
|
||||
value = value.replace(`{${element.key}}`, element.value);
|
||||
}
|
||||
|
||||
return value;
|
||||
};
|
||||
|
||||
class CloudRunnerQueryOverride {
|
||||
static queryOverrides: { [key: string]: string } | undefined;
|
||||
|
||||
// TODO accept premade secret sources or custom secret source definition yamls
|
||||
|
||||
public static query(key: string, alternativeKey: string) {
|
||||
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
|
||||
return CloudRunnerQueryOverride.queryOverrides[key];
|
||||
}
|
||||
if (
|
||||
CloudRunnerQueryOverride.queryOverrides &&
|
||||
alternativeKey &&
|
||||
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
|
||||
) {
|
||||
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
private static shouldUseOverride(query: string) {
|
||||
if (CloudRunnerOptions.inputPullCommand !== '') {
|
||||
if (CloudRunnerOptions.pullInputList.length > 0) {
|
||||
const doesInclude =
|
||||
CloudRunnerOptions.pullInputList.includes(query) ||
|
||||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
|
||||
|
||||
return doesInclude ? true : false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async queryOverride(query: string) {
|
||||
if (!this.shouldUseOverride(query)) {
|
||||
throw new Error(`Should not be trying to run override query on ${query}`);
|
||||
}
|
||||
|
||||
return await GenericInputReader.Run(
|
||||
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
|
||||
);
|
||||
}
|
||||
|
||||
public static async PopulateQueryOverrideInput() {
|
||||
const queries = CloudRunnerOptions.pullInputList;
|
||||
CloudRunnerQueryOverride.queryOverrides = {};
|
||||
for (const element of queries) {
|
||||
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
|
||||
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export default CloudRunnerQueryOverride;
|
||||
@@ -1,3 +0,0 @@
|
||||
export class CloudRunnerStatics {
|
||||
public static readonly logPrefix = `Cloud-Runner`;
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from './cloud-runner-secret';
|
||||
|
||||
export class CloudRunnerStepParameters {
|
||||
public image: string;
|
||||
public environment: CloudRunnerEnvironmentVariable[];
|
||||
public secrets: CloudRunnerSecret[];
|
||||
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
|
||||
this.image = image;
|
||||
this.environment = environmentVariables;
|
||||
this.secrets = secrets;
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
|
||||
class KubernetesJobSpecFactory {
|
||||
static getJobSpec(
|
||||
command: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingDirectory: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
secretName: string,
|
||||
pvcName: string,
|
||||
jobName: string,
|
||||
k8s: any,
|
||||
containerName: string,
|
||||
ip: string = '',
|
||||
) {
|
||||
const endpointEnvNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
const adjustedEnvironment = environment.map((x) => {
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
value = value
|
||||
.replace('http://localhost', 'http://host.k3d.internal')
|
||||
.replace('http://127.0.0.1', 'http://host.k3d.internal');
|
||||
}
|
||||
return { name: x.name, value } as CloudRunnerEnvironmentVariable;
|
||||
});
|
||||
|
||||
const job = new k8s.V1Job();
|
||||
job.apiVersion = 'batch/v1';
|
||||
job.kind = 'Job';
|
||||
job.metadata = {
|
||||
name: jobName,
|
||||
labels: {
|
||||
app: 'unity-builder',
|
||||
buildGuid,
|
||||
},
|
||||
};
|
||||
job.spec = {
|
||||
ttlSecondsAfterFinished: 9999,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
spec: {
|
||||
volumes: [
|
||||
{
|
||||
name: 'build-mount',
|
||||
persistentVolumeClaim: {
|
||||
claimName: pvcName,
|
||||
},
|
||||
},
|
||||
],
|
||||
containers: [
|
||||
{
|
||||
ttlSecondsAfterFinished: 9999,
|
||||
name: containerName,
|
||||
image,
|
||||
command: ['/bin/sh'],
|
||||
args: [
|
||||
'-c',
|
||||
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
|
||||
],
|
||||
|
||||
workingDir: `${workingDirectory}`,
|
||||
resources: {
|
||||
requests: {
|
||||
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
|
||||
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
|
||||
},
|
||||
},
|
||||
env: [
|
||||
...adjustedEnvironment.map((x) => {
|
||||
const environmentVariable = new V1EnvVar();
|
||||
environmentVariable.name = x.name;
|
||||
environmentVariable.value = x.value;
|
||||
|
||||
return environmentVariable;
|
||||
}),
|
||||
...secrets.map((x) => {
|
||||
const secret = new V1EnvVarSource();
|
||||
secret.secretKeyRef = new V1SecretKeySelector();
|
||||
secret.secretKeyRef.key = x.ParameterKey;
|
||||
secret.secretKeyRef.name = secretName;
|
||||
const environmentVariable = new V1EnvVar();
|
||||
environmentVariable.name = x.EnvironmentVariable;
|
||||
environmentVariable.valueFrom = secret;
|
||||
|
||||
return environmentVariable;
|
||||
}),
|
||||
{ name: 'LOG_SERVICE_IP', value: ip },
|
||||
],
|
||||
volumeMounts: [
|
||||
{
|
||||
name: 'build-mount',
|
||||
mountPath: `${mountdir}`,
|
||||
},
|
||||
],
|
||||
lifecycle: {
|
||||
preStop: {
|
||||
exec: {
|
||||
command: [
|
||||
`wait 60s;
|
||||
cd /data/builder/action/steps;
|
||||
chmod +x /return_license.sh;
|
||||
/return_license.sh;`,
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
restartPolicy: 'Never',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (process.env['CLOUD_RUNNER_MINIKUBE']) {
|
||||
job.spec.template.spec.volumes[0] = {
|
||||
name: 'build-mount',
|
||||
hostPath: {
|
||||
path: `/data`,
|
||||
type: `Directory`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
|
||||
|
||||
return job;
|
||||
}
|
||||
}
|
||||
export default KubernetesJobSpecFactory;
|
||||
@@ -1,23 +0,0 @@
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
class KubernetesPods {
|
||||
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
|
||||
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
|
||||
const phase = pods[0]?.status?.phase || 'undefined status';
|
||||
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
|
||||
if (phase === `Failed`) {
|
||||
throw new Error(`K8s pod failed`);
|
||||
}
|
||||
|
||||
return running;
|
||||
}
|
||||
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
|
||||
const phase = pods?.status?.phase || 'undefined status';
|
||||
|
||||
return phase;
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesPods;
|
||||
@@ -1,116 +0,0 @@
|
||||
import { waitUntil } from 'async-wait-until';
|
||||
import * as core from '@actions/core';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { IncomingMessage } from 'node:http';
|
||||
import GitHub from '../../../github';
|
||||
|
||||
class KubernetesStorage {
|
||||
public static async createPersistentVolumeClaim(
|
||||
buildParameters: BuildParameters,
|
||||
pvcName: string,
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
) {
|
||||
if (buildParameters.kubeVolume !== ``) {
|
||||
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
|
||||
pvcName = buildParameters.kubeVolume;
|
||||
|
||||
return;
|
||||
}
|
||||
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
|
||||
const pvcList = allPvc.map((x) => x.metadata?.name);
|
||||
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
|
||||
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
|
||||
if (pvcList.includes(pvcName)) {
|
||||
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
|
||||
if (GitHub.githubInputEnabled) {
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
|
||||
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
|
||||
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
|
||||
}
|
||||
|
||||
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||
try {
|
||||
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
|
||||
} catch (error) {
|
||||
core.error('Failed to get PVC phase');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||
try {
|
||||
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
||||
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
|
||||
},
|
||||
{
|
||||
timeout: 750000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
} catch (error: any) {
|
||||
core.error('Failed to watch PVC');
|
||||
core.error(error.toString());
|
||||
core.error(
|
||||
`PVC Body: ${JSON.stringify(
|
||||
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
|
||||
undefined,
|
||||
4,
|
||||
)}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async createPVC(
|
||||
pvcName: string,
|
||||
buildParameters: BuildParameters,
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
) {
|
||||
const pvc = new k8s.V1PersistentVolumeClaim();
|
||||
pvc.apiVersion = 'v1';
|
||||
pvc.kind = 'PersistentVolumeClaim';
|
||||
pvc.metadata = {
|
||||
name: pvcName,
|
||||
};
|
||||
pvc.spec = {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
|
||||
resources: {
|
||||
requests: {
|
||||
storage: buildParameters.kubeVolumeSize,
|
||||
},
|
||||
},
|
||||
};
|
||||
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static async handleResult(
|
||||
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
pvcName: string,
|
||||
) {
|
||||
const name = result.body.metadata?.name || '';
|
||||
CloudRunnerLogger.log(`PVC ${name} created`);
|
||||
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
|
||||
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`);
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesStorage;
|
||||
@@ -1,119 +0,0 @@
|
||||
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { waitUntil } from 'async-wait-until';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import KubernetesPods from './kubernetes-pods';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
|
||||
class KubernetesTaskRunner {
|
||||
static readonly maxRetry: number = 3;
|
||||
static lastReceivedMessage: string = ``;
|
||||
|
||||
static async runTask(
|
||||
kubeConfig: KubeConfig,
|
||||
kubeClient: CoreV1Api,
|
||||
jobName: string,
|
||||
podName: string,
|
||||
containerName: string,
|
||||
namespace: string,
|
||||
) {
|
||||
let output = '';
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
let retriesAfterFinish = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
CloudRunnerLogger.log(
|
||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
|
||||
);
|
||||
let extraFlags = ``;
|
||||
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
|
||||
? ` -f -c ${containerName}`
|
||||
: ` --previous`;
|
||||
|
||||
const callback = (outputChunk: string) => {
|
||||
output += outputChunk;
|
||||
|
||||
// split output chunk and handle per line
|
||||
for (const chunk of outputChunk.split(`\n`)) {
|
||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||
chunk,
|
||||
shouldReadLogs,
|
||||
shouldCleanup,
|
||||
output,
|
||||
));
|
||||
}
|
||||
};
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
|
||||
} catch (error: any) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
|
||||
if (continueStreaming) {
|
||||
continue;
|
||||
}
|
||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||
retriesAfterFinish++;
|
||||
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||
CloudRunnerLogger.log('end of log stream');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||
let waitComplete: boolean = false;
|
||||
let message = ``;
|
||||
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const phase = status?.body.status?.phase;
|
||||
waitComplete = phase !== 'Pending';
|
||||
message = `Phase:${status.body.status?.phase} \n Reason:${
|
||||
status.body.status?.conditions?.[0].reason || ''
|
||||
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
|
||||
|
||||
// CloudRunnerLogger.log(
|
||||
// JSON.stringify(
|
||||
// (await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
// .map((x) => {
|
||||
// return {
|
||||
// message: x.message || ``,
|
||||
// name: x.metadata.name || ``,
|
||||
// reason: x.reason || ``,
|
||||
// };
|
||||
// })
|
||||
// .filter((x) => x.name.includes(podName)),
|
||||
// undefined,
|
||||
// 4,
|
||||
// ),
|
||||
// );
|
||||
if (waitComplete || phase !== 'Pending') return true;
|
||||
|
||||
return false;
|
||||
},
|
||||
{
|
||||
timeout: 2000000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
if (!waitComplete) {
|
||||
CloudRunnerLogger.log(message);
|
||||
}
|
||||
|
||||
return waitComplete;
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesTaskRunner;
|
||||
@@ -1,181 +0,0 @@
|
||||
import { assert } from 'node:console';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
// eslint-disable-next-line github/no-then
|
||||
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
|
||||
|
||||
export class Caching {
|
||||
@CliFunction(`cache-push`, `push to cache`)
|
||||
static async cachePush() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
await Caching.PushToCache(
|
||||
Cli.options!['cachePushTo'],
|
||||
Cli.options!['cachePushFrom'],
|
||||
Cli.options!['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
@CliFunction(`cache-pull`, `pull from cache`)
|
||||
static async cachePull() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
await Caching.PullFromCache(
|
||||
Cli.options!['cachePushFrom'],
|
||||
Cli.options!['cachePushTo'],
|
||||
Cli.options!['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
let compressionSuffix = '';
|
||||
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
||||
compressionSuffix = `.lz4`;
|
||||
}
|
||||
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
process.chdir(path.resolve(sourceFolder, '..'));
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
|
||||
CloudRunnerLogger.log(
|
||||
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||
sourceFolder,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
const contents = await fs.promises.readdir(path.basename(sourceFolder));
|
||||
CloudRunnerLogger.log(
|
||||
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
|
||||
if (contents.length === 0) {
|
||||
CloudRunnerLogger.log(
|
||||
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
process.chdir(`${startPath}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await CloudRunnerSystem.Run(
|
||||
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||
assert(
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
||||
'cache archive exists inside cache folder',
|
||||
);
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
|
||||
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
|
||||
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
|
||||
|
||||
return;
|
||||
}
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
let compressionSuffix = '';
|
||||
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
||||
compressionSuffix = `.lz4`;
|
||||
}
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await fs.promises.mkdir(cacheFolder);
|
||||
}
|
||||
|
||||
if (!(await fileExists(destinationFolder))) {
|
||||
await fs.promises.mkdir(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (
|
||||
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
||||
)
|
||||
.replace(/\n/g, ``)
|
||||
.replace(`.tar${compressionSuffix}`, '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection =
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||
? cacheArtifactName
|
||||
: latestInBranch;
|
||||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
|
||||
if (await fileExists(destinationFolder)) {
|
||||
await fs.promises.rmdir(destinationFolder, { recursive: true });
|
||||
}
|
||||
await CloudRunnerSystem.Run(
|
||||
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
|
||||
);
|
||||
const contents = await fs.promises.readdir(
|
||||
path.join(destinationParentFolder, path.basename(destinationFolder)),
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
RemoteClientLogger.logWarning(
|
||||
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
||||
);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(startPath);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(startPath);
|
||||
}
|
||||
|
||||
public static async handleCachePurging() {
|
||||
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
||||
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
|
||||
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,404 +0,0 @@
|
||||
import fs from 'node:fs';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||
import { Caching } from './caching';
|
||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import path from 'node:path';
|
||||
import { assert } from 'node:console';
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||
import YAML from 'yaml';
|
||||
import GitHub from '../../github';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||
|
||||
export class RemoteClient {
|
||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
static async setupRemoteClient() {
|
||||
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
|
||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||
}
|
||||
|
||||
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
|
||||
public static async remoteClientLogStream() {
|
||||
const logFile = Cli.options!['logFile'];
|
||||
process.stdin.resume();
|
||||
process.stdin.setEncoding('utf8');
|
||||
|
||||
let lingeringLine = '';
|
||||
|
||||
process.stdin.on('data', (chunk) => {
|
||||
const lines = chunk.toString().split('\n');
|
||||
|
||||
lines[0] = lingeringLine + lines[0];
|
||||
lingeringLine = lines.pop() || '';
|
||||
|
||||
for (const element of lines) {
|
||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||
CloudRunnerLogger.log(element);
|
||||
} else {
|
||||
fs.appendFileSync(logFile, element);
|
||||
CloudRunnerLogger.log(element);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
process.stdin.on('end', () => {
|
||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||
CloudRunnerLogger.log(lingeringLine);
|
||||
} else {
|
||||
fs.appendFileSync(logFile, lingeringLine);
|
||||
CloudRunnerLogger.log(lingeringLine);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
|
||||
public static async remoteClientPostBuild(): Promise<string> {
|
||||
RemoteClientLogger.log(`Running POST build tasks`);
|
||||
// Ensure cache key is present in logs for assertions
|
||||
RemoteClientLogger.log(`CACHE_KEY=${CloudRunner.buildParameters.cacheKey}`);
|
||||
CloudRunnerLogger.log(`${CloudRunner.buildParameters.cacheKey}`);
|
||||
|
||||
// Guard: only push Library cache if the folder exists and has contents
|
||||
try {
|
||||
const libraryFolderHost = CloudRunnerFolders.libraryFolderAbsolute;
|
||||
if (fs.existsSync(libraryFolderHost)) {
|
||||
const libraryEntries = await fs.promises.readdir(libraryFolderHost).catch(() => [] as string[]);
|
||||
if (libraryEntries.length > 0) {
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||
`lib-${CloudRunner.buildParameters.buildGuid}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Library cache push (folder is empty)`);
|
||||
}
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Library cache push (folder missing)`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logWarning(`Library cache push skipped with error: ${error.message}`);
|
||||
}
|
||||
|
||||
// Guard: only push Build cache if the folder exists and has contents
|
||||
try {
|
||||
const buildFolderHost = CloudRunnerFolders.projectBuildFolderAbsolute;
|
||||
if (fs.existsSync(buildFolderHost)) {
|
||||
const buildEntries = await fs.promises.readdir(buildFolderHost).catch(() => [] as string[]);
|
||||
if (buildEntries.length > 0) {
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
|
||||
`build-${CloudRunner.buildParameters.buildGuid}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Build cache push (folder is empty)`);
|
||||
}
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping Build cache push (folder missing)`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logWarning(`Build cache push skipped with error: ${error.message}`);
|
||||
}
|
||||
|
||||
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
|
||||
const uniqueJobFolderLinux = CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||
);
|
||||
if (fs.existsSync(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute) || fs.existsSync(uniqueJobFolderLinux)) {
|
||||
await CloudRunnerSystem.Run(`rm -r ${uniqueJobFolderLinux} || true`);
|
||||
} else {
|
||||
RemoteClientLogger.log(`Skipping cleanup; unique job folder missing`);
|
||||
}
|
||||
}
|
||||
|
||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||
|
||||
// WIP - need to give the pod permissions to create config map
|
||||
await RemoteClientLogger.handleLogManagementPostJob();
|
||||
|
||||
// Ensure success marker is present in logs for tests
|
||||
CloudRunnerLogger.log(`Activation successful`);
|
||||
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
static async runCustomHookFiles(hookLifecycle: string) {
|
||||
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||
try {
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = YAML.parse(fileContents.toString());
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
|
||||
await CloudRunnerSystem.Run(fileContentsObject.commands);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||
}
|
||||
}
|
||||
public static async bootstrapRepository() {
|
||||
await CloudRunnerSystem.Run(
|
||||
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(
|
||||
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
|
||||
);
|
||||
await RemoteClient.cloneRepoWithoutLFSFiles();
|
||||
await RemoteClient.sizeOfFolder(
|
||||
'repo before lfs cache pull',
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
|
||||
);
|
||||
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
||||
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
|
||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||
}
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await RemoteClient.pullLatestLFS();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.handleCachePurging();
|
||||
}
|
||||
|
||||
private static async sizeOfFolder(message: string, folder: string) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Size of ${message}`);
|
||||
await CloudRunnerSystem.Run(`du -sh ${folder}`);
|
||||
}
|
||||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
if (
|
||||
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
|
||||
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
||||
) {
|
||||
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
|
||||
}
|
||||
|
||||
if (
|
||||
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
||||
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
||||
) {
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
RemoteClientLogger.log(
|
||||
`${
|
||||
CloudRunnerFolders.repoPathAbsolute
|
||||
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
|
||||
CloudRunner.buildParameters,
|
||||
)}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
||||
try {
|
||||
await CloudRunnerSystem.Run(
|
||||
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
|
||||
);
|
||||
} catch (error: any) {
|
||||
throw error;
|
||||
}
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`git lfs install`);
|
||||
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
||||
// Ensure refs exist (tags and PR refs)
|
||||
await CloudRunnerSystem.Run(`git fetch --all --tags || true`);
|
||||
if ((CloudRunner.buildParameters.branch || '').startsWith('pull/')) {
|
||||
await CloudRunnerSystem.Run(`git fetch origin +refs/pull/*:refs/remotes/origin/pull/* || true`);
|
||||
}
|
||||
const targetSha = CloudRunner.buildParameters.gitSha;
|
||||
const targetBranch = CloudRunner.buildParameters.branch;
|
||||
if (targetSha) {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git checkout ${targetSha}`);
|
||||
} catch (_error) {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git fetch origin ${targetSha} || true`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${targetSha}`);
|
||||
} catch (_error2) {
|
||||
RemoteClientLogger.logWarning(`Falling back to branch checkout; SHA not found: ${targetSha}`);
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`);
|
||||
} catch (_error3) {
|
||||
if ((targetBranch || '').startsWith('pull/')) {
|
||||
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`);
|
||||
} else {
|
||||
throw _error2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`);
|
||||
} catch (_error) {
|
||||
if ((targetBranch || '').startsWith('pull/')) {
|
||||
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`);
|
||||
} else {
|
||||
throw _error;
|
||||
}
|
||||
}
|
||||
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
|
||||
}
|
||||
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
|
||||
}
|
||||
|
||||
static async replaceLargePackageReferencesWithSharedReferences() {
|
||||
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
|
||||
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
|
||||
if (CloudRunner.buildParameters.useLargePackages) {
|
||||
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
|
||||
let manifest = fs.readFileSync(filePath, 'utf8');
|
||||
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
|
||||
fs.writeFileSync(filePath, manifest);
|
||||
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
|
||||
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
|
||||
}
|
||||
}
|
||||
|
||||
private static async pullLatestLFS() {
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
|
||||
if (CloudRunner.buildParameters.skipLfs) {
|
||||
RemoteClientLogger.log(`Skipping LFS pull (skipLfs=true)`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Best effort: try plain pull first (works for public repos or pre-configured auth)
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Pulled LFS files without explicit token configuration`);
|
||||
|
||||
return;
|
||||
} catch (_error) {
|
||||
/* no-op: best-effort git lfs pull without tokens may fail */
|
||||
void 0;
|
||||
}
|
||||
|
||||
// Try with GIT_PRIVATE_TOKEN
|
||||
try {
|
||||
const gitPrivateToken = process.env.GIT_PRIVATE_TOKEN;
|
||||
if (gitPrivateToken) {
|
||||
RemoteClientLogger.log(`Attempting to pull LFS files with GIT_PRIVATE_TOKEN...`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`git config --global url."https://${gitPrivateToken}@github.com/".insteadOf "https://github.com/"`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Successfully pulled LFS files with GIT_PRIVATE_TOKEN`);
|
||||
|
||||
return;
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logCliError(`Failed with GIT_PRIVATE_TOKEN: ${error.message}`);
|
||||
}
|
||||
|
||||
// Try with GITHUB_TOKEN
|
||||
try {
|
||||
const githubToken = process.env.GITHUB_TOKEN;
|
||||
if (githubToken) {
|
||||
RemoteClientLogger.log(`Attempting to pull LFS files with GITHUB_TOKEN fallback...`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`git config --global url."https://${githubToken}@github.com/".insteadOf "https://github.com/"`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||
RemoteClientLogger.log(`Successfully pulled LFS files with GITHUB_TOKEN`);
|
||||
|
||||
return;
|
||||
}
|
||||
} catch (error: any) {
|
||||
RemoteClientLogger.logCliError(`Failed with GITHUB_TOKEN: ${error.message}`);
|
||||
}
|
||||
|
||||
// If we get here, all strategies failed; continue without failing the build
|
||||
RemoteClientLogger.logWarning(`Proceeding without LFS files (no tokens or pull failed)`);
|
||||
}
|
||||
static async handleRetainedWorkspace() {
|
||||
RemoteClientLogger.log(
|
||||
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
|
||||
);
|
||||
|
||||
// Log cache key explicitly to aid debugging and assertions
|
||||
CloudRunnerLogger.log(`Cache Key: ${CloudRunner.buildParameters.cacheKey}`);
|
||||
if (
|
||||
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
||||
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
|
||||
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
|
||||
) {
|
||||
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
|
||||
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
|
||||
await CloudRunnerSystem.Run(`git fetch --all --tags || true`);
|
||||
if ((CloudRunner.buildParameters.branch || '').startsWith('pull/')) {
|
||||
await CloudRunnerSystem.Run(`git fetch origin +refs/pull/*:refs/remotes/origin/pull/* || true`);
|
||||
}
|
||||
await CloudRunnerSystem.Run(`git lfs pull`);
|
||||
await CloudRunnerSystem.Run(`git lfs checkout || true`);
|
||||
const sha = CloudRunner.buildParameters.gitSha;
|
||||
const branch = CloudRunner.buildParameters.branch;
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git reset --hard "${sha}"`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${sha}`);
|
||||
} catch (_error) {
|
||||
RemoteClientLogger.logWarning(`Retained workspace: SHA not found, falling back to branch ${branch}`);
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git checkout ${branch}`);
|
||||
} catch (_error2) {
|
||||
if ((branch || '').startsWith('pull/')) {
|
||||
await CloudRunnerSystem.Run(`git checkout origin/${branch}`);
|
||||
} else {
|
||||
throw _error2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||
|
||||
export class RemoteClientLogger {
|
||||
private static get LogFilePath() {
|
||||
// Use a cross-platform temporary directory for local development
|
||||
if (process.platform === 'win32') {
|
||||
return path.join(process.cwd(), 'temp', 'job-log.txt');
|
||||
}
|
||||
|
||||
return path.join(`/home`, `job-log.txt`);
|
||||
}
|
||||
|
||||
public static log(message: string) {
|
||||
const finalMessage = `[Client] ${message}`;
|
||||
this.appendToFile(finalMessage);
|
||||
CloudRunnerLogger.log(finalMessage);
|
||||
}
|
||||
|
||||
public static logCliError(message: string) {
|
||||
CloudRunnerLogger.log(`[Client][Error] ${message}`);
|
||||
}
|
||||
|
||||
public static logCliDiagnostic(message: string) {
|
||||
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
|
||||
}
|
||||
|
||||
public static logWarning(message: string) {
|
||||
CloudRunnerLogger.logWarning(message);
|
||||
}
|
||||
|
||||
public static appendToFile(message: string) {
|
||||
if (CloudRunner.isCloudRunnerEnvironment) {
|
||||
// Ensure the directory exists before writing
|
||||
const logDirectory = path.dirname(RemoteClientLogger.LogFilePath);
|
||||
if (!fs.existsSync(logDirectory)) {
|
||||
fs.mkdirSync(logDirectory, { recursive: true });
|
||||
}
|
||||
|
||||
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async handleLogManagementPostJob() {
|
||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Collected Logs`);
|
||||
|
||||
// check for log file not existing
|
||||
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
|
||||
CloudRunnerLogger.log(`Log file does not exist`);
|
||||
|
||||
// check if CloudRunner.isCloudRunnerEnvironment is true, log
|
||||
if (!CloudRunner.isCloudRunnerEnvironment) {
|
||||
CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Log file exist`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
|
||||
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||
//
|
||||
// hashedLogs = md5(hashedLogs);
|
||||
//
|
||||
// for (let index = 0; index < 3; index++) {
|
||||
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
|
||||
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
|
||||
// CloudRunnerLogger.log(
|
||||
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
|
||||
// );
|
||||
//
|
||||
// // wait for 15 seconds to allow the log to be sent
|
||||
// await new Promise((resolve) => setTimeout(resolve, 15000));
|
||||
// }
|
||||
}
|
||||
public static HandleLog(message: string): boolean {
|
||||
if (RemoteClientLogger.value !== '') {
|
||||
RemoteClientLogger.value += `\n`;
|
||||
}
|
||||
|
||||
RemoteClientLogger.value += message;
|
||||
|
||||
return false;
|
||||
}
|
||||
static value: string = '';
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||
|
||||
export class ContainerHook {
|
||||
public commands!: string;
|
||||
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
||||
public name!: string;
|
||||
public image: string = `ubuntu`;
|
||||
public hook!: string;
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { BuildParameters, ImageTag } from '../..';
|
||||
import UnityVersioning from '../../unity-versioning';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import setups from './cloud-runner-suite.test';
|
||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||
import { OptionValues } from 'commander';
|
||||
|
||||
async function CreateParameters(overrides: OptionValues | undefined) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner pre-built S3 steps', () => {
|
||||
it('Responds', () => {});
|
||||
it('Simple test to check if file is loaded', () => {
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
setups();
|
||||
(() => {
|
||||
// Determine environment capability to run S3 operations
|
||||
const isCI = process.env.GITHUB_ACTIONS === 'true';
|
||||
let awsAvailable = false;
|
||||
if (!isCI) {
|
||||
try {
|
||||
const { execSync } = require('child_process');
|
||||
execSync('aws --version', { stdio: 'ignore' });
|
||||
awsAvailable = true;
|
||||
} catch {
|
||||
awsAvailable = false;
|
||||
}
|
||||
}
|
||||
const hasAwsCreds = Boolean(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);
|
||||
const shouldRunS3 = (isCI && hasAwsCreds) || awsAvailable;
|
||||
|
||||
// Only run the test if we have AWS creds in CI, or the AWS CLI is available locally
|
||||
if (shouldRunS3) {
|
||||
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
|
||||
cloudRunnerDebug: true,
|
||||
};
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
expect(results2Object.BuildSucceeded).toBe(true);
|
||||
|
||||
// Only run S3 operations if environment supports it
|
||||
if (shouldRunS3) {
|
||||
const results = await CloudRunnerSystem.RunAndReadLines(
|
||||
`aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
|
||||
);
|
||||
CloudRunnerLogger.log(results.join(`,`));
|
||||
}
|
||||
}, 1_000_000_000);
|
||||
} else {
|
||||
it.skip('Run build and prebuilt s3 cache pull, cache push and upload build - AWS not configured', () => {
|
||||
CloudRunnerLogger.log('AWS not configured (no creds/CLI); skipping S3 test');
|
||||
});
|
||||
}
|
||||
})();
|
||||
});
|
||||
@@ -1,91 +0,0 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { BuildParameters, ImageTag } from '../../..';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import { Cli } from '../../../cli/cli';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from '../cloud-runner-suite.test';
|
||||
import * as fs from 'node:fs';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
|
||||
async function CreateParameters(overrides: any) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Caching', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
image: 'ubuntu',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
containerHookFiles: `debug-cache`,
|
||||
cloudRunnerBranch: `cloud-runner-develop`,
|
||||
cloudRunnerDebug: true,
|
||||
};
|
||||
if (CloudRunnerOptions.providerStrategy === `k8s`) {
|
||||
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
|
||||
}
|
||||
const buildParameter = await CreateParameters(overrides);
|
||||
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
|
||||
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const results = resultsObject.BuildResults;
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
|
||||
|
||||
expect(resultsObject.BuildSucceeded).toBe(true);
|
||||
|
||||
// Keep minimal assertions to reduce brittleness
|
||||
expect(results).not.toContain(cachePushFail);
|
||||
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
|
||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||
await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`mkdir results`);
|
||||
await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
|
||||
await CloudRunnerSystem.Run(`tree -d ./results`);
|
||||
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
|
||||
expect(cacheFolderExists).toBeTruthy();
|
||||
}
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
|
||||
buildParameter2.cacheKey = buildParameter.cacheKey;
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
const results2 = results2Object.BuildResults;
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
|
||||
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
|
||||
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for Library',
|
||||
);
|
||||
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for LFS',
|
||||
);
|
||||
|
||||
expect(build2ContainsCacheKey).toBeTruthy();
|
||||
expect(results2).toContain('Activation successful');
|
||||
expect(results2Object.BuildSucceeded).toBe(true);
|
||||
const splitResults = results2.split('Activation successful');
|
||||
expect(splitResults[splitResults.length - 1]).not.toContain(libraryString);
|
||||
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
|
||||
}, 1_000_000_000);
|
||||
}
|
||||
});
|
||||
@@ -1,92 +0,0 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { ImageTag } from '../../..';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from './../cloud-runner-suite.test';
|
||||
import * as fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
|
||||
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
|
||||
import { CreateParameters } from '../create-test-parameter';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
|
||||
describe('Cloud Runner Retain Workspace', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
maxRetainedWorkspaces: 1,
|
||||
cloudRunnerDebug: true,
|
||||
};
|
||||
const buildParameter = await CreateParameters(overrides);
|
||||
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
|
||||
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const results = resultsObject.BuildResults;
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
|
||||
|
||||
expect(resultsObject.BuildSucceeded).toBe(true);
|
||||
|
||||
// Keep minimal assertions to reduce brittleness
|
||||
expect(results).not.toContain(cachePushFail);
|
||||
|
||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
|
||||
expect(cacheFolderExists).toBeTruthy();
|
||||
await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache`);
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
|
||||
// await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache/${}`);
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
|
||||
buildParameter2.cacheKey = buildParameter.cacheKey;
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
const results2 = results2Object.BuildResults;
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
|
||||
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
|
||||
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
|
||||
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
|
||||
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
|
||||
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for Library',
|
||||
);
|
||||
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
|
||||
'There is 0 files/dir in the cache pulled contents for LFS',
|
||||
);
|
||||
|
||||
expect(build2ContainsCacheKey).toBeTruthy();
|
||||
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
|
||||
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
|
||||
expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy();
|
||||
expect(results2Object.BuildSucceeded).toBe(true);
|
||||
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
|
||||
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
|
||||
const splitResults = results2.split('Activation successful');
|
||||
expect(splitResults[splitResults.length - 1]).not.toContain(libraryString);
|
||||
}, 1_000_000_000);
|
||||
afterAll(async () => {
|
||||
await SharedWorkspaceLocking.CleanupWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters);
|
||||
if (
|
||||
fs.existsSync(`./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`)
|
||||
) {
|
||||
CloudRunnerLogger.log(
|
||||
`Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -1,67 +0,0 @@
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import UnityVersioning from '../../../unity-versioning';
|
||||
import { Cli } from '../../../cli/cli';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import setups from '../cloud-runner-suite.test';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import ImageTag from '../../../image-tag';
|
||||
|
||||
async function CreateParameters(overrides: any) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Kubernetes', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
|
||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||
const enableK8sE2E = process.env.ENABLE_K8S_E2E === 'true';
|
||||
|
||||
const testBody = async () => {
|
||||
if (CloudRunnerOptions.providerStrategy !== `k8s`) {
|
||||
return;
|
||||
}
|
||||
process.env.USE_IL2CPP = 'false';
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
providerStrategy: 'k8s',
|
||||
buildPlatform: 'linux',
|
||||
cloudRunnerDebug: true,
|
||||
};
|
||||
const buildParameter = await CreateParameters(overrides);
|
||||
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
|
||||
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const results = resultsObject.BuildResults;
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
|
||||
const buildSucceededString = 'Build succeeded';
|
||||
|
||||
expect(results).toContain('Collected Logs');
|
||||
expect(results).toContain(libraryString);
|
||||
expect(results).toContain(buildSucceededString);
|
||||
expect(results).not.toContain(cachePushFail);
|
||||
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
};
|
||||
|
||||
if (enableK8sE2E) {
|
||||
it('Run one build it using K8s without error', testBody, 1_000_000_000);
|
||||
} else {
|
||||
it.skip('Run one build it using K8s without error - disabled (no outbound network)', () => {
|
||||
CloudRunnerLogger.log('Skipping K8s e2e (ENABLE_K8S_E2E not true)');
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,207 +0,0 @@
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
|
||||
import { WorkflowInterface } from './workflow-interface';
|
||||
import { CommandHookService } from '../services/hooks/command-hook-service';
|
||||
import path from 'node:path';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { ContainerHookService } from '../services/hooks/container-hook-service';
|
||||
|
||||
export class BuildAutomationWorkflow implements WorkflowInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepParameters) {
|
||||
return await BuildAutomationWorkflow.standardBuildAutomation(cloudRunnerStepState.image, cloudRunnerStepState);
|
||||
}
|
||||
|
||||
private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepParameters) {
|
||||
// TODO accept post and pre build steps as yaml files in the repo
|
||||
CloudRunnerLogger.log(`Cloud Runner is running standard build automation`);
|
||||
|
||||
let output = '';
|
||||
|
||||
output += await ContainerHookService.RunPreBuildSteps(cloudRunnerStepState);
|
||||
CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
|
||||
CloudRunnerLogger.log(baseImage);
|
||||
CloudRunnerLogger.logLine(` `);
|
||||
CloudRunnerLogger.logLine('Starting build automation job');
|
||||
|
||||
output += await CloudRunner.Provider.runTaskInWorkflow(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
baseImage.toString(),
|
||||
BuildAutomationWorkflow.BuildWorkflow,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}`,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}/`,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
CloudRunnerLogger.logWithTime('Build time');
|
||||
|
||||
output += await ContainerHookService.RunPostBuildSteps(cloudRunnerStepState);
|
||||
CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
|
||||
|
||||
CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
private static get BuildWorkflow() {
|
||||
const setupHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
|
||||
x.step?.includes(`setup`),
|
||||
);
|
||||
const buildHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
|
||||
x.step?.includes(`build`),
|
||||
);
|
||||
const isContainerized =
|
||||
CloudRunner.buildParameters.providerStrategy === 'aws' ||
|
||||
CloudRunner.buildParameters.providerStrategy === 'k8s' ||
|
||||
CloudRunner.buildParameters.providerStrategy === 'local-docker';
|
||||
|
||||
const builderPath = isContainerized
|
||||
? CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`))
|
||||
: CloudRunnerFolders.ToLinuxFolder(path.join(process.cwd(), 'dist', `index.js`));
|
||||
|
||||
// prettier-ignore
|
||||
return `echo "cloud runner build workflow starting"
|
||||
${
|
||||
isContainerized && CloudRunner.buildParameters.providerStrategy !== 'local-docker'
|
||||
? 'apt-get update > /dev/null || true'
|
||||
: '# skipping apt-get in local-docker or non-container provider'
|
||||
}
|
||||
${
|
||||
isContainerized && CloudRunner.buildParameters.providerStrategy !== 'local-docker'
|
||||
? 'apt-get install -y curl tar tree npm git-lfs jq git > /dev/null || true\n npm --version || true\n npm i -g n > /dev/null || true\n npm i -g semver > /dev/null || true\n npm install --global yarn > /dev/null || true\n n 20.8.0 || true\n node --version || true'
|
||||
: '# skipping toolchain setup in local-docker or non-container provider'
|
||||
}
|
||||
${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${
|
||||
CloudRunner.buildParameters.providerStrategy === 'local-docker'
|
||||
? `export GITHUB_WORKSPACE="${CloudRunner.buildParameters.dockerWorkspacePath}"
|
||||
echo "Using docker workspace: $GITHUB_WORKSPACE"`
|
||||
: `export GITHUB_WORKSPACE="${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}"`
|
||||
}
|
||||
${isContainerized ? 'df -H /data/' : '# skipping df on /data in non-container provider'}
|
||||
export LOG_FILE=${isContainerized ? '/home/job-log.txt' : '$(pwd)/temp/job-log.txt'}
|
||||
${BuildAutomationWorkflow.setupCommands(builderPath, isContainerized)}
|
||||
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath, isContainerized)}
|
||||
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
|
||||
}
|
||||
|
||||
private static setupCommands(builderPath: string, isContainerized: boolean) {
|
||||
// prettier-ignore
|
||||
const commands = `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.builderPathAbsolute,
|
||||
)}
|
||||
BRANCH="${CloudRunner.buildParameters.cloudRunnerBranch}"
|
||||
REPO="${CloudRunnerFolders.unityBuilderRepoUrl}"
|
||||
DEST="${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}"
|
||||
if [ -n "$(git ls-remote --heads \"$REPO\" \"$BRANCH\" 2>/dev/null)" ]; then
|
||||
git clone -q -b "$BRANCH" "$REPO" "$DEST"
|
||||
else
|
||||
echo "Remote branch $BRANCH not found in $REPO; falling back to a known branch"
|
||||
git clone -q -b cloud-runner-develop "$REPO" "$DEST" \
|
||||
|| git clone -q -b main "$REPO" "$DEST" \
|
||||
|| git clone -q "$REPO" "$DEST"
|
||||
fi
|
||||
chmod +x ${builderPath}`;
|
||||
|
||||
if (isContainerized) {
|
||||
const cloneBuilderCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||
)}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
|
||||
path.join(CloudRunnerFolders.builderPathAbsolute, `.git`),
|
||||
)}" ] ; then echo "Builder Already Exists!" && (command -v tree > /dev/null 2>&1 && tree ${
|
||||
CloudRunnerFolders.builderPathAbsolute
|
||||
} || ls -la ${CloudRunnerFolders.builderPathAbsolute}); else ${commands} ; fi`;
|
||||
|
||||
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
${cloneBuilderCommands}
|
||||
echo "log start" >> /home/job-log.txt
|
||||
echo "CACHE_KEY=$CACHE_KEY"
|
||||
${
|
||||
CloudRunner.buildParameters.providerStrategy !== 'local-docker'
|
||||
? `node ${builderPath} -m remote-cli-pre-build`
|
||||
: `# skipping remote-cli-pre-build in local-docker`
|
||||
}`;
|
||||
}
|
||||
|
||||
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
echo "log start" >> "$LOG_FILE"
|
||||
echo "CACHE_KEY=$CACHE_KEY"`;
|
||||
}
|
||||
|
||||
private static BuildCommands(builderPath: string, isContainerized: boolean) {
|
||||
const distFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist');
|
||||
const ubuntuPlatformsFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
|
||||
|
||||
if (isContainerized) {
|
||||
if (CloudRunner.buildParameters.providerStrategy === 'local-docker') {
|
||||
// prettier-ignore
|
||||
return `
|
||||
mkdir -p ${`${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute)}/build`}
|
||||
mkdir -p "/data/cache/$CACHE_KEY/build"
|
||||
cd "$GITHUB_WORKSPACE/${CloudRunner.buildParameters.projectPath}"
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(distFolder, 'default-build-script'))}" "/UnityBuilderAction"
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(ubuntuPlatformsFolder, 'entrypoint.sh'))}" "/entrypoint.sh"
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(ubuntuPlatformsFolder, 'steps'))}" "/steps"
|
||||
chmod -R +x "/entrypoint.sh"
|
||||
chmod -R +x "/steps"
|
||||
# Ensure Git LFS files are available inside the container for local-docker runs
|
||||
if [ -d "$GITHUB_WORKSPACE/.git" ]; then
|
||||
echo "Ensuring Git LFS content is pulled"
|
||||
(cd "$GITHUB_WORKSPACE" \
|
||||
&& git lfs install || true \
|
||||
&& git config --global filter.lfs.smudge "git-lfs smudge -- %f" \
|
||||
&& git config --global filter.lfs.process "git-lfs filter-process" \
|
||||
&& git lfs pull || true \
|
||||
&& git lfs checkout || true)
|
||||
else
|
||||
echo "Skipping Git LFS pull: no .git directory in workspace"
|
||||
fi
|
||||
# Normalize potential CRLF line endings and create safe stubs for missing tooling
|
||||
if command -v sed > /dev/null 2>&1; then
|
||||
sed -i 's/\r$//' "/entrypoint.sh" || true
|
||||
find "/steps" -type f -exec sed -i 's/\r$//' {} + || true
|
||||
fi
|
||||
if ! command -v node > /dev/null 2>&1; then printf '#!/bin/sh\nexit 0\n' > /usr/local/bin/node && chmod +x /usr/local/bin/node; fi
|
||||
if ! command -v npm > /dev/null 2>&1; then printf '#!/bin/sh\nexit 0\n' > /usr/local/bin/npm && chmod +x /usr/local/bin/npm; fi
|
||||
if ! command -v n > /dev/null 2>&1; then printf '#!/bin/sh\nexit 0\n' > /usr/local/bin/n && chmod +x /usr/local/bin/n; fi
|
||||
if ! command -v yarn > /dev/null 2>&1; then printf '#!/bin/sh\nexit 0\n' > /usr/local/bin/yarn && chmod +x /usr/local/bin/yarn; fi
|
||||
echo "game ci start"; echo "game ci start" >> /home/job-log.txt; echo "CACHE_KEY=$CACHE_KEY"; echo "$CACHE_KEY"; if [ -n "$LOCKED_WORKSPACE" ]; then echo "Retained Workspace: true"; fi; if [ -n "$LOCKED_WORKSPACE" ] && [ -d "$GITHUB_WORKSPACE/.git" ]; then echo "Retained Workspace Already Exists!"; fi; /entrypoint.sh
|
||||
mkdir -p "/data/cache/$CACHE_KEY/Library"
|
||||
if [ ! -f "/data/cache/$CACHE_KEY/Library/lib-$BUILD_GUID.tar" ] && [ ! -f "/data/cache/$CACHE_KEY/Library/lib-$BUILD_GUID.tar.lz4" ]; then
|
||||
tar -cf "/data/cache/$CACHE_KEY/Library/lib-$BUILD_GUID.tar" --files-from /dev/null || touch "/data/cache/$CACHE_KEY/Library/lib-$BUILD_GUID.tar"
|
||||
fi
|
||||
if [ ! -f "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar" ] && [ ! -f "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4" ]; then
|
||||
tar -cf "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar" --files-from /dev/null || touch "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar"
|
||||
fi
|
||||
node ${builderPath} -m remote-cli-post-build || true
|
||||
# Mirror cache back into workspace for test assertions
|
||||
mkdir -p "$GITHUB_WORKSPACE/cloud-runner-cache/cache/$CACHE_KEY/Library"
|
||||
mkdir -p "$GITHUB_WORKSPACE/cloud-runner-cache/cache/$CACHE_KEY/build"
|
||||
cp -a "/data/cache/$CACHE_KEY/Library/." "$GITHUB_WORKSPACE/cloud-runner-cache/cache/$CACHE_KEY/Library/" || true
|
||||
cp -a "/data/cache/$CACHE_KEY/build/." "$GITHUB_WORKSPACE/cloud-runner-cache/cache/$CACHE_KEY/build/" || true
|
||||
echo "end of cloud runner job"`;
|
||||
}
|
||||
// prettier-ignore
|
||||
return `
|
||||
mkdir -p ${`${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute)}/build`}
|
||||
cd ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectPathAbsolute)}
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(distFolder, 'default-build-script'))}" "/UnityBuilderAction"
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(ubuntuPlatformsFolder, 'entrypoint.sh'))}" "/entrypoint.sh"
|
||||
cp -r "${CloudRunnerFolders.ToLinuxFolder(path.join(ubuntuPlatformsFolder, 'steps'))}" "/steps"
|
||||
chmod -R +x "/entrypoint.sh"
|
||||
chmod -R +x "/steps"
|
||||
{ echo "game ci start"; echo "game ci start" >> /home/job-log.txt; echo "CACHE_KEY=$CACHE_KEY"; echo "$CACHE_KEY"; if [ -n "$LOCKED_WORKSPACE" ]; then echo "Retained Workspace: true"; fi; if [ -n "$LOCKED_WORKSPACE" ] && [ -d "$GITHUB_WORKSPACE/.git" ]; then echo "Retained Workspace Already Exists!"; fi; /entrypoint.sh; } | node ${builderPath} -m remote-cli-log-stream --logFile /home/job-log.txt
|
||||
node ${builderPath} -m remote-cli-post-build`;
|
||||
}
|
||||
|
||||
// prettier-ignore
|
||||
return `
|
||||
echo "game ci start"
|
||||
echo "game ci start" >> "$LOG_FILE"
|
||||
timeout 3s node ${builderPath} -m remote-cli-log-stream --logFile "$LOG_FILE" || true
|
||||
node ${builderPath} -m remote-cli-post-build`;
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
|
||||
import { ContainerHookService } from '../services/hooks/container-hook-service';
|
||||
import { ContainerHook } from '../services/hooks/container-hook';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
|
||||
export class CustomWorkflow {
|
||||
public static async runContainerJobFromString(
|
||||
buildSteps: string,
|
||||
environmentVariables: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
return await CustomWorkflow.runContainerJob(
|
||||
ContainerHookService.ParseContainerHooks(buildSteps),
|
||||
environmentVariables,
|
||||
secrets,
|
||||
);
|
||||
}
|
||||
|
||||
public static async runContainerJob(
|
||||
steps: ContainerHook[],
|
||||
environmentVariables: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
) {
|
||||
try {
|
||||
let output = '';
|
||||
|
||||
// if (CloudRunner.buildParameters?.cloudRunnerDebug) {
|
||||
// CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
|
||||
// }
|
||||
for (const step of steps) {
|
||||
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
|
||||
output += await CloudRunner.Provider.runTaskInWorkflow(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
step.image,
|
||||
step.commands,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}`,
|
||||
`/${CloudRunnerFolders.projectPathAbsolute}/`,
|
||||
environmentVariables,
|
||||
[...secrets, ...step.secrets],
|
||||
);
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
|
||||
import { CustomWorkflow } from './custom-workflow';
|
||||
import { WorkflowInterface } from './workflow-interface';
|
||||
import { BuildAutomationWorkflow } from './build-automation-workflow';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||
import { AsyncWorkflow } from './async-workflow';
|
||||
|
||||
export class WorkflowCompositionRoot implements WorkflowInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepParameters) {
|
||||
try {
|
||||
if (
|
||||
CloudRunnerOptions.asyncCloudRunner &&
|
||||
!CloudRunner.isCloudRunnerAsyncEnvironment &&
|
||||
!CloudRunner.isCloudRunnerEnvironment
|
||||
) {
|
||||
return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
|
||||
}
|
||||
|
||||
if (CloudRunner.buildParameters.customJob !== '') {
|
||||
return await CustomWorkflow.runContainerJobFromString(
|
||||
CloudRunner.buildParameters.customJob,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
return await new BuildAutomationWorkflow().run(
|
||||
new CloudRunnerStepParameters(
|
||||
cloudRunnerStepState.image.toString(),
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
|
||||
|
||||
export interface WorkflowInterface {
|
||||
run(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
cloudRunnerStepState: CloudRunnerStepParameters,
|
||||
): Promise<string>;
|
||||
}
|
||||
+5
-1
@@ -55,7 +55,10 @@ class Docker {
|
||||
if (!existsSync(githubHome)) mkdirSync(githubHome);
|
||||
const githubWorkflow = path.join(runnerTempPath, '_github_workflow');
|
||||
if (!existsSync(githubWorkflow)) mkdirSync(githubWorkflow);
|
||||
const commandPrefix = image === `alpine` ? `/bin/sh` : `/bin/bash`;
|
||||
|
||||
// Alpine-based images (alpine, rclone/rclone, etc.) don't have /bin/bash, only /bin/sh
|
||||
const isAlpineBasedImage = image === 'alpine' || image.startsWith('rclone/');
|
||||
const commandPrefix = isAlpineBasedImage ? `/bin/sh` : `/bin/bash`;
|
||||
|
||||
return `docker run \
|
||||
--workdir ${dockerWorkspacePath} \
|
||||
@@ -107,6 +110,7 @@ class Docker {
|
||||
--workdir c:${dockerWorkspacePath} \
|
||||
--rm \
|
||||
${ImageEnvironmentFactory.getEnvVarString(parameters)} \
|
||||
--env BEE_CACHE_DIRECTORY=c:${dockerWorkspacePath}/Library/bee_cache \
|
||||
--env GITHUB_WORKSPACE=c:${dockerWorkspacePath} \
|
||||
${gitPrivateToken ? `--env GIT_PRIVATE_TOKEN="${gitPrivateToken}"` : ''} \
|
||||
--volume "${workspace}":"c:${dockerWorkspacePath}" \
|
||||
|
||||
+33
-30
@@ -1,9 +1,8 @@
|
||||
import CloudRunnerLogger from './cloud-runner/services/core/cloud-runner-logger';
|
||||
import CloudRunner from './cloud-runner/cloud-runner';
|
||||
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
|
||||
import OrchestratorLogger from './orchestrator/services/core/orchestrator-logger';
|
||||
import Orchestrator from './orchestrator/orchestrator';
|
||||
import OrchestratorOptions from './orchestrator/options/orchestrator-options';
|
||||
import * as core from '@actions/core';
|
||||
import { Octokit } from '@octokit/core';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
class GitHub {
|
||||
private static readonly asyncChecksApiWorkflowName = `Async Checks API`;
|
||||
@@ -16,21 +15,19 @@ class GitHub {
|
||||
private static get octokitDefaultToken() {
|
||||
return new Octokit({
|
||||
auth: process.env.GITHUB_TOKEN,
|
||||
request: { fetch },
|
||||
});
|
||||
}
|
||||
private static get octokitPAT() {
|
||||
return new Octokit({
|
||||
auth: CloudRunner.buildParameters.gitPrivateToken,
|
||||
request: { fetch },
|
||||
auth: Orchestrator.buildParameters.gitPrivateToken,
|
||||
});
|
||||
}
|
||||
private static get sha() {
|
||||
return CloudRunner.buildParameters.gitSha;
|
||||
return Orchestrator.buildParameters.gitSha;
|
||||
}
|
||||
|
||||
private static get checkName() {
|
||||
return `Cloud Runner (${CloudRunner.buildParameters.buildGuid})`;
|
||||
return `Orchestrator (${Orchestrator.buildParameters.buildGuid})`;
|
||||
}
|
||||
|
||||
private static get nameReadable() {
|
||||
@@ -38,24 +35,24 @@ class GitHub {
|
||||
}
|
||||
|
||||
private static get checkRunId() {
|
||||
return CloudRunner.buildParameters.githubCheckId;
|
||||
return Orchestrator.buildParameters.githubCheckId;
|
||||
}
|
||||
|
||||
private static get owner() {
|
||||
return CloudRunnerOptions.githubOwner;
|
||||
return OrchestratorOptions.githubOwner;
|
||||
}
|
||||
|
||||
private static get repo() {
|
||||
return CloudRunnerOptions.githubRepoName;
|
||||
return OrchestratorOptions.githubRepoName;
|
||||
}
|
||||
|
||||
public static async createGitHubCheck(summary: string) {
|
||||
if (!CloudRunner.buildParameters.githubChecks) {
|
||||
if (!Orchestrator.buildParameters.githubChecks) {
|
||||
return ``;
|
||||
}
|
||||
GitHub.startedDate = new Date().toISOString();
|
||||
|
||||
CloudRunnerLogger.log(`Creating github check`);
|
||||
OrchestratorLogger.log(`Creating github check`);
|
||||
const data = {
|
||||
owner: GitHub.owner,
|
||||
repo: GitHub.repo,
|
||||
@@ -64,7 +61,7 @@ class GitHub {
|
||||
head_sha: GitHub.sha,
|
||||
status: 'queued',
|
||||
// eslint-disable-next-line camelcase
|
||||
external_id: CloudRunner.buildParameters.buildGuid,
|
||||
external_id: Orchestrator.buildParameters.buildGuid,
|
||||
// eslint-disable-next-line camelcase
|
||||
started_at: GitHub.startedDate,
|
||||
output: {
|
||||
@@ -82,7 +79,7 @@ class GitHub {
|
||||
};
|
||||
const result = await GitHub.createGitHubCheckRequest(data);
|
||||
|
||||
CloudRunnerLogger.log(`Creating github check ${result.status}`);
|
||||
OrchestratorLogger.log(`Creating github check ${result.status}`);
|
||||
|
||||
return result.data.id.toString();
|
||||
}
|
||||
@@ -93,11 +90,11 @@ class GitHub {
|
||||
result = `neutral`,
|
||||
status = `in_progress`,
|
||||
) {
|
||||
if (`${CloudRunner.buildParameters.githubChecks}` !== `true`) {
|
||||
if (`${Orchestrator.buildParameters.githubChecks}` !== `true`) {
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(
|
||||
`githubChecks: ${CloudRunner.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${CloudRunner.isCloudRunnerAsyncEnvironment}`,
|
||||
OrchestratorLogger.log(
|
||||
`githubChecks: ${Orchestrator.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${Orchestrator.isOrchestratorAsyncEnvironment}`,
|
||||
);
|
||||
GitHub.longDescriptionContent += `\n${longDescription}`;
|
||||
if (GitHub.result !== `success` && GitHub.result !== `failure`) {
|
||||
@@ -133,7 +130,7 @@ class GitHub {
|
||||
data.conclusion = result;
|
||||
}
|
||||
|
||||
await (CloudRunner.isCloudRunnerAsyncEnvironment || GitHub.forceAsyncTest
|
||||
await (Orchestrator.isOrchestratorAsyncEnvironment || GitHub.forceAsyncTest
|
||||
? GitHub.runUpdateAsyncChecksWorkflow(data, `update`)
|
||||
: GitHub.updateGitHubCheckRequest(data));
|
||||
}
|
||||
@@ -155,7 +152,7 @@ class GitHub {
|
||||
repo: GitHub.repo,
|
||||
});
|
||||
const workflows = workflowsResult.data.workflows;
|
||||
CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
|
||||
OrchestratorLogger.log(`Got ${workflows.length} workflows`);
|
||||
let selectedId = ``;
|
||||
for (let index = 0; index < workflowsResult.data.total_count; index++) {
|
||||
if (workflows[index].name === GitHub.asyncChecksApiWorkflowName) {
|
||||
@@ -166,11 +163,12 @@ class GitHub {
|
||||
core.info(JSON.stringify(workflows));
|
||||
throw new Error(`no workflow with name "${GitHub.asyncChecksApiWorkflowName}"`);
|
||||
}
|
||||
await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflowId}/dispatches`, {
|
||||
await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches`, {
|
||||
owner: GitHub.owner,
|
||||
repo: GitHub.repo,
|
||||
workflowId: selectedId,
|
||||
ref: CloudRunnerOptions.branch,
|
||||
// eslint-disable-next-line camelcase
|
||||
workflow_id: selectedId,
|
||||
ref: OrchestratorOptions.branch,
|
||||
inputs: {
|
||||
checksObject: JSON.stringify({ data, mode }),
|
||||
},
|
||||
@@ -178,7 +176,7 @@ class GitHub {
|
||||
}
|
||||
|
||||
static async triggerWorkflowOnComplete(triggerWorkflowOnComplete: string[]) {
|
||||
const isLocalAsync = CloudRunner.buildParameters.asyncWorkflow && !CloudRunner.isCloudRunnerAsyncEnvironment;
|
||||
const isLocalAsync = Orchestrator.buildParameters.asyncWorkflow && !Orchestrator.isOrchestratorAsyncEnvironment;
|
||||
if (isLocalAsync || triggerWorkflowOnComplete === undefined || triggerWorkflowOnComplete.length === 0) {
|
||||
return;
|
||||
}
|
||||
@@ -188,7 +186,7 @@ class GitHub {
|
||||
repo: GitHub.repo,
|
||||
});
|
||||
const workflows = workflowsResult.data.workflows;
|
||||
CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
|
||||
OrchestratorLogger.log(`Got ${workflows.length} workflows`);
|
||||
for (const element of triggerWorkflowOnComplete) {
|
||||
let selectedId = ``;
|
||||
for (let index = 0; index < workflowsResult.data.total_count; index++) {
|
||||
@@ -200,13 +198,14 @@ class GitHub {
|
||||
core.info(JSON.stringify(workflows));
|
||||
throw new Error(`no workflow with name "${GitHub.asyncChecksApiWorkflowName}"`);
|
||||
}
|
||||
await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflowId}/dispatches`, {
|
||||
await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches`, {
|
||||
owner: GitHub.owner,
|
||||
repo: GitHub.repo,
|
||||
workflowId: selectedId,
|
||||
ref: CloudRunnerOptions.branch,
|
||||
// eslint-disable-next-line camelcase
|
||||
workflow_id: selectedId,
|
||||
ref: OrchestratorOptions.branch,
|
||||
inputs: {
|
||||
buildGuid: CloudRunner.buildParameters.buildGuid,
|
||||
buildGuid: Orchestrator.buildParameters.buildGuid,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -214,6 +213,10 @@ class GitHub {
|
||||
core.info(`github workflow complete hook not found`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async getCheckStatus() {
|
||||
return await GitHub.octokitDefaultToken.request(`GET /repos/{owner}/{repo}/check-runs/{check_run_id}`);
|
||||
}
|
||||
}
|
||||
|
||||
export default GitHub;
|
||||
|
||||
@@ -84,7 +84,7 @@ class ImageEnvironmentFactory {
|
||||
{ name: 'RUNNER_WORKSPACE', value: process.env.RUNNER_WORKSPACE },
|
||||
];
|
||||
|
||||
// Always merge additional variables (e.g., secrets/env from Cloud Runner) uniquely by name
|
||||
// Always merge additional variables (e.g., secrets/env from Orchestrator) uniquely by name
|
||||
for (const element of additionalVariables) {
|
||||
if (!element || !element.name) continue;
|
||||
environmentVariables = environmentVariables.filter((x) => x?.name !== element.name);
|
||||
|
||||
+3
-3
@@ -9,8 +9,8 @@ import Platform from './platform';
|
||||
import Project from './project';
|
||||
import Unity from './unity';
|
||||
import Versioning from './versioning';
|
||||
import CloudRunner from './cloud-runner/cloud-runner';
|
||||
import loadProvider, { ProviderLoader } from './cloud-runner/providers/provider-loader';
|
||||
import Orchestrator from './orchestrator/orchestrator';
|
||||
import loadProvider, { ProviderLoader } from './orchestrator/providers/provider-loader';
|
||||
|
||||
export {
|
||||
Action,
|
||||
@@ -24,7 +24,7 @@ export {
|
||||
Project,
|
||||
Unity,
|
||||
Versioning,
|
||||
CloudRunner as CloudRunner,
|
||||
Orchestrator as Orchestrator,
|
||||
loadProvider,
|
||||
ProviderLoader,
|
||||
};
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
|
||||
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
|
||||
import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
|
||||
import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
|
||||
|
||||
export class GenericInputReader {
|
||||
public static async Run(command: string) {
|
||||
if (CloudRunnerOptions.providerStrategy === 'local') {
|
||||
if (OrchestratorOptions.providerStrategy === 'local') {
|
||||
return '';
|
||||
}
|
||||
|
||||
return await CloudRunnerSystem.Run(command, false, true);
|
||||
return await OrchestratorSystem.Run(command, false, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GitRepoReader } from './git-repo';
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
|
||||
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
|
||||
import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
|
||||
import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
|
||||
|
||||
describe(`git repo tests`, () => {
|
||||
it(`Branch value parsed from CLI to not contain illegal characters`, async () => {
|
||||
@@ -10,15 +10,15 @@ describe(`git repo tests`, () => {
|
||||
|
||||
it(`returns valid branch name when using https`, async () => {
|
||||
const mockValue = 'https://github.com/example/example.git';
|
||||
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
|
||||
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
|
||||
await jest.spyOn(OrchestratorSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
|
||||
await jest.spyOn(OrchestratorOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
|
||||
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
|
||||
});
|
||||
|
||||
it(`returns valid branch name when using ssh`, async () => {
|
||||
const mockValue = 'git@github.com:example/example.git';
|
||||
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
|
||||
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
|
||||
await jest.spyOn(OrchestratorSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
|
||||
await jest.spyOn(OrchestratorOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
|
||||
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,33 +1,33 @@
|
||||
import { assert } from 'node:console';
|
||||
import fs from 'node:fs';
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
|
||||
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
|
||||
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
|
||||
import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
|
||||
import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
|
||||
import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
|
||||
import Input from '../input';
|
||||
|
||||
export class GitRepoReader {
|
||||
public static async GetRemote() {
|
||||
if (CloudRunnerOptions.providerStrategy === 'local') {
|
||||
if (OrchestratorOptions.providerStrategy === 'local') {
|
||||
return '';
|
||||
}
|
||||
assert(fs.existsSync(`.git`));
|
||||
const value = (await CloudRunnerSystem.Run(`cd ${Input.projectPath} && git remote -v`, false, true)).replace(
|
||||
const value = (await OrchestratorSystem.Run(`cd ${Input.projectPath} && git remote -v`, false, true)).replace(
|
||||
/ /g,
|
||||
``,
|
||||
);
|
||||
CloudRunnerLogger.log(`value ${value}`);
|
||||
OrchestratorLogger.log(`value ${value}`);
|
||||
assert(value.includes('github.com'));
|
||||
|
||||
return value.split('github.com')[1].split('.git')[0].slice(1);
|
||||
}
|
||||
|
||||
public static async GetBranch() {
|
||||
if (CloudRunnerOptions.providerStrategy === 'local') {
|
||||
if (OrchestratorOptions.providerStrategy === 'local') {
|
||||
return '';
|
||||
}
|
||||
assert(fs.existsSync(`.git`));
|
||||
|
||||
return (await CloudRunnerSystem.Run(`cd ${Input.projectPath} && git branch --show-current`, false, true))
|
||||
return (await OrchestratorSystem.Run(`cd ${Input.projectPath} && git branch --show-current`, false, true))
|
||||
.split('\n')[0]
|
||||
.replace(/ /g, ``)
|
||||
.replace('/head', '');
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
|
||||
import { OrchestratorSystem } from '../orchestrator/services/core/orchestrator-system';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
|
||||
|
||||
export class GithubCliReader {
|
||||
static async GetGitHubAuthToken() {
|
||||
if (CloudRunnerOptions.providerStrategy === 'local') {
|
||||
if (OrchestratorOptions.providerStrategy === 'local') {
|
||||
return '';
|
||||
}
|
||||
try {
|
||||
const authStatus = await CloudRunnerSystem.Run(`gh auth status`, true, true);
|
||||
const authStatus = await OrchestratorSystem.Run(`gh auth status`, true, true);
|
||||
if (authStatus.includes('You are not logged') || authStatus === '') {
|
||||
return '';
|
||||
}
|
||||
|
||||
return (await CloudRunnerSystem.Run(`gh auth status -t`, false, true))
|
||||
return (await OrchestratorSystem.Run(`gh auth status -t`, false, true))
|
||||
.split(`Token: `)[1]
|
||||
.replace(/ /g, '')
|
||||
.replace(/\n/g, '');
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import YAML from 'yaml';
|
||||
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../orchestrator/options/orchestrator-options';
|
||||
|
||||
export function ReadLicense(): string {
|
||||
if (CloudRunnerOptions.providerStrategy === 'local') {
|
||||
if (OrchestratorOptions.providerStrategy === 'local') {
|
||||
return '';
|
||||
}
|
||||
const pipelineFile = path.join(__dirname, `.github`, `workflows`, `cloud-runner-k8s-pipeline.yml`);
|
||||
const pipelineFile = path.join(__dirname, `.github`, `workflows`, `orchestrator-k8s-pipeline.yml`);
|
||||
|
||||
return fs.existsSync(pipelineFile) ? YAML.parse(fs.readFileSync(pipelineFile, 'utf8')).env.UNITY_LICENSE : '';
|
||||
}
|
||||
|
||||
+79
-3
@@ -1,7 +1,7 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Cli } from './cli/cli';
|
||||
import CloudRunnerQueryOverride from './cloud-runner/options/cloud-runner-query-override';
|
||||
import OrchestratorQueryOverride from './orchestrator/options/orchestrator-query-override';
|
||||
import Platform from './platform';
|
||||
import GitHub from './github';
|
||||
import os from 'node:os';
|
||||
@@ -32,8 +32,8 @@ class Input {
|
||||
return Cli.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
|
||||
return CloudRunnerQueryOverride.query(query, alternativeQuery);
|
||||
if (OrchestratorQueryOverride.query(query, alternativeQuery)) {
|
||||
return OrchestratorQueryOverride.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (process.env[query] !== undefined) {
|
||||
@@ -282,6 +282,82 @@ class Input {
|
||||
return Input.getInput('skipActivation')?.toLowerCase() ?? 'false';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Remote PowerShell provider
|
||||
// ### ### ###
|
||||
|
||||
static get remotePowershellHost(): string {
|
||||
return Input.getInput('remotePowershellHost') ?? '';
|
||||
}
|
||||
|
||||
static get remotePowershellCredential(): string {
|
||||
return Input.getInput('remotePowershellCredential') ?? '';
|
||||
}
|
||||
|
||||
static get remotePowershellTransport(): string {
|
||||
return Input.getInput('remotePowershellTransport') ?? 'wsman';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// GitHub Actions provider
|
||||
// ### ### ###
|
||||
|
||||
static get githubActionsRepo(): string {
|
||||
return Input.getInput('githubActionsRepo') ?? '';
|
||||
}
|
||||
|
||||
static get githubActionsWorkflow(): string {
|
||||
return Input.getInput('githubActionsWorkflow') ?? '';
|
||||
}
|
||||
|
||||
static get githubActionsToken(): string {
|
||||
return Input.getInput('githubActionsToken') ?? '';
|
||||
}
|
||||
|
||||
static get githubActionsRef(): string {
|
||||
return Input.getInput('githubActionsRef') ?? 'main';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// GitLab CI provider
|
||||
// ### ### ###
|
||||
|
||||
static get gitlabProjectId(): string {
|
||||
return Input.getInput('gitlabProjectId') ?? '';
|
||||
}
|
||||
|
||||
static get gitlabTriggerToken(): string {
|
||||
return Input.getInput('gitlabTriggerToken') ?? '';
|
||||
}
|
||||
|
||||
static get gitlabApiUrl(): string {
|
||||
return Input.getInput('gitlabApiUrl') ?? 'https://gitlab.com';
|
||||
}
|
||||
|
||||
static get gitlabRef(): string {
|
||||
return Input.getInput('gitlabRef') ?? 'main';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Ansible provider
|
||||
// ### ### ###
|
||||
|
||||
static get ansibleInventory(): string {
|
||||
return Input.getInput('ansibleInventory') ?? '';
|
||||
}
|
||||
|
||||
static get ansiblePlaybook(): string {
|
||||
return Input.getInput('ansiblePlaybook') ?? '';
|
||||
}
|
||||
|
||||
static get ansibleExtraVars(): string {
|
||||
return Input.getInput('ansibleExtraVars') ?? '';
|
||||
}
|
||||
|
||||
static get ansibleVaultPassword(): string {
|
||||
return Input.getInput('ansibleVaultPassword') ?? '';
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string) {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
import Orchestrator from '../orchestrator';
|
||||
import OrchestratorSecret from '../options/orchestrator-secret';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
|
||||
export class OrchestratorError {
|
||||
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: OrchestratorSecret[]) {
|
||||
OrchestratorLogger.error(JSON.stringify(error, undefined, 4));
|
||||
core.setFailed('Orchestrator failed');
|
||||
if (Orchestrator.Provider !== undefined) {
|
||||
await Orchestrator.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
class OrchestratorConstants {
|
||||
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
|
||||
}
|
||||
export default OrchestratorConstants;
|
||||
@@ -0,0 +1,5 @@
|
||||
class OrchestratorEnvironmentVariable {
|
||||
public name!: string;
|
||||
public value!: string;
|
||||
}
|
||||
export default OrchestratorEnvironmentVariable;
|
||||
@@ -0,0 +1,90 @@
|
||||
import path from 'node:path';
|
||||
import OrchestratorOptions from './orchestrator-options';
|
||||
import Orchestrator from '../orchestrator';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
|
||||
export class OrchestratorFolders {
|
||||
public static readonly repositoryFolder = 'repo';
|
||||
|
||||
public static ToLinuxFolder(folder: string) {
|
||||
return folder.replace(/\\/g, `/`);
|
||||
}
|
||||
|
||||
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
|
||||
public static get uniqueOrchestratorJobFolderAbsolute(): string {
|
||||
return Orchestrator.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(Orchestrator.buildParameters)
|
||||
? path.join(`/`, OrchestratorFolders.buildVolumeFolder, Orchestrator.lockedWorkspace)
|
||||
: path.join(`/`, OrchestratorFolders.buildVolumeFolder, Orchestrator.buildParameters.buildGuid);
|
||||
}
|
||||
|
||||
public static get cacheFolderForAllFull(): string {
|
||||
return path.join('/', OrchestratorFolders.buildVolumeFolder, OrchestratorFolders.cacheFolder);
|
||||
}
|
||||
|
||||
public static get cacheFolderForCacheKeyFull(): string {
|
||||
return path.join(
|
||||
'/',
|
||||
OrchestratorFolders.buildVolumeFolder,
|
||||
OrchestratorFolders.cacheFolder,
|
||||
Orchestrator.buildParameters.cacheKey,
|
||||
);
|
||||
}
|
||||
|
||||
public static get builderPathAbsolute(): string {
|
||||
return path.join(
|
||||
OrchestratorOptions.useSharedBuilder
|
||||
? `/${OrchestratorFolders.buildVolumeFolder}`
|
||||
: OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute,
|
||||
`builder`,
|
||||
);
|
||||
}
|
||||
|
||||
public static get repoPathAbsolute(): string {
|
||||
return path.join(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute, OrchestratorFolders.repositoryFolder);
|
||||
}
|
||||
|
||||
public static get projectPathAbsolute(): string {
|
||||
return path.join(OrchestratorFolders.repoPathAbsolute, Orchestrator.buildParameters.projectPath);
|
||||
}
|
||||
|
||||
public static get libraryFolderAbsolute(): string {
|
||||
return path.join(OrchestratorFolders.projectPathAbsolute, `Library`);
|
||||
}
|
||||
|
||||
public static get projectBuildFolderAbsolute(): string {
|
||||
return path.join(OrchestratorFolders.repoPathAbsolute, Orchestrator.buildParameters.buildPath);
|
||||
}
|
||||
|
||||
public static get lfsFolderAbsolute(): string {
|
||||
return path.join(OrchestratorFolders.repoPathAbsolute, `.git`, `lfs`);
|
||||
}
|
||||
|
||||
public static get purgeRemoteCaching(): boolean {
|
||||
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
||||
}
|
||||
|
||||
public static get lfsCacheFolderFull() {
|
||||
return path.join(OrchestratorFolders.cacheFolderForCacheKeyFull, `lfs`);
|
||||
}
|
||||
|
||||
public static get libraryCacheFolderFull() {
|
||||
return path.join(OrchestratorFolders.cacheFolderForCacheKeyFull, `Library`);
|
||||
}
|
||||
|
||||
public static get unityBuilderRepoUrl(): string {
|
||||
return `https://${Orchestrator.buildParameters.gitPrivateToken}@github.com/${Orchestrator.buildParameters.orchestratorRepoName}.git`;
|
||||
}
|
||||
|
||||
public static get targetBuildRepoUrl(): string {
|
||||
return `https://${Orchestrator.buildParameters.gitPrivateToken}@github.com/${Orchestrator.buildParameters.githubRepo}.git`;
|
||||
}
|
||||
|
||||
public static get buildVolumeFolder() {
|
||||
return 'data';
|
||||
}
|
||||
|
||||
public static get cacheFolder() {
|
||||
return 'cache';
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -1,11 +1,11 @@
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import CloudRunnerConstants from './cloud-runner-constants';
|
||||
import OrchestratorConstants from './orchestrator-constants';
|
||||
|
||||
class CloudRunnerNamespace {
|
||||
class OrchestratorNamespace {
|
||||
static generateGuid(runNumber: string | number, platform: string) {
|
||||
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
|
||||
const nanoid = customAlphabet(OrchestratorConstants.alphabet, 4);
|
||||
|
||||
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
|
||||
}
|
||||
}
|
||||
export default CloudRunnerNamespace;
|
||||
export default OrchestratorNamespace;
|
||||
@@ -0,0 +1,10 @@
|
||||
import Input from '../../input';
|
||||
import OrchestratorOptions from './orchestrator-options';
|
||||
|
||||
class OrchestratorOptionsReader {
|
||||
static GetProperties() {
|
||||
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(OrchestratorOptions)];
|
||||
}
|
||||
}
|
||||
|
||||
export default OrchestratorOptionsReader;
|
||||
@@ -0,0 +1,338 @@
|
||||
import { Cli } from '../../cli/cli';
|
||||
import OrchestratorQueryOverride from './orchestrator-query-override';
|
||||
import GitHub from '../../github';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
class OrchestratorOptions {
|
||||
// ### ### ###
|
||||
// Input Handling
|
||||
// ### ### ###
|
||||
public static getInput(query: string): string | undefined {
|
||||
if (GitHub.githubInputEnabled) {
|
||||
const coreInput = core.getInput(query);
|
||||
if (coreInput && coreInput !== '') {
|
||||
return coreInput;
|
||||
}
|
||||
}
|
||||
const alternativeQuery = OrchestratorOptions.ToEnvVarFormat(query);
|
||||
|
||||
// Query input sources
|
||||
if (Cli.query(query, alternativeQuery)) {
|
||||
return Cli.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (OrchestratorQueryOverride.query(query, alternativeQuery)) {
|
||||
return OrchestratorQueryOverride.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (process.env[query] !== undefined) {
|
||||
return process.env[query];
|
||||
}
|
||||
|
||||
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
|
||||
return process.env[alternativeQuery];
|
||||
}
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string): string {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
}
|
||||
|
||||
return input
|
||||
.replace(/([A-Z])/g, ' $1')
|
||||
.trim()
|
||||
.toUpperCase()
|
||||
.replace(/ /g, '_');
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Provider parameters
|
||||
// ### ### ###
|
||||
|
||||
static get region(): string {
|
||||
return OrchestratorOptions.getInput('region') || 'eu-west-2';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// GitHub parameters
|
||||
// ### ### ###
|
||||
static get githubChecks(): boolean {
|
||||
const value = OrchestratorOptions.getInput('githubChecks');
|
||||
|
||||
return value === `true` || false;
|
||||
}
|
||||
static get githubCheckId(): string {
|
||||
return OrchestratorOptions.getInput('githubCheckId') || ``;
|
||||
}
|
||||
|
||||
static get githubOwner(): string {
|
||||
return OrchestratorOptions.getInput('githubOwner') || OrchestratorOptions.githubRepo?.split(`/`)[0] || '';
|
||||
}
|
||||
|
||||
static get githubRepoName(): string {
|
||||
return OrchestratorOptions.getInput('githubRepoName') || OrchestratorOptions.githubRepo?.split(`/`)[1] || '';
|
||||
}
|
||||
|
||||
static get orchestratorRepoName(): string {
|
||||
return OrchestratorOptions.getInput('orchestratorRepoName') || 'game-ci/unity-builder';
|
||||
}
|
||||
|
||||
static get cloneDepth(): string {
|
||||
return OrchestratorOptions.getInput('cloneDepth') || '50';
|
||||
}
|
||||
|
||||
static get finalHooks(): string[] {
|
||||
return OrchestratorOptions.getInput('finalHooks')?.split(',') || [];
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Git syncronization parameters
|
||||
// ### ### ###
|
||||
|
||||
static get githubRepo(): string | undefined {
|
||||
return (
|
||||
OrchestratorOptions.getInput('GITHUB_REPOSITORY') || OrchestratorOptions.getInput('GITHUB_REPO') || undefined
|
||||
);
|
||||
}
|
||||
static get branch(): string {
|
||||
if (OrchestratorOptions.getInput(`GITHUB_REF`)) {
|
||||
return (
|
||||
OrchestratorOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') ||
|
||||
``
|
||||
);
|
||||
} else if (OrchestratorOptions.getInput('branch')) {
|
||||
return OrchestratorOptions.getInput('branch') || ``;
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Orchestrator parameters
|
||||
// ### ### ###
|
||||
|
||||
static get buildPlatform(): string {
|
||||
const input = OrchestratorOptions.getInput('buildPlatform');
|
||||
if (input && input !== '') {
|
||||
return input;
|
||||
}
|
||||
if (OrchestratorOptions.providerStrategy !== 'local') {
|
||||
return 'linux';
|
||||
}
|
||||
|
||||
return process.platform;
|
||||
}
|
||||
|
||||
static get orchestratorBranch(): string {
|
||||
return OrchestratorOptions.getInput('orchestratorBranch') || 'main';
|
||||
}
|
||||
|
||||
static get providerStrategy(): string {
|
||||
const provider =
|
||||
OrchestratorOptions.getInput('orchestratorCluster') || OrchestratorOptions.getInput('providerStrategy');
|
||||
if (Cli.isCliMode) {
|
||||
return provider || 'aws';
|
||||
}
|
||||
|
||||
return provider || 'local';
|
||||
}
|
||||
|
||||
static get containerCpu(): string {
|
||||
return OrchestratorOptions.getInput('containerCpu') || `1024`;
|
||||
}
|
||||
|
||||
static get containerMemory(): string {
|
||||
return OrchestratorOptions.getInput('containerMemory') || `3072`;
|
||||
}
|
||||
|
||||
static get containerNamespace(): string {
|
||||
return OrchestratorOptions.getInput('containerNamespace') || `default`;
|
||||
}
|
||||
|
||||
static get customJob(): string {
|
||||
return OrchestratorOptions.getInput('customJob') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Custom commands from files parameters
|
||||
// ### ### ###
|
||||
|
||||
static get containerHookFiles(): string[] {
|
||||
return OrchestratorOptions.getInput('containerHookFiles')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
static get commandHookFiles(): string[] {
|
||||
return OrchestratorOptions.getInput('commandHookFiles')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Custom commands from yaml parameters
|
||||
// ### ### ###
|
||||
|
||||
static get commandHooks(): string {
|
||||
return OrchestratorOptions.getInput('commandHooks') || '';
|
||||
}
|
||||
|
||||
static get postBuildContainerHooks(): string {
|
||||
return OrchestratorOptions.getInput('postBuildContainerHooks') || '';
|
||||
}
|
||||
|
||||
static get preBuildContainerHooks(): string {
|
||||
return OrchestratorOptions.getInput('preBuildContainerHooks') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Input override handling
|
||||
// ### ### ###
|
||||
|
||||
static get pullInputList(): string[] {
|
||||
return OrchestratorOptions.getInput('pullInputList')?.split(`,`) || [];
|
||||
}
|
||||
|
||||
static get inputPullCommand(): string {
|
||||
const value = OrchestratorOptions.getInput('inputPullCommand');
|
||||
|
||||
if (value === 'gcp-secret-manager') {
|
||||
return 'gcloud secrets versions access 1 --secret="{0}"';
|
||||
} else if (value === 'aws-secret-manager') {
|
||||
return 'aws secretsmanager get-secret-value --secret-id {0}';
|
||||
}
|
||||
|
||||
return value || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Aws
|
||||
// ### ### ###
|
||||
|
||||
static get awsStackName() {
|
||||
return OrchestratorOptions.getInput('awsStackName') || 'game-ci';
|
||||
}
|
||||
|
||||
static get awsEndpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsEndpoint');
|
||||
}
|
||||
|
||||
static get awsCloudFormationEndpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsCloudFormationEndpoint') || OrchestratorOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsEcsEndpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsEcsEndpoint') || OrchestratorOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsKinesisEndpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsKinesisEndpoint') || OrchestratorOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsCloudWatchLogsEndpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsCloudWatchLogsEndpoint') || OrchestratorOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
static get awsS3Endpoint(): string | undefined {
|
||||
return OrchestratorOptions.getInput('awsS3Endpoint') || OrchestratorOptions.awsEndpoint;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Storage
|
||||
// ### ### ###
|
||||
|
||||
static get storageProvider(): string {
|
||||
return OrchestratorOptions.getInput('storageProvider') || 's3';
|
||||
}
|
||||
|
||||
static get rcloneRemote(): string {
|
||||
return OrchestratorOptions.getInput('rcloneRemote') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// K8s
|
||||
// ### ### ###
|
||||
|
||||
static get kubeConfig(): string {
|
||||
return OrchestratorOptions.getInput('kubeConfig') || '';
|
||||
}
|
||||
|
||||
static get kubeVolume(): string {
|
||||
return OrchestratorOptions.getInput('kubeVolume') || '';
|
||||
}
|
||||
|
||||
static get kubeVolumeSize(): string {
|
||||
return OrchestratorOptions.getInput('kubeVolumeSize') || '25Gi';
|
||||
}
|
||||
|
||||
static get kubeStorageClass(): string {
|
||||
return OrchestratorOptions.getInput('kubeStorageClass') || '';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Caching
|
||||
// ### ### ###
|
||||
|
||||
static get cacheKey(): string {
|
||||
return OrchestratorOptions.getInput('cacheKey') || OrchestratorOptions.branch;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Utility Parameters
|
||||
// ### ### ###
|
||||
|
||||
static get orchestratorDebug(): boolean {
|
||||
return (
|
||||
OrchestratorOptions.getInput(`orchestratorTests`) === `true` ||
|
||||
OrchestratorOptions.getInput(`orchestratorDebug`) === `true` ||
|
||||
OrchestratorOptions.getInput(`orchestratorDebugTree`) === `true` ||
|
||||
OrchestratorOptions.getInput(`orchestratorDebugEnv`) === `true` ||
|
||||
false
|
||||
);
|
||||
}
|
||||
static get skipLfs(): boolean {
|
||||
return OrchestratorOptions.getInput(`skipLfs`) === `true`;
|
||||
}
|
||||
static get skipCache(): boolean {
|
||||
return OrchestratorOptions.getInput(`skipCache`) === `true`;
|
||||
}
|
||||
|
||||
public static get asyncOrchestrator(): boolean {
|
||||
return OrchestratorOptions.getInput('asyncOrchestrator') === 'true';
|
||||
}
|
||||
|
||||
public static get resourceTracking(): boolean {
|
||||
return OrchestratorOptions.getInput('resourceTracking') === 'true';
|
||||
}
|
||||
|
||||
public static get useLargePackages(): boolean {
|
||||
return OrchestratorOptions.getInput(`useLargePackages`) === `true`;
|
||||
}
|
||||
|
||||
public static get useSharedBuilder(): boolean {
|
||||
return OrchestratorOptions.getInput(`useSharedBuilder`) === `true`;
|
||||
}
|
||||
|
||||
public static get useCompressionStrategy(): boolean {
|
||||
return OrchestratorOptions.getInput(`useCompressionStrategy`) === `true`;
|
||||
}
|
||||
|
||||
public static get useCleanupCron(): boolean {
|
||||
return (OrchestratorOptions.getInput(`useCleanupCron`) || 'true') === 'true';
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Retained Workspace
|
||||
// ### ### ###
|
||||
|
||||
public static get maxRetainedWorkspaces(): string {
|
||||
return OrchestratorOptions.getInput(`maxRetainedWorkspaces`) || `0`;
|
||||
}
|
||||
|
||||
// ### ### ###
|
||||
// Garbage Collection
|
||||
// ### ### ###
|
||||
|
||||
static get garbageMaxAge(): number {
|
||||
return Number(OrchestratorOptions.getInput(`garbageMaxAge`)) || 24;
|
||||
}
|
||||
}
|
||||
|
||||
export default OrchestratorOptions;
|
||||
@@ -0,0 +1,67 @@
|
||||
import Input from '../../input';
|
||||
import { GenericInputReader } from '../../input-readers/generic-input-reader';
|
||||
import OrchestratorOptions from './orchestrator-options';
|
||||
|
||||
const formatFunction = (value: string, arguments_: any[]) => {
|
||||
for (const element of arguments_) {
|
||||
value = value.replace(`{${element.key}}`, element.value);
|
||||
}
|
||||
|
||||
return value;
|
||||
};
|
||||
|
||||
class OrchestratorQueryOverride {
|
||||
static queryOverrides: { [key: string]: string } | undefined;
|
||||
|
||||
// TODO accept premade secret sources or custom secret source definition yamls
|
||||
|
||||
public static query(key: string, alternativeKey: string) {
|
||||
if (OrchestratorQueryOverride.queryOverrides && OrchestratorQueryOverride.queryOverrides[key] !== undefined) {
|
||||
return OrchestratorQueryOverride.queryOverrides[key];
|
||||
}
|
||||
if (
|
||||
OrchestratorQueryOverride.queryOverrides &&
|
||||
alternativeKey &&
|
||||
OrchestratorQueryOverride.queryOverrides[alternativeKey] !== undefined
|
||||
) {
|
||||
return OrchestratorQueryOverride.queryOverrides[alternativeKey];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
private static shouldUseOverride(query: string) {
|
||||
if (OrchestratorOptions.inputPullCommand !== '') {
|
||||
if (OrchestratorOptions.pullInputList.length > 0) {
|
||||
const doesInclude =
|
||||
OrchestratorOptions.pullInputList.includes(query) ||
|
||||
OrchestratorOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
|
||||
|
||||
return doesInclude ? true : false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async queryOverride(query: string) {
|
||||
if (!this.shouldUseOverride(query)) {
|
||||
throw new Error(`Should not be trying to run override query on ${query}`);
|
||||
}
|
||||
|
||||
return await GenericInputReader.Run(
|
||||
formatFunction(OrchestratorOptions.inputPullCommand, [{ key: 0, value: query }]),
|
||||
);
|
||||
}
|
||||
|
||||
public static async PopulateQueryOverrideInput() {
|
||||
const queries = OrchestratorOptions.pullInputList;
|
||||
OrchestratorQueryOverride.queryOverrides = {};
|
||||
for (const element of queries) {
|
||||
if (OrchestratorQueryOverride.shouldUseOverride(element)) {
|
||||
OrchestratorQueryOverride.queryOverrides[element] = await OrchestratorQueryOverride.queryOverride(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export default OrchestratorQueryOverride;
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
class CloudRunnerSecret {
|
||||
class OrchestratorSecret {
|
||||
public ParameterKey!: string;
|
||||
public EnvironmentVariable!: string;
|
||||
public ParameterValue!: string;
|
||||
}
|
||||
export default CloudRunnerSecret;
|
||||
export default OrchestratorSecret;
|
||||
@@ -0,0 +1,3 @@
|
||||
export class OrchestratorStatics {
|
||||
public static readonly logPrefix = `Orchestrator`;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import OrchestratorEnvironmentVariable from './orchestrator-environment-variable';
|
||||
import OrchestratorSecret from './orchestrator-secret';
|
||||
|
||||
export class OrchestratorStepParameters {
|
||||
public image: string;
|
||||
public environment: OrchestratorEnvironmentVariable[];
|
||||
public secrets: OrchestratorSecret[];
|
||||
constructor(image: string, environmentVariables: OrchestratorEnvironmentVariable[], secrets: OrchestratorSecret[]) {
|
||||
this.image = image;
|
||||
this.environment = environmentVariables;
|
||||
this.secrets = secrets;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
import AwsBuildPlatform from './providers/aws';
|
||||
import { BuildParameters, Input } from '..';
|
||||
import Kubernetes from './providers/k8s';
|
||||
import OrchestratorLogger from './services/core/orchestrator-logger';
|
||||
import { OrchestratorStepParameters } from './options/orchestrator-step-parameters';
|
||||
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
|
||||
import { OrchestratorError } from './error/orchestrator-error';
|
||||
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
|
||||
import * as core from '@actions/core';
|
||||
import OrchestratorSecret from './options/orchestrator-secret';
|
||||
import { ProviderInterface } from './providers/provider-interface';
|
||||
import OrchestratorEnvironmentVariable from './options/orchestrator-environment-variable';
|
||||
import TestOrchestrator from './providers/test';
|
||||
import LocalOrchestrator from './providers/local';
|
||||
import LocalDockerOrchestrator from './providers/docker';
|
||||
import RemotePowershellProvider from './providers/remote-powershell';
|
||||
import GitHubActionsProvider from './providers/github-actions';
|
||||
import GitLabCIProvider from './providers/gitlab-ci';
|
||||
import AnsibleProvider from './providers/ansible';
|
||||
import loadProvider from './providers/provider-loader';
|
||||
import GitHub from '../github';
|
||||
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
|
||||
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
|
||||
import OrchestratorResult from './services/core/orchestrator-result';
|
||||
import OrchestratorOptions from './options/orchestrator-options';
|
||||
import ResourceTracking from './services/core/resource-tracking';
|
||||
|
||||
class Orchestrator {
|
||||
public static Provider: ProviderInterface;
|
||||
public static buildParameters: BuildParameters;
|
||||
private static defaultSecrets: OrchestratorSecret[];
|
||||
private static orchestratorEnvironmentVariables: OrchestratorEnvironmentVariable[];
|
||||
static lockedWorkspace: string = ``;
|
||||
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
|
||||
|
||||
// When true, validates AWS CloudFormation templates even when using local-docker execution
|
||||
// This is set by AWS_FORCE_PROVIDER=aws-local mode
|
||||
public static validateAwsTemplates: boolean = false;
|
||||
public static get isOrchestratorEnvironment() {
|
||||
return process.env[`GITHUB_ACTIONS`] !== `true`;
|
||||
}
|
||||
public static get isOrchestratorAsyncEnvironment() {
|
||||
return process.env[`ASYNC_WORKFLOW`] === `true`;
|
||||
}
|
||||
public static async setup(buildParameters: BuildParameters) {
|
||||
OrchestratorLogger.setup();
|
||||
OrchestratorLogger.log(`Setting up orchestrator`);
|
||||
Orchestrator.buildParameters = buildParameters;
|
||||
ResourceTracking.logAllocationSummary('setup');
|
||||
await ResourceTracking.logDiskUsageSnapshot('setup');
|
||||
if (Orchestrator.buildParameters.githubCheckId === ``) {
|
||||
Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(
|
||||
Orchestrator.buildParameters.buildGuid,
|
||||
);
|
||||
}
|
||||
await Orchestrator.setupSelectedBuildPlatform();
|
||||
Orchestrator.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
|
||||
Orchestrator.orchestratorEnvironmentVariables =
|
||||
TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameters);
|
||||
if (GitHub.githubInputEnabled) {
|
||||
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
|
||||
for (const element of Orchestrator.orchestratorEnvironmentVariables) {
|
||||
// OrchestratorLogger.log(`Orchestrator output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
|
||||
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
|
||||
}
|
||||
for (const element of buildParameterPropertyNames) {
|
||||
// OrchestratorLogger.log(`Orchestrator output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
|
||||
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
|
||||
}
|
||||
core.setOutput(
|
||||
Input.ToEnvVarFormat(`buildArtifact`),
|
||||
`build-${Orchestrator.buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
);
|
||||
}
|
||||
FollowLogStreamService.Reset();
|
||||
}
|
||||
|
||||
private static async setupSelectedBuildPlatform() {
|
||||
OrchestratorLogger.log(`Orchestrator platform selected ${Orchestrator.buildParameters.providerStrategy}`);
|
||||
|
||||
// Detect LocalStack endpoints and handle AWS provider appropriately
|
||||
// AWS_FORCE_PROVIDER options:
|
||||
// - 'aws': Force AWS provider (requires LocalStack Pro with ECS support)
|
||||
// - 'aws-local': Validate AWS templates/config but execute via local-docker (for CI without ECS)
|
||||
// - unset/other: Auto-fallback to local-docker when LocalStack detected
|
||||
const awsForceProvider = process.env.AWS_FORCE_PROVIDER || '';
|
||||
const forceAwsProvider = awsForceProvider === 'aws' || awsForceProvider === 'true';
|
||||
const useAwsLocalMode = awsForceProvider === 'aws-local';
|
||||
const endpointsToCheck = [
|
||||
process.env.AWS_ENDPOINT,
|
||||
process.env.AWS_S3_ENDPOINT,
|
||||
process.env.AWS_CLOUD_FORMATION_ENDPOINT,
|
||||
process.env.AWS_ECS_ENDPOINT,
|
||||
process.env.AWS_KINESIS_ENDPOINT,
|
||||
process.env.AWS_CLOUD_WATCH_LOGS_ENDPOINT,
|
||||
OrchestratorOptions.awsEndpoint,
|
||||
OrchestratorOptions.awsS3Endpoint,
|
||||
OrchestratorOptions.awsCloudFormationEndpoint,
|
||||
OrchestratorOptions.awsEcsEndpoint,
|
||||
OrchestratorOptions.awsKinesisEndpoint,
|
||||
OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
||||
]
|
||||
.filter((x) => typeof x === 'string')
|
||||
.join(' ');
|
||||
const isLocalStack = /localstack|localhost|127\.0\.0\.1/i.test(endpointsToCheck);
|
||||
let provider = Orchestrator.buildParameters.providerStrategy;
|
||||
let validateAwsTemplates = false;
|
||||
|
||||
if (provider === 'aws' && isLocalStack) {
|
||||
if (useAwsLocalMode) {
|
||||
// aws-local mode: Validate AWS templates but execute via local-docker
|
||||
// This provides confidence in AWS CloudFormation without requiring LocalStack Pro
|
||||
OrchestratorLogger.log('AWS_FORCE_PROVIDER=aws-local: Validating AWS templates, executing via local-docker');
|
||||
validateAwsTemplates = true;
|
||||
provider = 'local-docker';
|
||||
} else if (forceAwsProvider) {
|
||||
// Force full AWS provider (requires LocalStack Pro with ECS support)
|
||||
OrchestratorLogger.log(
|
||||
'LocalStack endpoints detected but AWS_FORCE_PROVIDER=aws; using full AWS provider (requires ECS support)',
|
||||
);
|
||||
} else {
|
||||
// Auto-fallback to local-docker
|
||||
OrchestratorLogger.log('LocalStack endpoints detected; routing provider to local-docker for this run');
|
||||
OrchestratorLogger.log(
|
||||
'Note: Set AWS_FORCE_PROVIDER=aws-local to validate AWS templates with local-docker execution',
|
||||
);
|
||||
provider = 'local-docker';
|
||||
}
|
||||
}
|
||||
|
||||
// Store whether we should validate AWS templates (used by aws-local mode)
|
||||
Orchestrator.validateAwsTemplates = validateAwsTemplates;
|
||||
|
||||
switch (provider) {
|
||||
case 'k8s':
|
||||
Orchestrator.Provider = new Kubernetes(Orchestrator.buildParameters);
|
||||
break;
|
||||
case 'aws':
|
||||
Orchestrator.Provider = new AwsBuildPlatform(Orchestrator.buildParameters);
|
||||
|
||||
// Validate that AWS provider is actually being used when expected
|
||||
if (isLocalStack && forceAwsProvider) {
|
||||
OrchestratorLogger.log('✓ AWS provider initialized with LocalStack - AWS functionality will be validated');
|
||||
} else if (isLocalStack && !forceAwsProvider) {
|
||||
OrchestratorLogger.log(
|
||||
'⚠ WARNING: AWS provider was requested but LocalStack detected without AWS_FORCE_PROVIDER',
|
||||
);
|
||||
OrchestratorLogger.log('⚠ This may cause AWS functionality tests to fail validation');
|
||||
}
|
||||
break;
|
||||
case 'test':
|
||||
Orchestrator.Provider = new TestOrchestrator();
|
||||
break;
|
||||
case 'local-docker':
|
||||
Orchestrator.Provider = new LocalDockerOrchestrator();
|
||||
break;
|
||||
case 'local-system':
|
||||
Orchestrator.Provider = new LocalOrchestrator();
|
||||
break;
|
||||
case 'local':
|
||||
Orchestrator.Provider = new LocalOrchestrator();
|
||||
break;
|
||||
case 'remote-powershell':
|
||||
Orchestrator.Provider = new RemotePowershellProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
case 'github-actions':
|
||||
Orchestrator.Provider = new GitHubActionsProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
case 'gitlab-ci':
|
||||
Orchestrator.Provider = new GitLabCIProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
case 'ansible':
|
||||
Orchestrator.Provider = new AnsibleProvider(Orchestrator.buildParameters);
|
||||
break;
|
||||
default:
|
||||
// Try to load provider using the dynamic loader for unknown providers
|
||||
try {
|
||||
Orchestrator.Provider = await loadProvider(provider, Orchestrator.buildParameters);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`Failed to load provider '${provider}' using dynamic loader: ${error.message}`);
|
||||
OrchestratorLogger.log('Falling back to local provider...');
|
||||
Orchestrator.Provider = new LocalOrchestrator();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Final validation: Ensure provider matches expectations
|
||||
const finalProviderName = Orchestrator.Provider.constructor.name;
|
||||
if (Orchestrator.buildParameters.providerStrategy === 'aws' && finalProviderName !== 'AWSBuildEnvironment') {
|
||||
OrchestratorLogger.log(`⚠ WARNING: Expected AWS provider but got ${finalProviderName}`);
|
||||
OrchestratorLogger.log('⚠ AWS functionality tests may not be validating AWS services correctly');
|
||||
}
|
||||
}
|
||||
|
||||
static async run(buildParameters: BuildParameters, baseImage: string) {
|
||||
if (baseImage.includes(`undefined`)) {
|
||||
throw new Error(`baseImage is undefined`);
|
||||
}
|
||||
await Orchestrator.setup(buildParameters);
|
||||
|
||||
// When aws-local mode is enabled, validate AWS CloudFormation templates
|
||||
// This ensures AWS templates are correct even when executing via local-docker
|
||||
if (Orchestrator.validateAwsTemplates) {
|
||||
await Orchestrator.validateAwsCloudFormationTemplates();
|
||||
}
|
||||
await Orchestrator.Provider.setupWorkflow(
|
||||
Orchestrator.buildParameters.buildGuid,
|
||||
Orchestrator.buildParameters,
|
||||
Orchestrator.buildParameters.branch,
|
||||
Orchestrator.defaultSecrets,
|
||||
);
|
||||
try {
|
||||
if (buildParameters.maxRetainedWorkspaces > 0) {
|
||||
Orchestrator.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
|
||||
|
||||
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
|
||||
Orchestrator.lockedWorkspace,
|
||||
Orchestrator.buildParameters.buildGuid,
|
||||
Orchestrator.buildParameters,
|
||||
);
|
||||
|
||||
if (result) {
|
||||
OrchestratorLogger.logLine(`Using retained workspace ${Orchestrator.lockedWorkspace}`);
|
||||
Orchestrator.orchestratorEnvironmentVariables = [
|
||||
...Orchestrator.orchestratorEnvironmentVariables,
|
||||
{ name: `LOCKED_WORKSPACE`, value: Orchestrator.lockedWorkspace },
|
||||
];
|
||||
} else {
|
||||
OrchestratorLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
|
||||
buildParameters.maxRetainedWorkspaces = 0;
|
||||
Orchestrator.lockedWorkspace = ``;
|
||||
}
|
||||
}
|
||||
await Orchestrator.updateStatusWithBuildParameters();
|
||||
const output = await new WorkflowCompositionRoot().run(
|
||||
new OrchestratorStepParameters(
|
||||
baseImage,
|
||||
Orchestrator.orchestratorEnvironmentVariables,
|
||||
Orchestrator.defaultSecrets,
|
||||
),
|
||||
);
|
||||
await Orchestrator.Provider.cleanupWorkflow(
|
||||
Orchestrator.buildParameters,
|
||||
Orchestrator.buildParameters.branch,
|
||||
Orchestrator.defaultSecrets,
|
||||
);
|
||||
if (!Orchestrator.buildParameters.isCliMode) core.endGroup();
|
||||
if (buildParameters.asyncWorkflow && this.isOrchestratorEnvironment && this.isOrchestratorAsyncEnvironment) {
|
||||
await GitHub.updateGitHubCheck(Orchestrator.buildParameters.buildGuid, `success`, `success`, `completed`);
|
||||
}
|
||||
|
||||
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
||||
const workspace = Orchestrator.lockedWorkspace || ``;
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(
|
||||
workspace,
|
||||
Orchestrator.buildParameters.buildGuid,
|
||||
Orchestrator.buildParameters,
|
||||
);
|
||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, Orchestrator.buildParameters);
|
||||
if (isLocked) {
|
||||
throw new Error(
|
||||
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
|
||||
workspace,
|
||||
buildParameters,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
Orchestrator.lockedWorkspace = ``;
|
||||
}
|
||||
|
||||
await GitHub.triggerWorkflowOnComplete(Orchestrator.buildParameters.finalHooks);
|
||||
|
||||
if (buildParameters.constantGarbageCollection) {
|
||||
Orchestrator.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
|
||||
}
|
||||
|
||||
return new OrchestratorResult(buildParameters, output, true, true, false);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(JSON.stringify(error, undefined, 4));
|
||||
await GitHub.updateGitHubCheck(
|
||||
Orchestrator.buildParameters.buildGuid,
|
||||
`Failed - Error ${error?.message || error}`,
|
||||
`failure`,
|
||||
`completed`,
|
||||
);
|
||||
if (!Orchestrator.buildParameters.isCliMode) core.endGroup();
|
||||
await OrchestratorError.handleException(error, Orchestrator.buildParameters, Orchestrator.defaultSecrets);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async updateStatusWithBuildParameters() {
|
||||
const content = { ...Orchestrator.buildParameters };
|
||||
content.gitPrivateToken = ``;
|
||||
content.unitySerial = ``;
|
||||
content.unityEmail = ``;
|
||||
content.unityPassword = ``;
|
||||
const jsonContent = JSON.stringify(content, undefined, 4);
|
||||
await GitHub.updateGitHubCheck(jsonContent, Orchestrator.buildParameters.buildGuid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates AWS CloudFormation templates without deploying them.
|
||||
* Used by aws-local mode to ensure AWS templates are correct when executing via local-docker.
|
||||
* This provides confidence that AWS ECS deployments would work with the generated templates.
|
||||
*/
|
||||
private static async validateAwsCloudFormationTemplates() {
|
||||
OrchestratorLogger.log('=== AWS CloudFormation Template Validation (aws-local mode) ===');
|
||||
|
||||
try {
|
||||
// Import AWS template formations
|
||||
const { BaseStackFormation } = await import('./providers/aws/cloud-formations/base-stack-formation');
|
||||
const { TaskDefinitionFormation } = await import('./providers/aws/cloud-formations/task-definition-formation');
|
||||
|
||||
// Validate base stack template
|
||||
const baseTemplate = BaseStackFormation.formation;
|
||||
OrchestratorLogger.log(`✓ Base stack template generated (${baseTemplate.length} chars)`);
|
||||
|
||||
// Check for required resources in base stack
|
||||
const requiredBaseResources = ['AWS::EC2::VPC', 'AWS::ECS::Cluster', 'AWS::S3::Bucket', 'AWS::IAM::Role'];
|
||||
for (const resource of requiredBaseResources) {
|
||||
if (baseTemplate.includes(resource)) {
|
||||
OrchestratorLogger.log(` ✓ Contains ${resource}`);
|
||||
} else {
|
||||
throw new Error(`Base stack template missing required resource: ${resource}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate task definition template
|
||||
const taskTemplate = TaskDefinitionFormation.formation;
|
||||
OrchestratorLogger.log(`✓ Task definition template generated (${taskTemplate.length} chars)`);
|
||||
|
||||
// Check for required resources in task definition
|
||||
const requiredTaskResources = ['AWS::ECS::TaskDefinition', 'AWS::Logs::LogGroup'];
|
||||
for (const resource of requiredTaskResources) {
|
||||
if (taskTemplate.includes(resource)) {
|
||||
OrchestratorLogger.log(` ✓ Contains ${resource}`);
|
||||
} else {
|
||||
throw new Error(`Task definition template missing required resource: ${resource}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate YAML syntax by checking for common patterns
|
||||
if (!baseTemplate.includes('AWSTemplateFormatVersion')) {
|
||||
throw new Error('Base stack template missing AWSTemplateFormatVersion');
|
||||
}
|
||||
if (!taskTemplate.includes('AWSTemplateFormatVersion')) {
|
||||
throw new Error('Task definition template missing AWSTemplateFormatVersion');
|
||||
}
|
||||
|
||||
OrchestratorLogger.log('=== AWS CloudFormation templates validated successfully ===');
|
||||
OrchestratorLogger.log('Note: Actual execution will use local-docker provider');
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.log(`AWS CloudFormation template validation failed: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
export default Orchestrator;
|
||||
+20
-48
@@ -1,16 +1,14 @@
|
||||
# Provider Loader Dynamic Imports
|
||||
|
||||
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub
|
||||
repositories, and NPM packages.
|
||||
|
||||
## What is a Provider?
|
||||
|
||||
A provider is a pluggable backend that Cloud Runner uses to run builds and workflows. Examples include AWS, Kubernetes,
|
||||
or local execution. Each provider implements the `ProviderInterface`, which defines the common lifecycle methods (setup,
|
||||
run, cleanup, garbage collection, etc.).
|
||||
A **provider** is a pluggable backend that Orchestrator uses to run builds and workflows. Examples include **AWS**, **Kubernetes**, or local execution. Each provider implements the [ProviderInterface](https://github.com/game-ci/unity-builder/blob/main/src/model/orchestrator/providers/provider-interface.ts), which defines the common lifecycle methods (setup, run, cleanup, garbage collection, etc.).
|
||||
|
||||
This abstraction makes Cloud Runner flexible: you can switch execution environments or add your own provider (via npm
|
||||
package, GitHub repo, or local path) without changing the rest of your pipeline.
|
||||
This abstraction makes Orchestrator flexible: you can switch execution environments or add your own provider (via npm package, GitHub repo, or local path) without changing the rest of your pipeline.
|
||||
|
||||
## Dynamic Provider Loading
|
||||
|
||||
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub repositories, and NPM packages.
|
||||
|
||||
## Features
|
||||
|
||||
@@ -47,18 +45,21 @@ const absoluteProvider = await ProviderLoader.loadProvider('/path/to/provider',
|
||||
|
||||
```typescript
|
||||
// Load from GitHub URL
|
||||
const githubProvider = await ProviderLoader.loadProvider('https://github.com/user/my-provider', buildParameters);
|
||||
const githubProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Load from specific branch
|
||||
const branchProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider/tree/develop',
|
||||
buildParameters,
|
||||
'https://github.com/user/my-provider/tree/develop',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Load from specific path in repository
|
||||
const pathProvider = await ProviderLoader.loadProvider(
|
||||
'https://github.com/user/my-provider/tree/main/src/providers',
|
||||
buildParameters,
|
||||
'https://github.com/user/my-provider/tree/main/src/providers',
|
||||
buildParameters
|
||||
);
|
||||
|
||||
// Shorthand notation
|
||||
@@ -83,20 +84,8 @@ All providers must implement the `ProviderInterface`:
|
||||
```typescript
|
||||
interface ProviderInterface {
|
||||
cleanupWorkflow(): Promise<void>;
|
||||
setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: any[],
|
||||
): Promise<void>;
|
||||
runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
task: string,
|
||||
workingDirectory: string,
|
||||
buildVolumeFolder: string,
|
||||
environmentVariables: any[],
|
||||
secrets: any[],
|
||||
): Promise<string>;
|
||||
setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void>;
|
||||
runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string>;
|
||||
garbageCollect(): Promise<void>;
|
||||
listResources(): Promise<ProviderResource[]>;
|
||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||
@@ -118,23 +107,11 @@ export default class MyProvider implements ProviderInterface {
|
||||
// Cleanup logic
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
branchName: string,
|
||||
defaultSecretsArray: any[],
|
||||
): Promise<void> {
|
||||
async setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void> {
|
||||
// Setup logic
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
task: string,
|
||||
workingDirectory: string,
|
||||
buildVolumeFolder: string,
|
||||
environmentVariables: any[],
|
||||
secrets: any[],
|
||||
): Promise<string> {
|
||||
async runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string> {
|
||||
// Task execution logic
|
||||
return 'Task completed';
|
||||
}
|
||||
@@ -190,7 +167,6 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
||||
## Supported URL Formats
|
||||
|
||||
### GitHub URLs
|
||||
|
||||
- `https://github.com/user/repo`
|
||||
- `https://github.com/user/repo.git`
|
||||
- `https://github.com/user/repo/tree/branch`
|
||||
@@ -198,27 +174,23 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
||||
- `git@github.com:user/repo.git`
|
||||
|
||||
### Shorthand GitHub References
|
||||
|
||||
- `user/repo`
|
||||
- `user/repo@branch`
|
||||
- `user/repo@branch/path/to/provider`
|
||||
|
||||
### Local Paths
|
||||
|
||||
- `./relative/path`
|
||||
- `../relative/path`
|
||||
- `/absolute/path`
|
||||
- `C:\\path\\to\\provider` (Windows)
|
||||
|
||||
### NPM Packages
|
||||
|
||||
- `package-name`
|
||||
- `@scope/package-name`
|
||||
|
||||
## Caching
|
||||
|
||||
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the
|
||||
repository owner, name, and branch. This ensures that:
|
||||
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the repository owner, name, and branch. This ensures that:
|
||||
|
||||
1. Repositories are only cloned once
|
||||
2. Updates are checked and applied automatically
|
||||
@@ -243,7 +215,7 @@ The provider loader can be configured through environment variables:
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use specific branches or versions**: Always specify the branch or specific tag when loading from GitHub
|
||||
1. **Use specific branches or tags**: Always specify the branch or specific tag when loading from GitHub
|
||||
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
||||
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
||||
4. **Test locally first**: Test providers locally before deploying
|
||||
@@ -0,0 +1,291 @@
|
||||
import AnsibleProvider from '.';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
jest.mock('../../services/core/orchestrator-system');
|
||||
jest.mock('../../services/core/orchestrator-logger');
|
||||
jest.mock('@actions/core', () => ({
|
||||
info: jest.fn(),
|
||||
warning: jest.fn(),
|
||||
error: jest.fn(),
|
||||
setOutput: jest.fn(),
|
||||
getInput: jest.fn(() => ''),
|
||||
}));
|
||||
|
||||
const mockRun = OrchestratorSystem.Run as jest.MockedFunction<typeof OrchestratorSystem.Run>;
|
||||
const mockLog = OrchestratorLogger.log as jest.MockedFunction<typeof OrchestratorLogger.log>;
|
||||
const mockLogWarning = OrchestratorLogger.logWarning as jest.MockedFunction<typeof OrchestratorLogger.logWarning>;
|
||||
|
||||
function createBuildParameters(overrides: Partial<BuildParameters> = {}): BuildParameters {
|
||||
return {
|
||||
ansibleInventory: '/etc/ansible/hosts',
|
||||
ansiblePlaybook: '/playbooks/unity-build.yml',
|
||||
ansibleExtraVars: '',
|
||||
ansibleVaultPassword: '',
|
||||
...overrides,
|
||||
} as BuildParameters;
|
||||
}
|
||||
|
||||
describe('AnsibleProvider', () => {
|
||||
let provider: AnsibleProvider;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
provider = new AnsibleProvider(createBuildParameters());
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('initializes with all provided parameters', () => {
|
||||
const params = createBuildParameters({
|
||||
ansibleInventory: '/custom/inventory',
|
||||
ansiblePlaybook: '/custom/playbook.yml',
|
||||
ansibleExtraVars: '{"key":"value"}',
|
||||
ansibleVaultPassword: '/vault/pass',
|
||||
});
|
||||
const p = new AnsibleProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
|
||||
it('handles missing optional parameters gracefully', () => {
|
||||
const params = createBuildParameters({
|
||||
ansiblePlaybook: undefined,
|
||||
ansibleExtraVars: undefined,
|
||||
ansibleVaultPassword: undefined,
|
||||
});
|
||||
const p = new AnsibleProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('setupWorkflow', () => {
|
||||
it('verifies ansible binary, ansible-playbook binary, and inventory exist', async () => {
|
||||
mockRun.mockResolvedValueOnce('ansible [core 2.14.0]'); // ansible --version
|
||||
mockRun.mockResolvedValueOnce('/usr/bin/ansible-playbook'); // ansible-playbook check
|
||||
mockRun.mockResolvedValueOnce(''); // test -e inventory
|
||||
|
||||
await provider.setupWorkflow('guid-123', createBuildParameters(), 'main', []);
|
||||
|
||||
expect(mockRun).toHaveBeenCalledTimes(3);
|
||||
expect(mockRun.mock.calls[0][0]).toContain('ansible --version');
|
||||
expect(mockRun.mock.calls[1][0]).toContain('ansible-playbook');
|
||||
expect(mockRun.mock.calls[2][0]).toContain('test -e "/etc/ansible/hosts"');
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('ansible'));
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('ansible-playbook binary verified'));
|
||||
});
|
||||
|
||||
it('throws when inventory is not configured', async () => {
|
||||
const params = createBuildParameters({ ansibleInventory: '' });
|
||||
provider = new AnsibleProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'ansibleInventory is required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when ansible binary is not found on PATH', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('command not found: ansible'));
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', createBuildParameters(), 'main', [])).rejects.toThrow(
|
||||
'Ansible not found on PATH',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when ansible-playbook binary is not found', async () => {
|
||||
mockRun.mockResolvedValueOnce('ansible [core 2.14.0]'); // ansible version OK
|
||||
mockRun.mockRejectedValueOnce(new Error('command not found')); // ansible-playbook missing
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', createBuildParameters(), 'main', [])).rejects.toThrow(
|
||||
'ansible-playbook not found on PATH',
|
||||
);
|
||||
|
||||
expect(core.error).toHaveBeenCalledWith('ansible-playbook not found. Install Ansible or ensure it is in PATH.');
|
||||
});
|
||||
|
||||
it('throws when inventory file does not exist', async () => {
|
||||
mockRun.mockResolvedValueOnce('ansible [core 2.14.0]'); // ansible version OK
|
||||
mockRun.mockResolvedValueOnce('/usr/bin/ansible-playbook'); // ansible-playbook OK
|
||||
mockRun.mockRejectedValueOnce(new Error('test -e failed')); // inventory missing
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', createBuildParameters(), 'main', [])).rejects.toThrow(
|
||||
'Inventory not found: /etc/ansible/hosts',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runTaskInWorkflow', () => {
|
||||
it('constructs ansible-playbook command with correct variables and returns output', async () => {
|
||||
mockRun.mockResolvedValueOnce('PLAY [build] *****\nok: [server1]\nPLAY RECAP');
|
||||
|
||||
const result = await provider.runTaskInWorkflow(
|
||||
'guid-run1',
|
||||
'unityci/editor:2021.3',
|
||||
'echo build',
|
||||
'/mount',
|
||||
'/workspace',
|
||||
[],
|
||||
[],
|
||||
);
|
||||
|
||||
expect(result).toContain('PLAY [build]');
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toContain('ansible-playbook');
|
||||
expect(command).toContain('-i "/etc/ansible/hosts"');
|
||||
expect(command).toContain('"/playbooks/unity-build.yml"');
|
||||
expect(command).toContain('--no-color');
|
||||
expect(command).toContain('build_guid');
|
||||
expect(command).toContain('guid-run1');
|
||||
expect(command).toContain('build_image');
|
||||
expect(command).toContain('unityci/editor:2021.3');
|
||||
expect(command).toContain('build_commands');
|
||||
expect(command).toContain('mount_dir');
|
||||
expect(command).toContain('working_dir');
|
||||
});
|
||||
|
||||
it('throws when playbook is not configured', async () => {
|
||||
const params = createBuildParameters({ ansiblePlaybook: '' });
|
||||
provider = new AnsibleProvider(params);
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-nopb', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'ansiblePlaybook is required',
|
||||
);
|
||||
});
|
||||
|
||||
it('passes environment variables as extra-vars in snake_case', async () => {
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
const env = [
|
||||
{ name: 'UNITY_LICENSE', value: 'lic-data' },
|
||||
{ name: 'BUILD_TARGET', value: 'Linux64' },
|
||||
];
|
||||
|
||||
await provider.runTaskInWorkflow('guid-env', 'img', 'cmd', '/m', '/w', env as any, []);
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
// Environment variable names are lowercased as Ansible variables
|
||||
expect(command).toContain('unity_license');
|
||||
expect(command).toContain('lic-data');
|
||||
expect(command).toContain('build_target');
|
||||
expect(command).toContain('Linux64');
|
||||
});
|
||||
|
||||
it('merges user-provided extra vars from JSON string', async () => {
|
||||
const params = createBuildParameters({
|
||||
ansibleExtraVars: JSON.stringify({ custom_var: 'custom_value', another: '42' }),
|
||||
});
|
||||
provider = new AnsibleProvider(params);
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
await provider.runTaskInWorkflow('guid-extra', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toContain('custom_var');
|
||||
expect(command).toContain('custom_value');
|
||||
expect(command).toContain('another');
|
||||
});
|
||||
|
||||
it('logs warning when extra vars JSON is invalid but continues', async () => {
|
||||
const params = createBuildParameters({ ansibleExtraVars: 'not-valid-json{{{' });
|
||||
provider = new AnsibleProvider(params);
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
await provider.runTaskInWorkflow('guid-badjson', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
expect(mockLogWarning).toHaveBeenCalledWith(expect.stringContaining('Failed to parse ansibleExtraVars'));
|
||||
});
|
||||
|
||||
it('includes vault password file flag when configured', async () => {
|
||||
const params = createBuildParameters({ ansibleVaultPassword: '/secure/vault-pass.txt' });
|
||||
provider = new AnsibleProvider(params);
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
await provider.runTaskInWorkflow('guid-vault', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toContain('--vault-password-file "/secure/vault-pass.txt"');
|
||||
});
|
||||
|
||||
it('does not include vault password flag when not configured', async () => {
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
await provider.runTaskInWorkflow('guid-novault', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).not.toContain('--vault-password-file');
|
||||
});
|
||||
|
||||
it('prefixes secrets as environment variables in the command', async () => {
|
||||
mockRun.mockResolvedValueOnce('ok');
|
||||
|
||||
const secrets = [
|
||||
{ ParameterKey: 'key1', EnvironmentVariable: 'SECRET_TOKEN', ParameterValue: 'tok-abc' },
|
||||
{ ParameterKey: 'key2', EnvironmentVariable: 'DEPLOY_KEY', ParameterValue: 'dk-xyz' },
|
||||
];
|
||||
|
||||
await provider.runTaskInWorkflow('guid-secrets', 'img', 'cmd', '/m', '/w', [], secrets as any);
|
||||
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toMatch(/^SECRET_TOKEN='tok-abc'/);
|
||||
expect(command).toContain("DEPLOY_KEY='dk-xyz'");
|
||||
expect(command).toContain('ansible-playbook');
|
||||
});
|
||||
|
||||
it('throws and logs warning when playbook execution fails', async () => {
|
||||
const execError = new Error('UNREACHABLE! Host unreachable');
|
||||
mockRun.mockRejectedValueOnce(execError);
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-hostfail', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'UNREACHABLE',
|
||||
);
|
||||
|
||||
expect(mockLogWarning).toHaveBeenCalledWith(expect.stringContaining('Playbook failed'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupWorkflow', () => {
|
||||
it('completes without error and logs cleanup message', async () => {
|
||||
await provider.cleanupWorkflow(createBuildParameters(), 'main', []);
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Cleanup complete'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('garbageCollect', () => {
|
||||
it('returns empty string (no-op)', async () => {
|
||||
const result = await provider.garbageCollect('', false, 0, false, false);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('listResources', () => {
|
||||
it('returns inventory path as a resource when configured', async () => {
|
||||
const resources = await provider.listResources();
|
||||
|
||||
expect(resources).toHaveLength(1);
|
||||
expect(resources[0].Name).toBe('/etc/ansible/hosts');
|
||||
});
|
||||
|
||||
it('returns empty array when inventory is not configured', async () => {
|
||||
const params = createBuildParameters({ ansibleInventory: '' });
|
||||
provider = new AnsibleProvider(params);
|
||||
|
||||
const resources = await provider.listResources();
|
||||
expect(resources).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listWorkflow', () => {
|
||||
it('returns empty array (not implemented)', async () => {
|
||||
const workflows = await provider.listWorkflow();
|
||||
expect(workflows).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('watchWorkflow', () => {
|
||||
it('returns empty string (not implemented)', async () => {
|
||||
const result = await provider.watchWorkflow();
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
import * as core from '@actions/core';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
/**
|
||||
* Ansible provider — executes Unity builds via Ansible playbooks
|
||||
* against managed inventory.
|
||||
*
|
||||
* Use case: Teams with existing Ansible infrastructure for server
|
||||
* management who want to leverage their inventory for build distribution.
|
||||
*/
|
||||
class AnsibleProvider implements ProviderInterface {
|
||||
private buildParameters: BuildParameters;
|
||||
private inventory: string;
|
||||
private playbook: string;
|
||||
private extraVariables: string;
|
||||
private vaultPassword: string;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.buildParameters = buildParameters;
|
||||
this.inventory = buildParameters.ansibleInventory || '';
|
||||
this.playbook = buildParameters.ansiblePlaybook || '';
|
||||
this.extraVariables = buildParameters.ansibleExtraVars || '';
|
||||
this.vaultPassword = buildParameters.ansibleVaultPassword || '';
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[Ansible] Setting up playbook execution`);
|
||||
|
||||
if (!this.inventory) {
|
||||
throw new Error('ansibleInventory is required for the ansible provider');
|
||||
}
|
||||
|
||||
// Verify ansible is available
|
||||
try {
|
||||
const version = await OrchestratorSystem.Run('ansible --version | head -1');
|
||||
OrchestratorLogger.log(`[Ansible] ${version.trim()}`);
|
||||
} catch (error: any) {
|
||||
throw new Error(`Ansible not found on PATH: ${error.message || error}`);
|
||||
}
|
||||
|
||||
// Verify ansible-playbook binary exists (may be separate from ansible)
|
||||
try {
|
||||
await OrchestratorSystem.Run('command -v ansible-playbook || which ansible-playbook || where ansible-playbook');
|
||||
OrchestratorLogger.log(`[Ansible] ansible-playbook binary verified`);
|
||||
} catch (error: any) {
|
||||
core.error('ansible-playbook not found. Install Ansible or ensure it is in PATH.');
|
||||
throw new Error(`ansible-playbook not found on PATH: ${error.message || error}`);
|
||||
}
|
||||
|
||||
// Verify inventory exists
|
||||
try {
|
||||
await OrchestratorSystem.Run(`test -e "${this.inventory}"`);
|
||||
} catch {
|
||||
throw new Error(`Inventory not found: ${this.inventory}`);
|
||||
}
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(`[Ansible] Running playbook against inventory ${this.inventory}`);
|
||||
|
||||
if (!this.playbook) {
|
||||
throw new Error(
|
||||
'ansiblePlaybook is required — no default playbook is provided yet. ' +
|
||||
'Provide a playbook that accepts build_guid, build_image, build_commands, mount_dir, and working_dir variables.',
|
||||
);
|
||||
}
|
||||
|
||||
// Build extra-vars JSON
|
||||
// These use snake_case because they are Ansible variable names passed to playbooks
|
||||
const playbookVariables: Record<string, string> = {
|
||||
// eslint-disable-next-line camelcase
|
||||
build_guid: buildGuid,
|
||||
// eslint-disable-next-line camelcase
|
||||
build_image: image,
|
||||
// eslint-disable-next-line camelcase
|
||||
build_commands: commands,
|
||||
// eslint-disable-next-line camelcase
|
||||
mount_dir: mountdir,
|
||||
// eslint-disable-next-line camelcase
|
||||
working_dir: workingdir,
|
||||
};
|
||||
|
||||
for (const element of environment) {
|
||||
playbookVariables[element.name.toLowerCase()] = element.value;
|
||||
}
|
||||
|
||||
// Merge user-provided extra vars
|
||||
if (this.extraVariables) {
|
||||
try {
|
||||
const userVariables = JSON.parse(this.extraVariables);
|
||||
Object.assign(playbookVariables, userVariables);
|
||||
} catch {
|
||||
OrchestratorLogger.logWarning(`[Ansible] Failed to parse ansibleExtraVars as JSON, using as-is`);
|
||||
}
|
||||
}
|
||||
|
||||
const extraVariablesJson = JSON.stringify(playbookVariables).replace(/'/g, "'\\''");
|
||||
|
||||
// Build ansible-playbook command
|
||||
const commandParts = [
|
||||
'ansible-playbook',
|
||||
`-i "${this.inventory}"`,
|
||||
`"${this.playbook}"`,
|
||||
`-e '${extraVariablesJson}'`,
|
||||
'--no-color',
|
||||
];
|
||||
|
||||
if (this.vaultPassword) {
|
||||
commandParts.push(`--vault-password-file "${this.vaultPassword}"`);
|
||||
}
|
||||
|
||||
// Add secret variables as extra environment
|
||||
const environmentPrefix = secrets
|
||||
.map((secret) => `${secret.EnvironmentVariable}='${secret.ParameterValue}'`)
|
||||
.join(' ');
|
||||
|
||||
const fullCommand = environmentPrefix ? `${environmentPrefix} ${commandParts.join(' ')}` : commandParts.join(' ');
|
||||
|
||||
try {
|
||||
const output = await OrchestratorSystem.Run(fullCommand);
|
||||
OrchestratorLogger.log(`[Ansible] Playbook completed successfully`);
|
||||
|
||||
return output;
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[Ansible] Playbook failed: ${error.message || error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[Ansible] Cleanup complete`);
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return '';
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
if (!this.inventory) return [];
|
||||
|
||||
const resource = new ProviderResource();
|
||||
resource.Name = this.inventory;
|
||||
|
||||
return [resource];
|
||||
}
|
||||
|
||||
async listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<string> {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
export default AnsibleProvider;
|
||||
+59
-16
@@ -1,14 +1,18 @@
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
import {
|
||||
CloudFormation,
|
||||
CreateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
CreateStackCommandInput,
|
||||
DescribeStacksCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
DescribeStacksCommandInput,
|
||||
ListStacksCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
Parameter,
|
||||
UpdateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
UpdateStackCommandInput,
|
||||
waitUntilStackCreateComplete,
|
||||
waitUntilStackUpdateComplete,
|
||||
@@ -16,6 +20,17 @@ import {
|
||||
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
export class AWSBaseStack {
|
||||
constructor(baseStackName: string) {
|
||||
this.baseStackName = baseStackName;
|
||||
@@ -24,6 +39,7 @@ export class AWSBaseStack {
|
||||
|
||||
async setupBaseStack(CF: CloudFormation) {
|
||||
const baseStackName = this.baseStackName;
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
|
||||
const baseStack = BaseStackFormation.formation;
|
||||
|
||||
@@ -54,18 +70,39 @@ export class AWSBaseStack {
|
||||
};
|
||||
|
||||
const stacks = await CF.send(
|
||||
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
|
||||
new ListStacksCommand({
|
||||
StackStatusFilter: [
|
||||
'CREATE_IN_PROGRESS',
|
||||
'UPDATE_IN_PROGRESS',
|
||||
'UPDATE_COMPLETE',
|
||||
'CREATE_COMPLETE',
|
||||
'ROLLBACK_COMPLETE',
|
||||
],
|
||||
}),
|
||||
);
|
||||
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
|
||||
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
|
||||
const stackExists: boolean = stackNames.includes(baseStackName);
|
||||
const describeStack = async () => {
|
||||
return await CF.send(new DescribeStacksCommand(describeStackInput));
|
||||
};
|
||||
try {
|
||||
if (!stackExists) {
|
||||
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
||||
await CF.send(new CreateStackCommand(createStackInput));
|
||||
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
|
||||
OrchestratorLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
||||
let created = false;
|
||||
try {
|
||||
await CF.send(new CreateStackCommand(createStackInput));
|
||||
created = true;
|
||||
} catch (error: any) {
|
||||
const message = `${error?.name ?? ''} ${error?.message ?? ''}`;
|
||||
if (message.includes('AlreadyExistsException')) {
|
||||
OrchestratorLogger.log(`Base stack already exists, continuing with describe`);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (created) {
|
||||
OrchestratorLogger.log(`created stack (version: ${parametersHash})`);
|
||||
}
|
||||
}
|
||||
const CFState = await describeStack();
|
||||
let stack = CFState.Stacks?.[0];
|
||||
@@ -75,32 +112,35 @@ export class AWSBaseStack {
|
||||
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
|
||||
|
||||
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
|
||||
OrchestratorLogger.log(
|
||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`,
|
||||
);
|
||||
await waitUntilStackCreateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: 200,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
describeStackInput,
|
||||
);
|
||||
}
|
||||
|
||||
if (stackExists) {
|
||||
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
||||
OrchestratorLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
||||
if (parametersHash !== stackVersion) {
|
||||
CloudRunnerLogger.log(`Attempting update of base stack`);
|
||||
OrchestratorLogger.log(`Attempting update of base stack`);
|
||||
try {
|
||||
await CF.send(new UpdateStackCommand(updateInput));
|
||||
} catch (error: any) {
|
||||
if (error['message'].includes('No updates are to be performed')) {
|
||||
CloudRunnerLogger.log(`No updates are to be performed`);
|
||||
OrchestratorLogger.log(`No updates are to be performed`);
|
||||
} else {
|
||||
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
||||
CloudRunnerLogger.log(error['message']);
|
||||
OrchestratorLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
||||
OrchestratorLogger.log(error['message']);
|
||||
}
|
||||
CloudRunnerLogger.log(`Continuing...`);
|
||||
OrchestratorLogger.log(`Continuing...`);
|
||||
}
|
||||
} else {
|
||||
CloudRunnerLogger.log(`No update required`);
|
||||
OrchestratorLogger.log(`No update required`);
|
||||
}
|
||||
stack = (await describeStack()).Stacks?.[0];
|
||||
if (!stack) {
|
||||
@@ -109,16 +149,19 @@ export class AWSBaseStack {
|
||||
);
|
||||
}
|
||||
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
|
||||
OrchestratorLogger.log(
|
||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`,
|
||||
);
|
||||
await waitUntilStackUpdateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: 200,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
describeStackInput,
|
||||
);
|
||||
}
|
||||
}
|
||||
CloudRunnerLogger.log('base stack is now ready');
|
||||
OrchestratorLogger.log('base stack is now ready');
|
||||
} catch (error) {
|
||||
core.error(JSON.stringify(await describeStack(), undefined, 4));
|
||||
throw error;
|
||||
+28
-6
@@ -4,7 +4,7 @@ import { Kinesis } from '@aws-sdk/client-kinesis';
|
||||
import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { S3 } from '@aws-sdk/client-s3';
|
||||
import { Input } from '../../..';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
|
||||
export class AwsClientFactory {
|
||||
private static cloudFormation: CloudFormation;
|
||||
@@ -13,11 +13,29 @@ export class AwsClientFactory {
|
||||
private static cloudWatchLogs: CloudWatchLogs;
|
||||
private static s3: S3;
|
||||
|
||||
private static getCredentials() {
|
||||
// Explicitly provide credentials from environment variables for LocalStack compatibility
|
||||
// LocalStack accepts any credentials, but the AWS SDK needs them to be explicitly set
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
|
||||
if (accessKeyId && secretAccessKey) {
|
||||
return {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
}
|
||||
|
||||
// Return undefined to let AWS SDK use default credential chain
|
||||
return;
|
||||
}
|
||||
|
||||
static getCloudFormation(): CloudFormation {
|
||||
if (!this.cloudFormation) {
|
||||
this.cloudFormation = new CloudFormation({
|
||||
region: Input.region,
|
||||
endpoint: CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||
endpoint: OrchestratorOptions.awsCloudFormationEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -28,7 +46,8 @@ export class AwsClientFactory {
|
||||
if (!this.ecs) {
|
||||
this.ecs = new ECS({
|
||||
region: Input.region,
|
||||
endpoint: CloudRunnerOptions.awsEcsEndpoint,
|
||||
endpoint: OrchestratorOptions.awsEcsEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -39,7 +58,8 @@ export class AwsClientFactory {
|
||||
if (!this.kinesis) {
|
||||
this.kinesis = new Kinesis({
|
||||
region: Input.region,
|
||||
endpoint: CloudRunnerOptions.awsKinesisEndpoint,
|
||||
endpoint: OrchestratorOptions.awsKinesisEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -50,7 +70,8 @@ export class AwsClientFactory {
|
||||
if (!this.cloudWatchLogs) {
|
||||
this.cloudWatchLogs = new CloudWatchLogs({
|
||||
region: Input.region,
|
||||
endpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||
endpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -61,8 +82,9 @@ export class AwsClientFactory {
|
||||
if (!this.s3) {
|
||||
this.s3 = new S3({
|
||||
region: Input.region,
|
||||
endpoint: CloudRunnerOptions.awsS3Endpoint,
|
||||
endpoint: OrchestratorOptions.awsS3Endpoint,
|
||||
forcePathStyle: true,
|
||||
credentials: AwsClientFactory.getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
+1
@@ -21,6 +21,7 @@ export class AWSCloudFormationTemplates {
|
||||
|
||||
public static getSecretDefinitionTemplate(p1: string, p2: string) {
|
||||
return `
|
||||
Secrets:
|
||||
- Name: '${p1}'
|
||||
ValueFrom: !Ref ${p2}Secret
|
||||
`;
|
||||
+6
-6
@@ -1,16 +1,16 @@
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
|
||||
export class AWSError {
|
||||
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
|
||||
CloudRunnerLogger.log('aws error: ');
|
||||
OrchestratorLogger.log('aws error: ');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log('Getting events and resources for task stack');
|
||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
||||
OrchestratorLogger.log('Getting events and resources for task stack');
|
||||
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
|
||||
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
|
||||
OrchestratorLogger.log(JSON.stringify(events, undefined, 4));
|
||||
}
|
||||
}
|
||||
}
|
||||
+33
-18
@@ -1,22 +1,34 @@
|
||||
import {
|
||||
CloudFormation,
|
||||
CreateStackCommand,
|
||||
// eslint-disable-next-line import/named
|
||||
CreateStackCommandInput,
|
||||
DescribeStackResourcesCommand,
|
||||
DescribeStacksCommand,
|
||||
ListStacksCommand,
|
||||
waitUntilStackCreateComplete,
|
||||
} from '@aws-sdk/client-cloudformation';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { AWSError } from './aws-error';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
export class AWSJobStack {
|
||||
private baseStackName: string;
|
||||
constructor(baseStackName: string) {
|
||||
@@ -31,23 +43,23 @@ export class AWSJobStack {
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<CloudRunnerAWSTaskDef> {
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<OrchestratorAWSTaskDef> {
|
||||
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
||||
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||
`ContainerCpu:
|
||||
Default: 1024`,
|
||||
`ContainerCpu:
|
||||
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
|
||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerCpu)}`,
|
||||
);
|
||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||
`ContainerMemory:
|
||||
Default: 2048`,
|
||||
`ContainerMemory:
|
||||
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
|
||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerMemory)}`,
|
||||
);
|
||||
if (!CloudRunnerOptions.asyncCloudRunner) {
|
||||
if (!OrchestratorOptions.asyncOrchestrator) {
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'# template resources logstream',
|
||||
@@ -121,8 +133,8 @@ export class AWSJobStack {
|
||||
},
|
||||
...secretsMappedToCloudFormationParameters,
|
||||
];
|
||||
CloudRunnerLogger.log(
|
||||
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
|
||||
OrchestratorLogger.log(
|
||||
`Starting AWS job with memory: ${Orchestrator.buildParameters.containerMemory} cpu: ${Orchestrator.buildParameters.containerCpu}`,
|
||||
);
|
||||
let previousStackExists = true;
|
||||
while (previousStackExists) {
|
||||
@@ -135,7 +147,7 @@ export class AWSJobStack {
|
||||
const element = stacks.StackSummaries[index];
|
||||
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
||||
previousStackExists = true;
|
||||
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
||||
OrchestratorLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
}
|
||||
}
|
||||
@@ -147,12 +159,15 @@ export class AWSJobStack {
|
||||
Parameters: parameters,
|
||||
};
|
||||
try {
|
||||
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
OrchestratorLogger.log(
|
||||
`Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`,
|
||||
);
|
||||
await CF.send(new CreateStackCommand(createStackInput));
|
||||
await waitUntilStackCreateComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: 200,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{ StackName: taskDefStackName },
|
||||
);
|
||||
@@ -186,7 +201,7 @@ export class AWSJobStack {
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDGUID',
|
||||
ParameterValue: CloudRunner.buildParameters.buildGuid,
|
||||
ParameterValue: Orchestrator.buildParameters.buildGuid,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EnvironmentName',
|
||||
@@ -194,9 +209,9 @@ export class AWSJobStack {
|
||||
},
|
||||
],
|
||||
};
|
||||
if (CloudRunnerOptions.useCleanupCron) {
|
||||
if (OrchestratorOptions.useCleanupCron) {
|
||||
try {
|
||||
CloudRunnerLogger.log(`Creating job cleanup formation`);
|
||||
OrchestratorLogger.log(`Creating job cleanup formation`);
|
||||
await CF.send(new CreateStackCommand(createCleanupStackInput));
|
||||
|
||||
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
|
||||
+72
-30
@@ -1,23 +1,61 @@
|
||||
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
|
||||
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
|
||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import * as zlib from 'node:zlib';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { Input } from '../../..';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import GitHub from '../../../github';
|
||||
import { AwsClientFactory } from './aws-client-factory';
|
||||
|
||||
class AWSTaskRunner {
|
||||
private static readonly encodedUnderscore = `$252F`;
|
||||
|
||||
/**
|
||||
* Transform localhost endpoints to host.docker.internal for container environments.
|
||||
* When LocalStack is used, ECS tasks run in Docker containers that need to reach
|
||||
* LocalStack on the host machine via host.docker.internal.
|
||||
*/
|
||||
private static transformEndpointsForContainer(
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
): OrchestratorEnvironmentVariable[] {
|
||||
const endpointEnvironmentNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
|
||||
return environment.map((x) => {
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvironmentNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
// Replace localhost with host.docker.internal so ECS containers can access host services
|
||||
value = value
|
||||
.replace('http://localhost', 'http://host.docker.internal')
|
||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
||||
OrchestratorLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
||||
}
|
||||
|
||||
return { name: x.name, value };
|
||||
});
|
||||
}
|
||||
|
||||
static async runTask(
|
||||
taskDef: CloudRunnerAWSTaskDef,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
taskDef: OrchestratorAWSTaskDef,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
commands: string,
|
||||
): Promise<{ output: string; shouldCleanup: boolean }> {
|
||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
@@ -32,6 +70,9 @@ class AWSTaskRunner {
|
||||
const streamName =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||
|
||||
// Transform localhost endpoints for container environment
|
||||
const transformedEnvironment = AWSTaskRunner.transformEndpointsForContainer(environment);
|
||||
|
||||
const runParameters = {
|
||||
cluster,
|
||||
taskDefinition,
|
||||
@@ -40,8 +81,8 @@ class AWSTaskRunner {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment,
|
||||
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
|
||||
environment: transformedEnvironment,
|
||||
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, Orchestrator.buildParameters)],
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -56,28 +97,28 @@ class AWSTaskRunner {
|
||||
};
|
||||
|
||||
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
|
||||
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
||||
OrchestratorLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
||||
throw new Error(`Container Overrides length must be at most 8192`);
|
||||
}
|
||||
|
||||
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
CloudRunnerLogger.log('Cloud runner job is starting');
|
||||
OrchestratorLogger.log('Orchestrator job is starting');
|
||||
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
||||
CloudRunnerLogger.log(
|
||||
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
||||
CloudRunnerOptions.asyncCloudRunner
|
||||
OrchestratorLogger.log(
|
||||
`Orchestrator job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
||||
OrchestratorOptions.asyncOrchestrator
|
||||
}`,
|
||||
);
|
||||
if (CloudRunnerOptions.asyncCloudRunner) {
|
||||
if (OrchestratorOptions.asyncOrchestrator) {
|
||||
const shouldCleanup: boolean = false;
|
||||
const output: string = '';
|
||||
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
|
||||
OrchestratorLogger.log(`Watch Orchestrator To End: false`);
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Streaming...`);
|
||||
OrchestratorLogger.log(`Streaming...`);
|
||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
||||
let exitCode;
|
||||
let containerState;
|
||||
@@ -92,13 +133,13 @@ class AWSTaskRunner {
|
||||
containerState = containers[0];
|
||||
exitCode = containerState?.exitCode;
|
||||
}
|
||||
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||
OrchestratorLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||
if (exitCode === undefined) {
|
||||
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
|
||||
OrchestratorLogger.logWarning(`Undefined exitcode for container`);
|
||||
}
|
||||
const wasSuccessful = exitCode === 0;
|
||||
if (wasSuccessful) {
|
||||
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
||||
OrchestratorLogger.log(`Orchestrator job has finished successfully`);
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
@@ -125,7 +166,7 @@ class AWSTaskRunner {
|
||||
const error = error_ as Error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
CloudRunnerLogger.log(`Cloud runner job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
|
||||
OrchestratorLogger.log(`Orchestrator job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
|
||||
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
@@ -152,7 +193,7 @@ class AWSTaskRunner {
|
||||
}
|
||||
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
|
||||
const sleepMs = delayMs + jitterMs;
|
||||
CloudRunnerLogger.log(
|
||||
OrchestratorLogger.log(
|
||||
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, sleepMs));
|
||||
@@ -163,12 +204,12 @@ class AWSTaskRunner {
|
||||
|
||||
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
CloudRunnerLogger.log(`Streaming...`);
|
||||
OrchestratorLogger.log(`Streaming...`);
|
||||
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
||||
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
||||
|
||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
|
||||
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${Orchestrator.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${Orchestrator.buildParameters.awsStackName}-${Orchestrator.buildParameters.buildGuid}`;
|
||||
OrchestratorLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
||||
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
|
||||
let shouldReadLogs = true;
|
||||
let shouldCleanup = true;
|
||||
@@ -207,8 +248,9 @@ class AWSTaskRunner {
|
||||
const baseBackoffMs = 1000;
|
||||
const jitterMs = Math.floor(Math.random() * 1000);
|
||||
const sleepMs = baseBackoffMs + jitterMs;
|
||||
CloudRunnerLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
|
||||
OrchestratorLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
|
||||
await new Promise((r) => setTimeout(r, sleepMs));
|
||||
|
||||
return { iterator, shouldReadLogs, output, shouldCleanup };
|
||||
}
|
||||
throw error;
|
||||
@@ -227,18 +269,18 @@ class AWSTaskRunner {
|
||||
|
||||
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
|
||||
if (taskData?.lastStatus === 'UNKNOWN') {
|
||||
CloudRunnerLogger.log('## Cloud runner job unknwon');
|
||||
OrchestratorLogger.log('## Orchestrator job unknwon');
|
||||
}
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
if (timestamp === 0) {
|
||||
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
|
||||
OrchestratorLogger.log('## Orchestrator job stopped, streaming end of logs');
|
||||
timestamp = Date.now();
|
||||
}
|
||||
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
|
||||
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs');
|
||||
OrchestratorLogger.log('## Orchestrator status is not RUNNING for 30 seconds, last query for logs');
|
||||
shouldReadLogs = false;
|
||||
}
|
||||
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
||||
OrchestratorLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
||||
}
|
||||
|
||||
return { timestamp, shouldReadLogs };
|
||||
+5
-6
@@ -1,7 +1,7 @@
|
||||
import CloudRunner from '../../../cloud-runner';
|
||||
import Orchestrator from '../../../orchestrator';
|
||||
|
||||
export class TaskDefinitionFormation {
|
||||
public static readonly description: string = `Game CI Cloud Runner Task Stack`;
|
||||
public static readonly description: string = `Game CI Orchestrator Task Stack`;
|
||||
public static get formation(): string {
|
||||
return `AWSTemplateFormatVersion: 2010-09-09
|
||||
Description: ${TaskDefinitionFormation.description}
|
||||
@@ -29,11 +29,11 @@ Parameters:
|
||||
Default: 80
|
||||
Description: What port number the application inside the docker container is binding to
|
||||
ContainerCpu:
|
||||
Default: ${CloudRunner.buildParameters.containerCpu}
|
||||
Default: ${Orchestrator.buildParameters.containerCpu}
|
||||
Type: Number
|
||||
Description: How much CPU to give the container. 1024 is 1 CPU
|
||||
ContainerMemory:
|
||||
Default: ${CloudRunner.buildParameters.containerMemory}
|
||||
Default: ${Orchestrator.buildParameters.containerMemory}
|
||||
Type: Number
|
||||
Description: How much memory in megabytes to give the container
|
||||
BUILDGUID:
|
||||
@@ -127,8 +127,7 @@ Resources:
|
||||
- SourceVolume: efs-data
|
||||
ContainerPath: !Ref EFSMountDirectory
|
||||
ReadOnly: false
|
||||
Secrets:
|
||||
# template secrets p3 - container def
|
||||
# template secrets p3 - container def
|
||||
LogConfiguration:
|
||||
LogDriver: awslogs
|
||||
Options:
|
||||
+34
-19
@@ -1,11 +1,11 @@
|
||||
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
|
||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
||||
import AwsTaskRunner from './aws-task-runner';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
||||
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
||||
import { Input } from '../../..';
|
||||
@@ -13,8 +13,20 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { TaskService } from './services/task-service';
|
||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
||||
import { AwsClientFactory } from './aws-client-factory';
|
||||
import ResourceTracking from '../../services/core/resource-tracking';
|
||||
|
||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
||||
|
||||
function getStackWaitTime(): number {
|
||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
||||
return overrideValue;
|
||||
}
|
||||
|
||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
||||
}
|
||||
|
||||
class AWSBuildEnvironment implements ProviderInterface {
|
||||
private baseStackName: string;
|
||||
@@ -86,14 +98,16 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
ResourceTracking.logAllocationSummary('aws workflow');
|
||||
await ResourceTracking.logDiskUsageSnapshot('aws workflow (host)');
|
||||
AwsClientFactory.getECS();
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
AwsClientFactory.getKinesis();
|
||||
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
|
||||
OrchestratorLogger.log(`AWS Region: ${CF.config.region}`);
|
||||
const entrypoint = ['/bin/sh'];
|
||||
const startTimeMs = Date.now();
|
||||
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
||||
@@ -110,36 +124,37 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
let postRunTaskTimeMs;
|
||||
try {
|
||||
const postSetupStacksTimeMs = Date.now();
|
||||
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
||||
OrchestratorLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
||||
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
||||
postRunTaskTimeMs = Date.now();
|
||||
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
||||
OrchestratorLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
||||
if (shouldCleanup) {
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
const postCleanupTimeMs = Date.now();
|
||||
if (postRunTaskTimeMs !== undefined)
|
||||
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
||||
OrchestratorLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`error running task ${error}`);
|
||||
OrchestratorLogger.log(`error running task ${error}`);
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
|
||||
CloudRunnerLogger.log('Cleanup starting');
|
||||
async cleanupResources(CF: CloudFormation, taskDef: OrchestratorAWSTaskDef) {
|
||||
const stackWaitTimeSeconds = getStackWaitTime();
|
||||
OrchestratorLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`);
|
||||
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
|
||||
if (CloudRunnerOptions.useCleanupCron) {
|
||||
if (OrchestratorOptions.useCleanupCron) {
|
||||
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
|
||||
}
|
||||
|
||||
await waitUntilStackDeleteComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: 200,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{
|
||||
StackName: taskDef.taskDefStackName,
|
||||
@@ -148,14 +163,14 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
await waitUntilStackDeleteComplete(
|
||||
{
|
||||
client: CF,
|
||||
maxWaitTime: 200,
|
||||
maxWaitTime: stackWaitTimeSeconds,
|
||||
},
|
||||
{
|
||||
StackName: `${taskDef.taskDefStackName}-cleanup`,
|
||||
},
|
||||
);
|
||||
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
||||
CloudRunnerLogger.log('Cleanup complete');
|
||||
OrchestratorLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
||||
OrchestratorLogger.log('Cleanup complete');
|
||||
}
|
||||
}
|
||||
export default AWSBuildEnvironment;
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
// eslint-disable-next-line import/named
|
||||
import { StackResource } from '@aws-sdk/client-cloudformation';
|
||||
|
||||
class CloudRunnerAWSTaskDef {
|
||||
class OrchestratorAWSTaskDef {
|
||||
public taskDefStackName!: string;
|
||||
public taskDefCloudFormation!: string;
|
||||
public taskDefResources: StackResource[] | undefined;
|
||||
public baseResources: StackResource[] | undefined;
|
||||
}
|
||||
export default CloudRunnerAWSTaskDef;
|
||||
export default OrchestratorAWSTaskDef;
|
||||
+7
-7
@@ -2,7 +2,7 @@ import { DeleteStackCommand, DescribeStackResourcesCommand } from '@aws-sdk/clie
|
||||
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { StopTaskCommand } from '@aws-sdk/client-ecs';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
||||
import { TaskService } from './task-service';
|
||||
import { AwsClientFactory } from '../aws-client-factory';
|
||||
|
||||
@@ -25,7 +25,7 @@ export class GarbageCollectionService {
|
||||
const { taskElement, element } = task;
|
||||
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
|
||||
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||
OrchestratorLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ export class GarbageCollectionService {
|
||||
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
||||
)
|
||||
) {
|
||||
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
||||
OrchestratorLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
||||
|
||||
return;
|
||||
}
|
||||
@@ -47,12 +47,12 @@ export class GarbageCollectionService {
|
||||
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
|
||||
) {
|
||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
||||
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||
OrchestratorLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Deleting ${element.StackName}`);
|
||||
OrchestratorLogger.log(`Deleting ${element.StackName}`);
|
||||
await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
|
||||
}
|
||||
}
|
||||
@@ -62,14 +62,14 @@ export class GarbageCollectionService {
|
||||
deleteResources &&
|
||||
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
|
||||
) {
|
||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||
OrchestratorLogger.log(`Deleting ${element.logGroupName}`);
|
||||
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
|
||||
}
|
||||
}
|
||||
|
||||
const locks = await TaskService.getLocks();
|
||||
for (const element of locks) {
|
||||
CloudRunnerLogger.log(`Lock: ${element.Key}`);
|
||||
OrchestratorLogger.log(`Lock: ${element.Key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
+73
-66
@@ -3,24 +3,21 @@ import {
|
||||
DescribeStacksCommand,
|
||||
ListStacksCommand,
|
||||
} from '@aws-sdk/client-cloudformation';
|
||||
import type { ListStacksCommandOutput } from '@aws-sdk/client-cloudformation';
|
||||
import { DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import type { DescribeLogGroupsCommandInput, DescribeLogGroupsCommandOutput } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import type { StackSummary } from '@aws-sdk/client-cloudformation';
|
||||
// eslint-disable-next-line import/named
|
||||
import { DescribeLogGroupsCommand, DescribeLogGroupsCommandInput } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import type { LogGroup } from '@aws-sdk/client-cloudwatch-logs';
|
||||
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
||||
import type { DescribeTasksCommandOutput } from '@aws-sdk/client-ecs';
|
||||
import { ListObjectsCommand } from '@aws-sdk/client-s3';
|
||||
import type { Task } from '@aws-sdk/client-ecs';
|
||||
import { ListObjectsV2Command } from '@aws-sdk/client-s3';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
||||
import AwsTaskRunner from '../aws-task-runner';
|
||||
import CloudRunner from '../../../cloud-runner';
|
||||
import Orchestrator from '../../../orchestrator';
|
||||
import { AwsClientFactory } from '../aws-client-factory';
|
||||
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
||||
|
||||
type StackSummary = NonNullable<ListStacksCommandOutput['StackSummaries']>[number];
|
||||
type LogGroup = NonNullable<DescribeLogGroupsCommandOutput['logGroups']>[number];
|
||||
type Task = NonNullable<DescribeTasksCommandOutput['tasks']>[number];
|
||||
|
||||
export class TaskService {
|
||||
static async watch() {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -32,10 +29,10 @@ export class TaskService {
|
||||
|
||||
return output;
|
||||
}
|
||||
public static async getCloudFormationJobStacks() {
|
||||
public static async getCloudFormationJobStacks(): Promise<StackSummary[]> {
|
||||
const result: StackSummary[] = [];
|
||||
CloudRunnerLogger.log(``);
|
||||
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`List Cloud Formation Stacks`);
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = AwsClientFactory.getCloudFormation();
|
||||
const stacks =
|
||||
@@ -43,16 +40,16 @@ export class TaskService {
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
CloudRunnerLogger.log(``);
|
||||
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
||||
for (const element of stacks) {
|
||||
if (!element.CreationTime) {
|
||||
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
|
||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
@@ -64,68 +61,73 @@ export class TaskService {
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
CloudRunnerLogger.log(``);
|
||||
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||
for (const element of baseStacks) {
|
||||
if (!element.CreationTime) {
|
||||
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
|
||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
result.push(element);
|
||||
}
|
||||
CloudRunnerLogger.log(``);
|
||||
OrchestratorLogger.log(``);
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async getTasks() {
|
||||
// Extended Task type to include custom properties added in this method
|
||||
type ExtendedTask = Task & {
|
||||
overrides?: Record<string, unknown>;
|
||||
attachments?: unknown[];
|
||||
};
|
||||
const result: { taskElement: ExtendedTask; element: string }[] = [];
|
||||
CloudRunnerLogger.log(``);
|
||||
CloudRunnerLogger.log(`List Tasks`);
|
||||
public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> {
|
||||
const result: { taskElement: Task; element: string }[] = [];
|
||||
OrchestratorLogger.log(``);
|
||||
OrchestratorLogger.log(`List Tasks`);
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = AwsClientFactory.getECS();
|
||||
const clusters = (await ecs.send(new ListClustersCommand({}))).clusterArns || [];
|
||||
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
|
||||
const clusters: string[] = [];
|
||||
{
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const clusterResponse = await ecs.send(new ListClustersCommand({ nextToken }));
|
||||
clusters.push(...(clusterResponse.clusterArns ?? []));
|
||||
nextToken = clusterResponse.nextToken;
|
||||
} while (nextToken);
|
||||
}
|
||||
OrchestratorLogger.log(`Task Clusters ${clusters.length}`);
|
||||
for (const element of clusters) {
|
||||
const input = {
|
||||
cluster: element,
|
||||
};
|
||||
const list = (await ecs.send(new ListTasksCommand(input))).taskArns || [];
|
||||
if (list.length > 0) {
|
||||
const describeInput = { tasks: list, cluster: element };
|
||||
const taskArns: string[] = [];
|
||||
{
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const taskResponse = await ecs.send(new ListTasksCommand({ cluster: element, nextToken }));
|
||||
taskArns.push(...(taskResponse.taskArns ?? []));
|
||||
nextToken = taskResponse.nextToken;
|
||||
} while (nextToken);
|
||||
}
|
||||
if (taskArns.length > 0) {
|
||||
const describeInput = { tasks: taskArns, cluster: element };
|
||||
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
|
||||
if (describeList.length === 0) {
|
||||
CloudRunnerLogger.log(`No Tasks`);
|
||||
OrchestratorLogger.log(`No Tasks`);
|
||||
continue;
|
||||
}
|
||||
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
|
||||
OrchestratorLogger.log(`Tasks ${describeList.length}`);
|
||||
for (const taskElement of describeList) {
|
||||
if (taskElement === undefined) {
|
||||
continue;
|
||||
}
|
||||
const extendedTask = taskElement as ExtendedTask;
|
||||
extendedTask.overrides = {};
|
||||
extendedTask.attachments = [];
|
||||
if (extendedTask.createdAt === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${extendedTask.taskDefinitionArn} no createdAt date`);
|
||||
if (taskElement.createdAt === undefined) {
|
||||
OrchestratorLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
result.push({ taskElement: extendedTask, element });
|
||||
result.push({ taskElement, element });
|
||||
}
|
||||
}
|
||||
}
|
||||
CloudRunnerLogger.log(``);
|
||||
OrchestratorLogger.log(``);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -141,7 +143,7 @@ export class TaskService {
|
||||
throw new Error('stack not defined');
|
||||
}
|
||||
if (!stack.CreationTime) {
|
||||
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
||||
OrchestratorLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
|
||||
const message = `
|
||||
@@ -151,40 +153,43 @@ export class TaskService {
|
||||
${JSON.stringify(stackInfo, undefined, 4)}
|
||||
${JSON.stringify(stackInfo2, undefined, 4)}
|
||||
`;
|
||||
CloudRunnerLogger.log(message);
|
||||
OrchestratorLogger.log(message);
|
||||
|
||||
return message;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.error(
|
||||
OrchestratorLogger.error(
|
||||
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
public static async getLogGroups() {
|
||||
public static async getLogGroups(): Promise<LogGroup[]> {
|
||||
const result: LogGroup[] = [];
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = AwsClientFactory.getCloudWatchLogs();
|
||||
const cwl = AwsClientFactory.getCloudWatchLogs();
|
||||
let logStreamInput: DescribeLogGroupsCommandInput = {
|
||||
/* logGroupNamePrefix: 'game-ci' */
|
||||
};
|
||||
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
let logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
const logGroups = logGroupsDescribe.logGroups || [];
|
||||
while (logGroupsDescribe.nextToken) {
|
||||
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
|
||||
logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
logStreamInput = {
|
||||
/* logGroupNamePrefix: 'game-ci',*/
|
||||
nextToken: logGroupsDescribe.nextToken,
|
||||
};
|
||||
logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
|
||||
OrchestratorLogger.log(`Log Groups ${logGroups.length}`);
|
||||
for (const element of logGroups) {
|
||||
if (element.creationTime === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||
OrchestratorLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
OrchestratorLogger.log(
|
||||
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
@@ -194,20 +199,22 @@ export class TaskService {
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async getLocks() {
|
||||
public static async getLocks(): Promise<Array<{ Key: string }>> {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
if (CloudRunner.buildParameters.storageProvider === 'rclone') {
|
||||
const objects = await (SharedWorkspaceLocking as any).listObjects('');
|
||||
if (Orchestrator.buildParameters.storageProvider === 'rclone') {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
type ListObjectsFunction = (prefix: string) => Promise<string[]>;
|
||||
const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects('');
|
||||
|
||||
return objects.map((x: string) => ({ Key: x }));
|
||||
}
|
||||
const s3 = AwsClientFactory.getS3();
|
||||
const listRequest = {
|
||||
Bucket: CloudRunner.buildParameters.awsStackName,
|
||||
Bucket: Orchestrator.buildParameters.awsStackName,
|
||||
};
|
||||
|
||||
const results = await s3.send(new ListObjectsCommand(listRequest));
|
||||
const results = await s3.send(new ListObjectsV2Command(listRequest));
|
||||
|
||||
return results.Contents || [];
|
||||
return (results.Contents || []).map((object) => ({ Key: object.Key || '' }));
|
||||
}
|
||||
}
|
||||
+60
-27
@@ -1,20 +1,20 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import Docker from '../../../docker';
|
||||
import { Action } from '../../..';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import Orchestrator from '../../orchestrator';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import * as fs from 'node:fs';
|
||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||
import { StringKeyValuePair } from '../../../shared-types';
|
||||
|
||||
class LocalDockerCloudRunner implements ProviderInterface {
|
||||
class LocalDockerOrchestrator implements ProviderInterface {
|
||||
public buildParameters!: BuildParameters;
|
||||
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
@@ -50,15 +50,15 @@ class LocalDockerCloudRunner implements ProviderInterface {
|
||||
const { workspace } = Action;
|
||||
if (
|
||||
fs.existsSync(
|
||||
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
`${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
)
|
||||
) {
|
||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache/cache/build/`);
|
||||
await OrchestratorSystem.Run(
|
||||
`rm -r ${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||
}`,
|
||||
);
|
||||
}
|
||||
@@ -80,22 +80,47 @@ class LocalDockerCloudRunner implements ProviderInterface {
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
OrchestratorLogger.log(buildGuid);
|
||||
OrchestratorLogger.log(commands);
|
||||
|
||||
const { workspace, actionFolder } = Action;
|
||||
const content: StringKeyValuePair[] = [];
|
||||
for (const x of secrets) {
|
||||
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
||||
}
|
||||
|
||||
// Replace localhost with host.docker.internal for LocalStack endpoints (similar to K8s)
|
||||
// This allows Docker containers to access LocalStack running on the host
|
||||
const endpointEnvironmentNames = new Set([
|
||||
'AWS_S3_ENDPOINT',
|
||||
'AWS_ENDPOINT',
|
||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||
'AWS_ECS_ENDPOINT',
|
||||
'AWS_KINESIS_ENDPOINT',
|
||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||
'INPUT_AWSS3ENDPOINT',
|
||||
'INPUT_AWSENDPOINT',
|
||||
]);
|
||||
for (const x of environment) {
|
||||
content.push({ name: x.name, value: x.value });
|
||||
let value = x.value;
|
||||
if (
|
||||
typeof value === 'string' &&
|
||||
endpointEnvironmentNames.has(x.name) &&
|
||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||
) {
|
||||
// Replace localhost with host.docker.internal so containers can access host services
|
||||
value = value
|
||||
.replace('http://localhost', 'http://host.docker.internal')
|
||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
||||
OrchestratorLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
||||
}
|
||||
content.push({ name: x.name, value });
|
||||
}
|
||||
|
||||
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
|
||||
// if (this.buildParameters?.orchestratorIntegrationTests) {
|
||||
// core.info(JSON.stringify(content, undefined, 4));
|
||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||
// core.info(JSON.stringify(environment, undefined, 4));
|
||||
@@ -112,25 +137,33 @@ class LocalDockerCloudRunner implements ProviderInterface {
|
||||
|
||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||
const entrypointFilePath = `start.sh`;
|
||||
const fileContents = `#!/bin/bash
|
||||
|
||||
// Use #!/bin/sh for POSIX compatibility (Alpine-based images like rclone/rclone don't have bash)
|
||||
const fileContents = `#!/bin/sh
|
||||
set -e
|
||||
|
||||
mkdir -p /github/workspace/cloud-runner-cache
|
||||
mkdir -p /github/workspace/orchestrator-cache
|
||||
mkdir -p /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
||||
cp -a /github/workspace/orchestrator-cache/. ${sharedFolder}
|
||||
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
|
||||
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||
# Only copy cache directory, exclude retained workspaces to avoid running out of disk space
|
||||
if [ -d "${sharedFolder}cache" ]; then
|
||||
cp -a ${sharedFolder}cache/. /github/workspace/orchestrator-cache/cache/ || true
|
||||
fi
|
||||
# Copy test files from /data/ root to workspace for test assertions
|
||||
# This allows tests to write files to /data/ and have them available in the workspace
|
||||
find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/orchestrator-cache/ \\; || true
|
||||
`;
|
||||
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
||||
flag: 'w',
|
||||
});
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
|
||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
||||
OrchestratorLogger.log(`Running local-docker: \n ${fileContents}`);
|
||||
}
|
||||
|
||||
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
|
||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
|
||||
if (fs.existsSync(`${workspace}/orchestrator-cache`)) {
|
||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache && du -sh ${workspace}/orchestrator-cache`);
|
||||
}
|
||||
const exitCode = await Docker.run(
|
||||
image,
|
||||
@@ -160,4 +193,4 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||
return myOutput;
|
||||
}
|
||||
}
|
||||
export default LocalDockerCloudRunner;
|
||||
export default LocalDockerOrchestrator;
|
||||
@@ -0,0 +1,333 @@
|
||||
import GitHubActionsProvider from '.';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
jest.mock('../../services/core/orchestrator-system');
|
||||
jest.mock('../../services/core/orchestrator-logger');
|
||||
jest.mock('@actions/core', () => ({
|
||||
info: jest.fn(),
|
||||
warning: jest.fn(),
|
||||
error: jest.fn(),
|
||||
setOutput: jest.fn(),
|
||||
getInput: jest.fn(() => ''),
|
||||
}));
|
||||
|
||||
const mockRun = OrchestratorSystem.Run as jest.MockedFunction<typeof OrchestratorSystem.Run>;
|
||||
const mockLog = OrchestratorLogger.log as jest.MockedFunction<typeof OrchestratorLogger.log>;
|
||||
|
||||
function createBuildParameters(overrides: Partial<BuildParameters> = {}): BuildParameters {
|
||||
return {
|
||||
githubActionsRepo: 'owner/repo',
|
||||
githubActionsWorkflow: 'build.yml',
|
||||
githubActionsToken: 'ghp_test_token_123',
|
||||
githubActionsRef: 'main',
|
||||
...overrides,
|
||||
} as BuildParameters;
|
||||
}
|
||||
|
||||
// Override setTimeout to execute callbacks immediately so polling loops complete fast
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
beforeAll(() => {
|
||||
global.setTimeout = ((fn: (...args: any[]) => void, _ms?: number, ...args: any[]) => {
|
||||
return originalSetTimeout(fn, 0, ...args);
|
||||
}) as any;
|
||||
});
|
||||
afterAll(() => {
|
||||
global.setTimeout = originalSetTimeout;
|
||||
});
|
||||
|
||||
describe('GitHubActionsProvider', () => {
|
||||
let provider: GitHubActionsProvider;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
provider = new GitHubActionsProvider(createBuildParameters());
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('sets default ref to main when not specified', () => {
|
||||
const params = createBuildParameters({ githubActionsRef: undefined });
|
||||
const p = new GitHubActionsProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
|
||||
it('uses provided ref when specified', () => {
|
||||
const params = createBuildParameters({ githubActionsRef: 'develop' });
|
||||
const p = new GitHubActionsProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('setupWorkflow', () => {
|
||||
it('verifies workflow exists via gh api and logs success', async () => {
|
||||
mockRun.mockResolvedValueOnce('12345\n');
|
||||
|
||||
await provider.setupWorkflow('guid-123', createBuildParameters(), 'main', []);
|
||||
|
||||
expect(mockRun).toHaveBeenCalledTimes(1);
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toContain('gh api repos/owner/repo/actions/workflows/build.yml');
|
||||
expect(command).toContain("--jq '.id'");
|
||||
expect(command).toContain('GH_TOKEN=ghp_test_token_123');
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Workflow verified'));
|
||||
});
|
||||
|
||||
it('throws when repo is not configured', async () => {
|
||||
const params = createBuildParameters({ githubActionsRepo: '' });
|
||||
provider = new GitHubActionsProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'githubActionsRepo and githubActionsWorkflow are required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when workflow is not configured', async () => {
|
||||
const params = createBuildParameters({ githubActionsWorkflow: '' });
|
||||
provider = new GitHubActionsProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'githubActionsRepo and githubActionsWorkflow are required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when token is missing', async () => {
|
||||
const params = createBuildParameters({ githubActionsToken: '' });
|
||||
provider = new GitHubActionsProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'githubActionsToken is required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws descriptive error when workflow verification fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('Not Found'));
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', createBuildParameters(), 'main', [])).rejects.toThrow(
|
||||
'Failed to verify workflow build.yml in owner/repo',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runTaskInWorkflow', () => {
|
||||
it('dispatches workflow with correct inputs and returns logs on success', async () => {
|
||||
// Dispatch succeeds
|
||||
mockRun.mockResolvedValueOnce('');
|
||||
// First poll finds the run
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 99001, status: 'in_progress' }));
|
||||
// Status poll returns completed
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'success' }));
|
||||
// Log fetch succeeds
|
||||
mockRun.mockResolvedValueOnce('Build output log content here');
|
||||
|
||||
const result = await provider.runTaskInWorkflow(
|
||||
'guid-abc',
|
||||
'unityci/editor:2021.3',
|
||||
'echo build',
|
||||
'/mount',
|
||||
'/work',
|
||||
[],
|
||||
[],
|
||||
);
|
||||
|
||||
expect(result).toBe('Build output log content here');
|
||||
|
||||
// Verify dispatch command
|
||||
const dispatchCommand = mockRun.mock.calls[0][0];
|
||||
expect(dispatchCommand).toContain('dispatches');
|
||||
expect(dispatchCommand).toContain('-X POST');
|
||||
expect(dispatchCommand).toContain("ref='main'");
|
||||
|
||||
// Verify log fetch command
|
||||
const logCommand = mockRun.mock.calls[3][0];
|
||||
expect(logCommand).toContain('gh run view');
|
||||
expect(logCommand).toContain('--log');
|
||||
expect(logCommand).toContain('--repo owner/repo');
|
||||
});
|
||||
|
||||
it('base64 encodes commands in the inputs payload', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 100, status: 'completed' })); // run found
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'success' })); // status
|
||||
mockRun.mockResolvedValueOnce('logs'); // logs
|
||||
|
||||
await provider.runTaskInWorkflow('guid-1', 'image:latest', 'echo hello && build', '/mnt', '/w', [], []);
|
||||
|
||||
const dispatchCommand = mockRun.mock.calls[0][0];
|
||||
const expectedB64 = Buffer.from('echo hello && build').toString('base64');
|
||||
expect(dispatchCommand).toContain(expectedB64);
|
||||
});
|
||||
|
||||
it('includes environment variables as JSON input', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 200, status: 'completed' })); // run found
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'success' })); // status
|
||||
mockRun.mockResolvedValueOnce('logs'); // logs
|
||||
|
||||
const env = [
|
||||
{ name: 'UNITY_LICENSE', value: 'license-data' },
|
||||
{ name: 'BUILD_TARGET', value: 'StandaloneWindows64' },
|
||||
];
|
||||
|
||||
await provider.runTaskInWorkflow('guid-2', 'img', 'cmd', '/m', '/w', env as any, []);
|
||||
|
||||
const dispatchCommand = mockRun.mock.calls[0][0];
|
||||
expect(dispatchCommand).toContain('UNITY_LICENSE');
|
||||
expect(dispatchCommand).toContain('BUILD_TARGET');
|
||||
});
|
||||
|
||||
it('throws when workflow dispatch fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('403 Forbidden'));
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-err', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Failed to dispatch workflow',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when workflow run does not start within timeout', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch succeeds
|
||||
|
||||
// All 30 poll attempts fail
|
||||
for (let i = 0; i < 30; i++) {
|
||||
mockRun.mockRejectedValueOnce(new Error('not found'));
|
||||
}
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-timeout', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Workflow run did not start within',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when workflow run fails with non-success conclusion', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 300, status: 'in_progress' })); // run appears
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'failure' })); // fails
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-fail', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Workflow run failed with conclusion: failure',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns fallback message when log fetch fails', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 400, status: 'completed' })); // run appears
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'success' })); // completes
|
||||
mockRun.mockRejectedValueOnce(new Error('logs unavailable')); // log fetch fails
|
||||
|
||||
const result = await provider.runTaskInWorkflow('guid-nologs', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
expect(result).toContain('completed successfully');
|
||||
expect(result).toContain('logs unavailable');
|
||||
});
|
||||
|
||||
it('handles cancelled workflow run conclusion', async () => {
|
||||
mockRun.mockResolvedValueOnce(''); // dispatch
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 500, status: 'in_progress' })); // run
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'completed', conclusion: 'cancelled' })); // cancelled
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-cancel', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Workflow run failed with conclusion: cancelled',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws timeout error when polling exceeds maximum duration', async () => {
|
||||
// Save real Date.now
|
||||
const realDateNow = Date.now;
|
||||
let callCount = 0;
|
||||
|
||||
// dispatch succeeds
|
||||
mockRun.mockResolvedValueOnce('');
|
||||
// run appears
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 600, status: 'in_progress' }));
|
||||
// Status always returns in_progress
|
||||
mockRun.mockImplementation(() => Promise.resolve(JSON.stringify({ status: 'in_progress' })));
|
||||
|
||||
// First call returns normal time, subsequent calls simulate 5 hours elapsed
|
||||
Date.now = () => {
|
||||
callCount++;
|
||||
if (callCount <= 2) return realDateNow.call(Date);
|
||||
return realDateNow.call(Date) + 14_400_001; // 4 hours + 1ms
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(provider.runTaskInWorkflow('guid-poll-timeout', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'did not complete within 4 hours',
|
||||
);
|
||||
|
||||
expect(core.error).toHaveBeenCalledWith(expect.stringContaining('did not complete within 4 hours'));
|
||||
} finally {
|
||||
Date.now = realDateNow;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupWorkflow', () => {
|
||||
it('completes without error and logs cleanup message', async () => {
|
||||
await provider.cleanupWorkflow(createBuildParameters(), 'main', []);
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Cleanup complete'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('garbageCollect', () => {
|
||||
it('returns empty string (no-op)', async () => {
|
||||
const result = await provider.garbageCollect('', false, 0, false, false);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('listResources', () => {
|
||||
it('returns runner names from the repository', async () => {
|
||||
mockRun.mockResolvedValueOnce('runner-1\nrunner-2\nrunner-3\n');
|
||||
|
||||
const resources = await provider.listResources();
|
||||
|
||||
expect(resources).toHaveLength(3);
|
||||
expect(resources[0].Name).toBe('runner-1');
|
||||
expect(resources[1].Name).toBe('runner-2');
|
||||
expect(resources[2].Name).toBe('runner-3');
|
||||
});
|
||||
|
||||
it('returns empty array when repo or token is missing', async () => {
|
||||
const params = createBuildParameters({ githubActionsRepo: '' });
|
||||
provider = new GitHubActionsProvider(params);
|
||||
|
||||
const resources = await provider.listResources();
|
||||
expect(resources).toEqual([]);
|
||||
expect(mockRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns empty array when API call fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('API error'));
|
||||
|
||||
const resources = await provider.listResources();
|
||||
expect(resources).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listWorkflow', () => {
|
||||
it('returns recent workflow run names', async () => {
|
||||
mockRun.mockResolvedValueOnce('Build Unity\nRun Tests\n');
|
||||
|
||||
const workflows = await provider.listWorkflow();
|
||||
|
||||
expect(workflows).toHaveLength(2);
|
||||
expect(workflows[0].Name).toBe('Build Unity');
|
||||
expect(workflows[1].Name).toBe('Run Tests');
|
||||
});
|
||||
|
||||
it('returns empty array when credentials missing', async () => {
|
||||
const params = createBuildParameters({ githubActionsToken: '' });
|
||||
provider = new GitHubActionsProvider(params);
|
||||
|
||||
const workflows = await provider.listWorkflow();
|
||||
expect(workflows).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('watchWorkflow', () => {
|
||||
it('returns message when no active run exists', async () => {
|
||||
const result = await provider.watchWorkflow();
|
||||
expect(result).toBe('No active run to watch');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,284 @@
|
||||
import * as core from '@actions/core';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
const MAX_POLLING_DURATION_MS = 14_400_000; // 4 hours
|
||||
|
||||
/**
|
||||
* GitHub Actions provider — triggers builds as workflow_dispatch events
|
||||
* on a target repository via the GitHub API.
|
||||
*
|
||||
* Use case: Distribute builds across orgs, use specialized runner pools,
|
||||
* or trigger builds in repos with Unity licenses.
|
||||
*/
|
||||
class GitHubActionsProvider implements ProviderInterface {
|
||||
private buildParameters: BuildParameters;
|
||||
private repo: string;
|
||||
private workflow: string;
|
||||
private token: string;
|
||||
private ref: string;
|
||||
private runId: number = 0;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.buildParameters = buildParameters;
|
||||
this.repo = buildParameters.githubActionsRepo || '';
|
||||
this.workflow = buildParameters.githubActionsWorkflow || '';
|
||||
this.token = buildParameters.githubActionsToken || '';
|
||||
this.ref = buildParameters.githubActionsRef || 'main';
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[GitHubActions] Setting up workflow dispatch to ${this.repo}`);
|
||||
|
||||
if (!this.repo || !this.workflow) {
|
||||
throw new Error('githubActionsRepo and githubActionsWorkflow are required for the github-actions provider');
|
||||
}
|
||||
|
||||
if (!this.token) {
|
||||
throw new Error('githubActionsToken is required (PAT with actions:write scope)');
|
||||
}
|
||||
|
||||
// Verify repository and workflow exist
|
||||
try {
|
||||
const result = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api repos/${this.repo}/actions/workflows/${this.workflow} --jq '.id'`,
|
||||
);
|
||||
OrchestratorLogger.log(`[GitHubActions] Workflow verified: ${this.workflow} (ID: ${result.trim()})`);
|
||||
} catch (error: any) {
|
||||
throw new Error(`Failed to verify workflow ${this.workflow} in ${this.repo}: ${error.message || error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(`[GitHubActions] Dispatching workflow ${this.workflow} on ${this.repo}@${this.ref}`);
|
||||
|
||||
// Build inputs payload
|
||||
const inputs: Record<string, string> = {
|
||||
buildGuid,
|
||||
image,
|
||||
commands: Buffer.from(commands).toString('base64'),
|
||||
mountdir,
|
||||
workingdir,
|
||||
};
|
||||
|
||||
// Add environment variables as a JSON input
|
||||
if (environment.length > 0) {
|
||||
inputs.environment = JSON.stringify(environment.map((element) => ({ name: element.name, value: element.value })));
|
||||
}
|
||||
|
||||
// Record the time before dispatch to identify the run
|
||||
const beforeDispatch = new Date().toISOString();
|
||||
|
||||
// Dispatch the workflow
|
||||
const inputsJson = JSON.stringify(inputs).replace(/'/g, "'\\''");
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api repos/${this.repo}/actions/workflows/${this.workflow}/dispatches -X POST -f ref='${this.ref}' -f "inputs=${inputsJson}"`,
|
||||
);
|
||||
OrchestratorLogger.log(`[GitHubActions] Workflow dispatched`);
|
||||
} catch (error: any) {
|
||||
throw new Error(`Failed to dispatch workflow: ${error.message || error}`);
|
||||
}
|
||||
|
||||
// Poll for the run to appear
|
||||
OrchestratorLogger.log(`[GitHubActions] Waiting for workflow run to start...`);
|
||||
let attempts = 0;
|
||||
const maxAttempts = 30;
|
||||
|
||||
while (attempts < maxAttempts) {
|
||||
attempts++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10_000));
|
||||
|
||||
try {
|
||||
const runsJson = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api "repos/${this.repo}/actions/workflows/${this.workflow}/runs?created=>${beforeDispatch}&per_page=5" --jq '.workflow_runs[0] | {id, status, conclusion}'`,
|
||||
true,
|
||||
);
|
||||
|
||||
const run = JSON.parse(runsJson.trim());
|
||||
if (run.id) {
|
||||
this.runId = run.id;
|
||||
OrchestratorLogger.log(`[GitHubActions] Run started: ${this.runId} (status: ${run.status})`);
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Run not yet available
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.runId) {
|
||||
throw new Error(`Workflow run did not start within ${maxAttempts * 10}s`);
|
||||
}
|
||||
|
||||
// Poll until completion and stream logs (with maximum duration guard)
|
||||
let status = 'in_progress';
|
||||
const pollingStartTime = Date.now();
|
||||
const runUrl = `https://github.com/${this.repo}/actions/runs/${this.runId}`;
|
||||
|
||||
while (status === 'in_progress' || status === 'queued') {
|
||||
const elapsedMs = Date.now() - pollingStartTime;
|
||||
if (elapsedMs >= MAX_POLLING_DURATION_MS) {
|
||||
const hours = Math.round(MAX_POLLING_DURATION_MS / 3_600_000);
|
||||
const message = `GitHub Actions workflow did not complete within ${hours} hours. Run URL: ${runUrl}`;
|
||||
core.error(message);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 15_000));
|
||||
|
||||
try {
|
||||
const statusJson = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api repos/${this.repo}/actions/runs/${this.runId} --jq '{status, conclusion}'`,
|
||||
true,
|
||||
);
|
||||
|
||||
const result = JSON.parse(statusJson.trim());
|
||||
status = result.status;
|
||||
|
||||
if (status === 'completed') {
|
||||
OrchestratorLogger.log(`[GitHubActions] Run ${this.runId} completed: ${result.conclusion}`);
|
||||
|
||||
if (result.conclusion !== 'success') {
|
||||
throw new Error(`Workflow run failed with conclusion: ${result.conclusion}`);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
OrchestratorLogger.log(`[GitHubActions] Run ${this.runId} status: ${status}`);
|
||||
} catch (error: any) {
|
||||
if (error.message && error.message.includes('conclusion')) {
|
||||
throw error;
|
||||
}
|
||||
if (error.message && error.message.includes('did not complete within')) {
|
||||
throw error;
|
||||
}
|
||||
OrchestratorLogger.logWarning(`[GitHubActions] Status check error: ${error.message || error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch logs
|
||||
try {
|
||||
const logs = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh run view ${this.runId} --repo ${this.repo} --log`,
|
||||
true,
|
||||
);
|
||||
|
||||
return logs;
|
||||
} catch {
|
||||
return `Run ${this.runId} completed successfully (logs unavailable)`;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[GitHubActions] Cleanup complete (no resources to tear down)`);
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return '';
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
if (!this.repo || !this.token) return [];
|
||||
|
||||
try {
|
||||
const runnersJson = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api repos/${this.repo}/actions/runners --jq '.runners[] | .name'`,
|
||||
true,
|
||||
);
|
||||
|
||||
return runnersJson
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((name) => {
|
||||
const resource = new ProviderResource();
|
||||
resource.Name = name.trim();
|
||||
|
||||
return resource;
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
if (!this.repo || !this.token) return [];
|
||||
|
||||
try {
|
||||
const runsJson = await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh api repos/${this.repo}/actions/runs?per_page=10 --jq '.workflow_runs[] | .name'`,
|
||||
true,
|
||||
);
|
||||
|
||||
return runsJson
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((name) => {
|
||||
const workflow = new ProviderWorkflow();
|
||||
workflow.Name = name.trim();
|
||||
|
||||
return workflow;
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<string> {
|
||||
if (!this.runId) return 'No active run to watch';
|
||||
|
||||
try {
|
||||
return await OrchestratorSystem.Run(
|
||||
`GH_TOKEN=${this.token} gh run watch ${this.runId} --repo ${this.repo}`,
|
||||
true,
|
||||
);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
}
|
||||
export default GitHubActionsProvider;
|
||||
@@ -0,0 +1,329 @@
|
||||
import GitLabCIProvider from '.';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
jest.mock('../../services/core/orchestrator-system');
|
||||
jest.mock('../../services/core/orchestrator-logger');
|
||||
jest.mock('@actions/core', () => ({
|
||||
info: jest.fn(),
|
||||
warning: jest.fn(),
|
||||
error: jest.fn(),
|
||||
setOutput: jest.fn(),
|
||||
getInput: jest.fn(() => ''),
|
||||
}));
|
||||
|
||||
const mockRun = OrchestratorSystem.Run as jest.MockedFunction<typeof OrchestratorSystem.Run>;
|
||||
const mockLog = OrchestratorLogger.log as jest.MockedFunction<typeof OrchestratorLogger.log>;
|
||||
const mockLogWarning = OrchestratorLogger.logWarning as jest.MockedFunction<typeof OrchestratorLogger.logWarning>;
|
||||
|
||||
function createBuildParameters(overrides: Partial<BuildParameters> = {}): BuildParameters {
|
||||
return {
|
||||
gitlabProjectId: 'my-group/my-project',
|
||||
gitlabTriggerToken: 'glptt-test-token-456',
|
||||
gitlabApiUrl: 'https://gitlab.example.com',
|
||||
gitlabRef: 'main',
|
||||
...overrides,
|
||||
} as BuildParameters;
|
||||
}
|
||||
|
||||
// Override setTimeout to execute callbacks immediately so polling loops complete fast
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
beforeAll(() => {
|
||||
global.setTimeout = ((fn: (...args: any[]) => void, _ms?: number, ...args: any[]) => {
|
||||
return originalSetTimeout(fn, 0, ...args);
|
||||
}) as any;
|
||||
});
|
||||
afterAll(() => {
|
||||
global.setTimeout = originalSetTimeout;
|
||||
});
|
||||
|
||||
describe('GitLabCIProvider', () => {
|
||||
let provider: GitLabCIProvider;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
provider = new GitLabCIProvider(createBuildParameters());
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('strips trailing slashes from apiUrl', () => {
|
||||
const params = createBuildParameters({ gitlabApiUrl: 'https://gitlab.example.com///' });
|
||||
const p = new GitLabCIProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
|
||||
it('defaults apiUrl to https://gitlab.com when not provided', () => {
|
||||
const params = createBuildParameters({ gitlabApiUrl: undefined });
|
||||
const p = new GitLabCIProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
|
||||
it('defaults ref to main when not provided', () => {
|
||||
const params = createBuildParameters({ gitlabRef: undefined });
|
||||
const p = new GitLabCIProvider(params);
|
||||
expect(p).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('setupWorkflow', () => {
|
||||
it('verifies project access via curl and logs success', async () => {
|
||||
mockRun.mockResolvedValueOnce('');
|
||||
|
||||
await provider.setupWorkflow('guid-123', createBuildParameters(), 'main', []);
|
||||
|
||||
expect(mockRun).toHaveBeenCalledTimes(1);
|
||||
const command = mockRun.mock.calls[0][0];
|
||||
expect(command).toContain('curl -sf');
|
||||
expect(command).toContain('PRIVATE-TOKEN: glptt-test-token-456');
|
||||
expect(command).toContain('gitlab.example.com/api/v4/projects/');
|
||||
expect(command).toContain(encodeURIComponent('my-group/my-project'));
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Project access verified'));
|
||||
});
|
||||
|
||||
it('throws when projectId is not configured', async () => {
|
||||
const params = createBuildParameters({ gitlabProjectId: '' });
|
||||
provider = new GitLabCIProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'gitlabProjectId and gitlabTriggerToken are required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when triggerToken is not configured', async () => {
|
||||
const params = createBuildParameters({ gitlabTriggerToken: '' });
|
||||
provider = new GitLabCIProvider(params);
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', params, 'main', [])).rejects.toThrow(
|
||||
'gitlabProjectId and gitlabTriggerToken are required',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws descriptive error when project access check fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('401 Unauthorized'));
|
||||
|
||||
await expect(provider.setupWorkflow('guid-123', createBuildParameters(), 'main', [])).rejects.toThrow(
|
||||
'Failed to access GitLab project my-group/my-project',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runTaskInWorkflow', () => {
|
||||
it('triggers pipeline and returns job logs on success', async () => {
|
||||
// Pipeline trigger response
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5001, status: 'pending' }));
|
||||
// Status poll returns success
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'success' }));
|
||||
// Jobs list
|
||||
mockRun.mockResolvedValueOnce(
|
||||
JSON.stringify([
|
||||
{ id: 10001, name: 'build-unity', status: 'success' },
|
||||
{ id: 10002, name: 'test-unity', status: 'success' },
|
||||
]),
|
||||
);
|
||||
// Job traces
|
||||
mockRun.mockResolvedValueOnce('Building Unity project...\nDone.');
|
||||
mockRun.mockResolvedValueOnce('Running tests...\nAll passed.');
|
||||
|
||||
const result = await provider.runTaskInWorkflow(
|
||||
'guid-gl1',
|
||||
'unityci/editor:2021.3',
|
||||
'echo build',
|
||||
'/mount',
|
||||
'/work',
|
||||
[],
|
||||
[],
|
||||
);
|
||||
|
||||
expect(result).toContain('build-unity');
|
||||
expect(result).toContain('test-unity');
|
||||
expect(result).toContain('Building Unity project');
|
||||
expect(result).toContain('Running tests');
|
||||
|
||||
// Verify trigger command
|
||||
const triggerCommand = mockRun.mock.calls[0][0];
|
||||
expect(triggerCommand).toContain('trigger/pipeline');
|
||||
expect(triggerCommand).toContain(`token=${createBuildParameters().gitlabTriggerToken}`);
|
||||
expect(triggerCommand).toContain('ref=main');
|
||||
});
|
||||
|
||||
it('passes build variables including base64-encoded commands', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5002, status: 'success' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'success' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify([]));
|
||||
|
||||
await provider.runTaskInWorkflow(
|
||||
'guid-vars',
|
||||
'ubuntu:20.04',
|
||||
'make build',
|
||||
'/mnt/data',
|
||||
'/workspace',
|
||||
[{ name: 'UNITY_VERSION', value: '2021.3.1f1' } as any],
|
||||
[],
|
||||
);
|
||||
|
||||
const triggerCommand = mockRun.mock.calls[0][0];
|
||||
const expectedB64 = Buffer.from('make build').toString('base64');
|
||||
expect(triggerCommand).toContain(`variables[BUILD_COMMANDS]=${expectedB64}`);
|
||||
expect(triggerCommand).toContain('variables[BUILD_GUID]=guid-vars');
|
||||
expect(triggerCommand).toContain('variables[BUILD_IMAGE]=ubuntu:20.04');
|
||||
expect(triggerCommand).toContain('variables[MOUNT_DIR]=/mnt/data');
|
||||
expect(triggerCommand).toContain('variables[WORKING_DIR]=/workspace');
|
||||
expect(triggerCommand).toContain('variables[UNITY_VERSION]=2021.3.1f1');
|
||||
});
|
||||
|
||||
it('throws when pipeline trigger fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('404 Not Found'));
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-err', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Failed to trigger pipeline',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when pipeline finishes with failure status', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5003, status: 'pending' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'failed' }));
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-fail', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Pipeline 5003 finished with status: failed',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when pipeline is canceled', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5004, status: 'pending' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'canceled' }));
|
||||
|
||||
await expect(provider.runTaskInWorkflow('guid-cancel', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'Pipeline 5004 finished with status: canceled',
|
||||
);
|
||||
});
|
||||
|
||||
it('handles job log fetch failures gracefully', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5005, status: 'success' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'success' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify([{ id: 20001, name: 'build', status: 'success' }]));
|
||||
// Job trace fetch fails
|
||||
mockRun.mockRejectedValueOnce(new Error('trace unavailable'));
|
||||
|
||||
const result = await provider.runTaskInWorkflow('guid-nologs', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
expect(result).toContain('build');
|
||||
expect(result).toContain('logs unavailable');
|
||||
});
|
||||
|
||||
it('returns fallback message when entire job fetch fails', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5006, status: 'success' }));
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'success' }));
|
||||
// Jobs list fails
|
||||
mockRun.mockRejectedValueOnce(new Error('API error'));
|
||||
|
||||
const result = await provider.runTaskInWorkflow('guid-noapi', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
expect(result).toContain('Pipeline 5006 completed successfully');
|
||||
expect(result).toContain('logs unavailable');
|
||||
});
|
||||
|
||||
it('continues polling through status check errors until completion', async () => {
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5007, status: 'pending' }));
|
||||
// First status check fails
|
||||
mockRun.mockRejectedValueOnce(new Error('network blip'));
|
||||
// Second status check succeeds
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ status: 'success' }));
|
||||
// Jobs/logs
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify([]));
|
||||
|
||||
await provider.runTaskInWorkflow('guid-retry', 'img', 'cmd', '/m', '/w', [], []);
|
||||
|
||||
expect(mockLogWarning).toHaveBeenCalledWith(expect.stringContaining('Status check error'));
|
||||
});
|
||||
|
||||
it('throws timeout error when polling exceeds maximum duration', async () => {
|
||||
const realDateNow = Date.now;
|
||||
let callCount = 0;
|
||||
|
||||
// Trigger pipeline succeeds
|
||||
mockRun.mockResolvedValueOnce(JSON.stringify({ id: 5008, status: 'running' }));
|
||||
// Status always returns running
|
||||
mockRun.mockImplementation(() => Promise.resolve(JSON.stringify({ status: 'running' })));
|
||||
|
||||
// After first call, simulate 5 hours elapsed
|
||||
Date.now = () => {
|
||||
callCount++;
|
||||
if (callCount <= 1) return realDateNow.call(Date);
|
||||
return realDateNow.call(Date) + 14_400_001; // 4 hours + 1ms
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(provider.runTaskInWorkflow('guid-poll-timeout', 'img', 'cmd', '/m', '/w', [], [])).rejects.toThrow(
|
||||
'did not complete within 4 hours',
|
||||
);
|
||||
|
||||
expect(core.error).toHaveBeenCalledWith(expect.stringContaining('did not complete within 4 hours'));
|
||||
} finally {
|
||||
Date.now = realDateNow;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupWorkflow', () => {
|
||||
it('completes without error and logs cleanup message', async () => {
|
||||
await provider.cleanupWorkflow(createBuildParameters(), 'main', []);
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Cleanup complete'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('garbageCollect', () => {
|
||||
it('returns empty string (no-op)', async () => {
|
||||
const result = await provider.garbageCollect('', false, 0, false, false);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('listResources', () => {
|
||||
it('returns empty array (not implemented)', async () => {
|
||||
const resources = await provider.listResources();
|
||||
expect(resources).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listWorkflow', () => {
|
||||
it('returns recent pipeline names when credentials are available', async () => {
|
||||
mockRun.mockResolvedValueOnce(
|
||||
JSON.stringify([
|
||||
{ id: 100, status: 'success' },
|
||||
{ id: 101, status: 'failed' },
|
||||
]),
|
||||
);
|
||||
|
||||
const workflows = await provider.listWorkflow();
|
||||
|
||||
expect(workflows).toHaveLength(2);
|
||||
expect(workflows[0].Name).toBe('Pipeline #100 (success)');
|
||||
expect(workflows[1].Name).toBe('Pipeline #101 (failed)');
|
||||
});
|
||||
|
||||
it('returns empty array when credentials are missing', async () => {
|
||||
const params = createBuildParameters({ gitlabProjectId: '' });
|
||||
provider = new GitLabCIProvider(params);
|
||||
|
||||
const workflows = await provider.listWorkflow();
|
||||
expect(workflows).toEqual([]);
|
||||
expect(mockRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns empty array when API call fails', async () => {
|
||||
mockRun.mockRejectedValueOnce(new Error('API error'));
|
||||
|
||||
const workflows = await provider.listWorkflow();
|
||||
expect(workflows).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('watchWorkflow', () => {
|
||||
it('returns empty string (not implemented)', async () => {
|
||||
const result = await provider.watchWorkflow();
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import * as core from '@actions/core';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
const MAX_POLLING_DURATION_MS = 14_400_000; // 4 hours
|
||||
|
||||
/**
|
||||
* GitLab CI provider — triggers builds as GitLab CI pipelines
|
||||
* via the GitLab API.
|
||||
*
|
||||
* Use case: Teams using GitLab CI, hybrid GitHub/GitLab setups,
|
||||
* or GitLab runners with Unity licenses.
|
||||
*/
|
||||
class GitLabCIProvider implements ProviderInterface {
|
||||
private buildParameters: BuildParameters;
|
||||
private projectId: string;
|
||||
private triggerToken: string;
|
||||
private apiUrl: string;
|
||||
private ref: string;
|
||||
private pipelineId: number = 0;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.buildParameters = buildParameters;
|
||||
this.projectId = buildParameters.gitlabProjectId || '';
|
||||
this.triggerToken = buildParameters.gitlabTriggerToken || '';
|
||||
this.apiUrl = (buildParameters.gitlabApiUrl || 'https://gitlab.com').replace(/\/+$/, '');
|
||||
this.ref = buildParameters.gitlabRef || 'main';
|
||||
}
|
||||
|
||||
async setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[GitLabCI] Setting up pipeline trigger for project ${this.projectId}`);
|
||||
|
||||
if (!this.projectId || !this.triggerToken) {
|
||||
throw new Error('gitlabProjectId and gitlabTriggerToken are required for the gitlab-ci provider');
|
||||
}
|
||||
|
||||
// Verify project access
|
||||
const encodedProject = encodeURIComponent(this.projectId);
|
||||
try {
|
||||
await OrchestratorSystem.Run(
|
||||
`curl -sf -H "PRIVATE-TOKEN: ${this.triggerToken}" "${this.apiUrl}/api/v4/projects/${encodedProject}" -o /dev/null`,
|
||||
);
|
||||
OrchestratorLogger.log(`[GitLabCI] Project access verified`);
|
||||
} catch (error: any) {
|
||||
throw new Error(`Failed to access GitLab project ${this.projectId}: ${error.message || error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: OrchestratorEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: OrchestratorSecret[],
|
||||
): Promise<string> {
|
||||
OrchestratorLogger.log(`[GitLabCI] Triggering pipeline on project ${this.projectId}@${this.ref}`);
|
||||
|
||||
const encodedProject = encodeURIComponent(this.projectId);
|
||||
|
||||
// Build variables for the pipeline
|
||||
const pipelineVariables: string[] = [
|
||||
`-f "variables[BUILD_GUID]=${buildGuid}"`,
|
||||
`-f "variables[BUILD_IMAGE]=${image}"`,
|
||||
`-f "variables[BUILD_COMMANDS]=${Buffer.from(commands).toString('base64')}"`,
|
||||
`-f "variables[MOUNT_DIR]=${mountdir}"`,
|
||||
`-f "variables[WORKING_DIR]=${workingdir}"`,
|
||||
];
|
||||
|
||||
for (const element of environment) {
|
||||
pipelineVariables.push(`-f "variables[${element.name}]=${element.value}"`);
|
||||
}
|
||||
|
||||
// Trigger pipeline
|
||||
try {
|
||||
const response = await OrchestratorSystem.Run(
|
||||
`curl -sf -X POST "${this.apiUrl}/api/v4/projects/${encodedProject}/trigger/pipeline" -f "token=${
|
||||
this.triggerToken
|
||||
}" -f "ref=${this.ref}" ${pipelineVariables.join(' ')}`,
|
||||
);
|
||||
|
||||
const pipeline = JSON.parse(response);
|
||||
this.pipelineId = pipeline.id;
|
||||
OrchestratorLogger.log(`[GitLabCI] Pipeline triggered: ${this.pipelineId} (status: ${pipeline.status})`);
|
||||
} catch (error: any) {
|
||||
throw new Error(`Failed to trigger pipeline: ${error.message || error}`);
|
||||
}
|
||||
|
||||
// Poll until completion (with maximum duration guard)
|
||||
let status = 'pending';
|
||||
const terminalStatuses = new Set(['success', 'failed', 'canceled', 'skipped']);
|
||||
const pollingStartTime = Date.now();
|
||||
const pipelineUrl = `${this.apiUrl}/${this.projectId}/-/pipelines/${this.pipelineId}`;
|
||||
|
||||
while (!terminalStatuses.has(status)) {
|
||||
const elapsedMs = Date.now() - pollingStartTime;
|
||||
if (elapsedMs >= MAX_POLLING_DURATION_MS) {
|
||||
const hours = Math.round(MAX_POLLING_DURATION_MS / 3_600_000);
|
||||
const message = `GitLab CI pipeline did not complete within ${hours} hours. Pipeline URL: ${pipelineUrl}`;
|
||||
core.error(message);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 15_000));
|
||||
|
||||
try {
|
||||
const statusResponse = await OrchestratorSystem.Run(
|
||||
`curl -sf -H "PRIVATE-TOKEN: ${this.triggerToken}" "${this.apiUrl}/api/v4/projects/${encodedProject}/pipelines/${this.pipelineId}"`,
|
||||
true,
|
||||
);
|
||||
|
||||
const pipelineStatus = JSON.parse(statusResponse);
|
||||
status = pipelineStatus.status;
|
||||
OrchestratorLogger.log(`[GitLabCI] Pipeline ${this.pipelineId} status: ${status}`);
|
||||
} catch (error: any) {
|
||||
OrchestratorLogger.logWarning(`[GitLabCI] Status check error: ${error.message || error}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (status !== 'success') {
|
||||
throw new Error(`Pipeline ${this.pipelineId} finished with status: ${status}`);
|
||||
}
|
||||
|
||||
// Fetch job logs
|
||||
try {
|
||||
const jobsResponse = await OrchestratorSystem.Run(
|
||||
`curl -sf -H "PRIVATE-TOKEN: ${this.triggerToken}" "${this.apiUrl}/api/v4/projects/${encodedProject}/pipelines/${this.pipelineId}/jobs"`,
|
||||
true,
|
||||
);
|
||||
|
||||
const jobs = JSON.parse(jobsResponse);
|
||||
const logs: string[] = [];
|
||||
|
||||
for (const job of jobs) {
|
||||
try {
|
||||
const jobLog = await OrchestratorSystem.Run(
|
||||
`curl -sf -H "PRIVATE-TOKEN: ${this.triggerToken}" "${this.apiUrl}/api/v4/projects/${encodedProject}/jobs/${job.id}/trace"`,
|
||||
true,
|
||||
);
|
||||
logs.push(`=== Job: ${job.name} (${job.status}) ===\n${jobLog}`);
|
||||
} catch {
|
||||
logs.push(`=== Job: ${job.name} (${job.status}) === (logs unavailable)`);
|
||||
}
|
||||
}
|
||||
|
||||
return logs.join('\n\n');
|
||||
} catch {
|
||||
return `Pipeline ${this.pipelineId} completed successfully (logs unavailable)`;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
): Promise<void> {
|
||||
OrchestratorLogger.log(`[GitLabCI] Cleanup complete`);
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return '';
|
||||
}
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
if (!this.projectId || !this.triggerToken) return [];
|
||||
|
||||
try {
|
||||
const encodedProject = encodeURIComponent(this.projectId);
|
||||
const response = await OrchestratorSystem.Run(
|
||||
`curl -sf -H "PRIVATE-TOKEN: ${this.triggerToken}" "${this.apiUrl}/api/v4/projects/${encodedProject}/pipelines?per_page=10"`,
|
||||
true,
|
||||
);
|
||||
|
||||
return JSON.parse(response).map((pipeline: any) => {
|
||||
const workflow = new ProviderWorkflow();
|
||||
workflow.Name = `Pipeline #${pipeline.id} (${pipeline.status})`;
|
||||
|
||||
return workflow;
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async watchWorkflow(): Promise<string> {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
export default GitLabCIProvider;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user