mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-07 22:43:55 -07:00
Compare commits
110 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cb6b30300e | |||
| e6686e4d61 | |||
| 9b84c58b3b | |||
| f38f6848c8 | |||
| 825f116f84 | |||
| ac1c6d16db | |||
| 1815f1a414 | |||
| 8aa16937eb | |||
| 44bbd8c657 | |||
| 5914a0fd20 | |||
| a676329053 | |||
| 1423ed868e | |||
| b50c5c04ce | |||
| 79bd967fb5 | |||
| be0139ec6d | |||
| d6cc45383d | |||
| bd1be2e474 | |||
| 98963da430 | |||
| fd74d25ac9 | |||
| a0cb4ff559 | |||
| edc1df78b3 | |||
| 7779839e46 | |||
| 85bb3d9d50 | |||
| 307a2aa562 | |||
| df650638a8 | |||
| 831b913577 | |||
| f4d46125f8 | |||
| 1d2d9044df | |||
| 5d667ab72b | |||
| 73de3d49a9 | |||
| 94daf5affe | |||
| ee01652e7e | |||
| 3f8fbb9693 | |||
| 431a471303 | |||
| f50fd8ebb2 | |||
| 364f9a79f7 | |||
| c2a7091efa | |||
| 43c11e7f14 | |||
| d58c3d6d5f | |||
| d800b1044c | |||
| 4e3546c9bd | |||
| ce848c7a6d | |||
| 8f66ff2893 | |||
| d3e23a8c70 | |||
| 0876bd4321 | |||
| c62465ad70 | |||
| 32265f47aa | |||
| dda7de4882 | |||
| 71895ac520 | |||
| f6f813b5e1 | |||
| 26fcfceaa8 | |||
| f7df350964 | |||
| af988e6d2a | |||
| f7725a72d6 | |||
| c5f2078fcb | |||
| b8c3ad1227 | |||
| c28831ce79 | |||
| 3570d40148 | |||
| 2d7374bec4 | |||
| 9e6d69f9f5 | |||
| 16d1156834 | |||
| 91872a2361 | |||
| f06dd86acf | |||
| c676d1dc4d | |||
| a04f7d8eef | |||
| 4c3d97dcdb | |||
| 82060437f1 | |||
| 277dcabde2 | |||
| 1e2fa056a8 | |||
| 3de8cac128 | |||
| 4f5155d536 | |||
| d8ad8f9a5a | |||
| 0c57572a1c | |||
| f00d7c8add | |||
| 70fcc1ae2f | |||
| 9b205ac903 | |||
| afdc987ae3 | |||
| 52b79b2a94 | |||
| e9af7641b7 | |||
| bad80a45d9 | |||
| 1e57879d8d | |||
| 5d0450de7b | |||
| 12b6aaae61 | |||
| 016692526b | |||
| 4b178e0114 | |||
| 6c4a85a2a0 | |||
| a4a3612fcf | |||
| 962603b7b3 | |||
| 8acf3ccca3 | |||
| ec93ad51d9 | |||
| c3e0ee6d1a | |||
| f2dbcdf433 | |||
| c8f881a385 | |||
| eb8b92cda1 | |||
| 0650d1de5c | |||
| e9a60d4ec8 | |||
| 6e13713bb2 | |||
| fa6440db27 | |||
| 5b34e4df94 | |||
| 12e5985cf8 | |||
| a0833df59e | |||
| 92eaa73a2d | |||
| b662a6fa0e | |||
| 9ed94b241f | |||
| 36503e30c0 | |||
| 01bbef7a89 | |||
| 1815c3c880 | |||
| 10fc07a79b | |||
| db9fc17071 | |||
| a1f3d9ecd4 |
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"files.autoSave": "on",
|
||||||
|
"files.autoSaveWhen": "on",
|
||||||
|
"files.autoSaveDelay": 1000,
|
||||||
|
|
||||||
|
"editor.formatOnSave": false,
|
||||||
|
"editor.formatOnPaste": false,
|
||||||
|
"editor.formatOnType": false,
|
||||||
|
|
||||||
|
"editor.codeActionsOnSave": {},
|
||||||
|
|
||||||
|
"git.autorefresh": false,
|
||||||
|
"git.confirmSync": false,
|
||||||
|
"git.autofetch": false,
|
||||||
|
|
||||||
|
"editor.defaultFormatter": null
|
||||||
|
}
|
||||||
|
|
||||||
+1
-2
@@ -14,8 +14,7 @@
|
|||||||
"env": {
|
"env": {
|
||||||
"node": true,
|
"node": true,
|
||||||
"es6": true,
|
"es6": true,
|
||||||
"jest/globals": true,
|
"jest/globals": true
|
||||||
"es2020": true
|
|
||||||
},
|
},
|
||||||
"rules": {
|
"rules": {
|
||||||
// Error out for code formatting errors
|
// Error out for code formatting errors
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ jobs:
|
|||||||
buildForAllPlatformsMacOS:
|
buildForAllPlatformsMacOS:
|
||||||
name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
|
name: ${{ matrix.targetPlatform }} on ${{ matrix.unityVersion }}
|
||||||
runs-on: macos-latest
|
runs-on: macos-latest
|
||||||
continue-on-error: true
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -25,13 +24,6 @@ jobs:
|
|||||||
targetPlatform:
|
targetPlatform:
|
||||||
- StandaloneOSX # Build a MacOS executable
|
- StandaloneOSX # Build a MacOS executable
|
||||||
- iOS # Build an iOS executable
|
- iOS # Build an iOS executable
|
||||||
include:
|
|
||||||
# Additionally test enableGpu build for a standalone windows target
|
|
||||||
- unityVersion: 6000.0.36f1
|
|
||||||
targetPlatform: StandaloneOSX
|
|
||||||
- unityVersion: 6000.0.36f1
|
|
||||||
targetPlatform: StandaloneOSX
|
|
||||||
buildProfile: 'Assets/Settings/Build Profiles/Sample macOS Build Profile.asset'
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
###########################
|
###########################
|
||||||
@@ -73,7 +65,6 @@ jobs:
|
|||||||
projectPath: ${{ matrix.projectPath }}
|
projectPath: ${{ matrix.projectPath }}
|
||||||
unityVersion: ${{ matrix.unityVersion }}
|
unityVersion: ${{ matrix.unityVersion }}
|
||||||
targetPlatform: ${{ matrix.targetPlatform }}
|
targetPlatform: ${{ matrix.targetPlatform }}
|
||||||
buildProfile: ${{ matrix.buildProfile }}
|
|
||||||
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
||||||
# We use dirty build because we are replacing the default project settings file above
|
# We use dirty build because we are replacing the default project settings file above
|
||||||
allowDirtyBuild: true
|
allowDirtyBuild: true
|
||||||
@@ -83,6 +74,6 @@ jobs:
|
|||||||
###########################
|
###########################
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})
|
||||||
path: build
|
path: build
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|||||||
@@ -34,12 +34,7 @@ jobs:
|
|||||||
unityVersion: 2023.2.2f1
|
unityVersion: 2023.2.2f1
|
||||||
targetPlatform: StandaloneWindows64
|
targetPlatform: StandaloneWindows64
|
||||||
enableGpu: true
|
enableGpu: true
|
||||||
- unityVersion: 6000.0.36f1
|
|
||||||
targetPlatform: StandaloneWindows64
|
|
||||||
- unityVersion: 6000.0.36f1
|
|
||||||
targetPlatform: StandaloneWindows64
|
|
||||||
buildProfile: 'Assets/Settings/Build Profiles/Sample Windows Build Profile.asset'
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
###########################
|
###########################
|
||||||
# Checkout #
|
# Checkout #
|
||||||
@@ -84,7 +79,6 @@ jobs:
|
|||||||
projectPath: ${{ matrix.projectPath }}
|
projectPath: ${{ matrix.projectPath }}
|
||||||
unityVersion: ${{ matrix.unityVersion }}
|
unityVersion: ${{ matrix.unityVersion }}
|
||||||
targetPlatform: ${{ matrix.targetPlatform }}
|
targetPlatform: ${{ matrix.targetPlatform }}
|
||||||
buildProfile: ${{ matrix.buildProfile }}
|
|
||||||
enableGpu: ${{ matrix.enableGpu }}
|
enableGpu: ${{ matrix.enableGpu }}
|
||||||
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
|
||||||
allowDirtyBuild: true
|
allowDirtyBuild: true
|
||||||
@@ -146,6 +140,6 @@ jobs:
|
|||||||
###########################
|
###########################
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}
|
||||||
path: build
|
path: build
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
name: Cleanup (cron)
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: '30 10 * * SUN' # every sunday at 10:30
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deleteArtifacts:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Delete old artifacts
|
||||||
|
uses: kolpav/purge-artifacts-action@v1
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
expire-in: 21 days
|
||||||
|
cleanupCloudRunner:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
if: github.event.event_type != 'pull_request_target'
|
||||||
|
with:
|
||||||
|
lfs: true
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: '18'
|
||||||
|
- run: yarn
|
||||||
|
- run: yarn run cli --help
|
||||||
|
env:
|
||||||
|
AWS_REGION: eu-west-2
|
||||||
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||||
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||||
|
AWS_DEFAULT_REGION: eu-west-2
|
||||||
|
- run: yarn run cli -m list-resources
|
||||||
|
env:
|
||||||
|
AWS_REGION: eu-west-2
|
||||||
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||||
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||||
|
AWS_DEFAULT_REGION: eu-west-2
|
||||||
+13
-15
@@ -18,16 +18,15 @@ env:
|
|||||||
GKE_CLUSTER: 'game-ci-github-pipelines'
|
GKE_CLUSTER: 'game-ci-github-pipelines'
|
||||||
GCP_LOGGING: true
|
GCP_LOGGING: true
|
||||||
GCP_PROJECT: unitykubernetesbuilder
|
GCP_PROJECT: unitykubernetesbuilder
|
||||||
GCP_LOG_FILE: ${{ github.workspace }}/orchestrator-logs.txt
|
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
|
||||||
# Commented out: Using LocalStack tests instead of real AWS
|
AWS_REGION: eu-west-2
|
||||||
# AWS_REGION: eu-west-2
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||||
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||||
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
AWS_DEFAULT_REGION: eu-west-2
|
||||||
# AWS_DEFAULT_REGION: eu-west-2
|
AWS_STACK_NAME: game-ci-github-pipelines
|
||||||
# AWS_STACK_NAME: game-ci-github-pipelines
|
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||||
ORCHESTRATOR_BRANCH: ${{ github.ref }}
|
CLOUD_RUNNER_DEBUG: true
|
||||||
ORCHESTRATOR_DEBUG: true
|
CLOUD_RUNNER_DEBUG_TREE: true
|
||||||
ORCHESTRATOR_DEBUG_TREE: true
|
|
||||||
DEBUG: true
|
DEBUG: true
|
||||||
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
||||||
PROJECT_PATH: test-project
|
PROJECT_PATH: test-project
|
||||||
@@ -47,14 +46,13 @@ jobs:
|
|||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
TARGET_PLATFORM: StandaloneWindows64
|
TARGET_PLATFORM: StandaloneWindows64
|
||||||
orchestratorTests: true
|
cloudRunnerTests: true
|
||||||
versioning: None
|
versioning: None
|
||||||
ORCHESTRATOR_CLUSTER: local-docker
|
CLOUD_RUNNER_CLUSTER: local-docker
|
||||||
# Commented out: Using LocalStack tests instead of real AWS
|
AWS_STACK_NAME: game-ci-github-pipelines
|
||||||
# AWS_STACK_NAME: game-ci-github-pipelines
|
|
||||||
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
|
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
|
||||||
run: |
|
run: |
|
||||||
git clone -b main https://github.com/game-ci/unity-builder
|
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
|
||||||
cd unity-builder
|
cd unity-builder
|
||||||
yarn
|
yarn
|
||||||
ls
|
ls
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
name: cloud-runner-integrity-localstack
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
runGithubIntegrationTests:
|
||||||
|
description: 'Run GitHub Checks integration tests'
|
||||||
|
required: false
|
||||||
|
default: 'false'
|
||||||
|
type: string
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
checks: write
|
||||||
|
contents: read
|
||||||
|
actions: write
|
||||||
|
packages: read
|
||||||
|
pull-requests: write
|
||||||
|
statuses: write
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
env:
|
||||||
|
AWS_REGION: us-east-1
|
||||||
|
AWS_DEFAULT_REGION: us-east-1
|
||||||
|
AWS_STACK_NAME: game-ci-local
|
||||||
|
AWS_ENDPOINT: http://localhost:4566
|
||||||
|
AWS_ENDPOINT_URL: http://localhost:4566
|
||||||
|
AWS_ACCESS_KEY_ID: test
|
||||||
|
AWS_SECRET_ACCESS_KEY: test
|
||||||
|
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||||
|
DEBUG: true
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
USE_IL2CPP: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
tests:
|
||||||
|
name: Cloud Runner Tests (LocalStack)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
services:
|
||||||
|
localstack:
|
||||||
|
image: localstack/localstack
|
||||||
|
ports:
|
||||||
|
- 4566:4566
|
||||||
|
env:
|
||||||
|
SERVICES: cloudformation,ecs,kinesis,cloudwatch,s3,logs
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
test:
|
||||||
|
- 'cloud-runner-end2end-locking'
|
||||||
|
- 'cloud-runner-end2end-caching'
|
||||||
|
- 'cloud-runner-end2end-retaining'
|
||||||
|
- 'cloud-runner-caching'
|
||||||
|
- 'cloud-runner-environment'
|
||||||
|
- 'cloud-runner-image'
|
||||||
|
- 'cloud-runner-hooks'
|
||||||
|
- 'cloud-runner-local-persistence'
|
||||||
|
- 'cloud-runner-locking-core'
|
||||||
|
- 'cloud-runner-locking-get-locked'
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
lfs: false
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: 'yarn'
|
||||||
|
- run: yarn install --frozen-lockfile
|
||||||
|
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||||
|
timeout-minutes: 60
|
||||||
|
env:
|
||||||
|
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||||
|
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||||
|
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
TARGET_PLATFORM: StandaloneWindows64
|
||||||
|
cloudRunnerTests: true
|
||||||
|
versioning: None
|
||||||
|
KUBE_STORAGE_CLASS: local-path
|
||||||
|
PROVIDER_STRATEGY: aws
|
||||||
|
AWS_ACCESS_KEY_ID: test
|
||||||
|
AWS_SECRET_ACCESS_KEY: test
|
||||||
|
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
name: cloud-runner-integrity
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
runGithubIntegrationTests:
|
||||||
|
description: 'Run GitHub Checks integration tests'
|
||||||
|
required: false
|
||||||
|
default: 'false'
|
||||||
|
type: string
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
checks: write
|
||||||
|
contents: read
|
||||||
|
actions: write
|
||||||
|
packages: read
|
||||||
|
pull-requests: write
|
||||||
|
statuses: write
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
env:
|
||||||
|
AWS_REGION: eu-west-2
|
||||||
|
AWS_DEFAULT_REGION: eu-west-2
|
||||||
|
AWS_STACK_NAME: game-ci-team-pipelines
|
||||||
|
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
||||||
|
DEBUG: true
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
USE_IL2CPP: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
k8s:
|
||||||
|
name: Cloud Runner Tests (K8s)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
# K8s runs (k3s)
|
||||||
|
- test: 'cloud-runner-end2end-caching'
|
||||||
|
provider: k8s
|
||||||
|
- test: 'cloud-runner-end2end-retaining'
|
||||||
|
provider: k8s
|
||||||
|
- test: 'cloud-runner-hooks'
|
||||||
|
provider: k8s
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
lfs: false
|
||||||
|
# Set up Kubernetes (k3s via k3d) only for k8s matrix entries
|
||||||
|
- name: Set up kubectl
|
||||||
|
if: ${{ matrix.provider == 'k8s' }}
|
||||||
|
uses: azure/setup-kubectl@v4
|
||||||
|
with:
|
||||||
|
version: 'v1.29.0'
|
||||||
|
- name: Install k3d
|
||||||
|
if: ${{ matrix.provider == 'k8s' }}
|
||||||
|
run: |
|
||||||
|
curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
|
||||||
|
k3d version | cat
|
||||||
|
- name: Create k3s cluster (k3d)
|
||||||
|
if: ${{ matrix.provider == 'k8s' }}
|
||||||
|
run: |
|
||||||
|
k3d cluster create unity-builder --agents 1 --wait
|
||||||
|
kubectl config current-context | cat
|
||||||
|
- name: Verify cluster readiness
|
||||||
|
if: ${{ matrix.provider == 'k8s' }}
|
||||||
|
run: |
|
||||||
|
for i in {1..60}; do kubectl get nodes && break || sleep 5; done
|
||||||
|
kubectl get storageclass
|
||||||
|
- name: Start LocalStack (S3)
|
||||||
|
uses: localstack/setup-localstack@v0.2.3
|
||||||
|
with:
|
||||||
|
install-awslocal: true
|
||||||
|
- name: Create S3 bucket for tests (host LocalStack)
|
||||||
|
run: |
|
||||||
|
awslocal s3 mb s3://$AWS_STACK_NAME || true
|
||||||
|
awslocal s3 ls
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: 'yarn'
|
||||||
|
- run: yarn install --frozen-lockfile
|
||||||
|
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||||
|
timeout-minutes: 60
|
||||||
|
env:
|
||||||
|
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||||
|
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||||
|
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
TARGET_PLATFORM: StandaloneWindows64
|
||||||
|
cloudRunnerTests: true
|
||||||
|
versioning: None
|
||||||
|
KUBE_STORAGE_CLASS: ${{ matrix.provider == 'k8s' && 'local-path' || '' }}
|
||||||
|
PROVIDER_STRATEGY: ${{ matrix.provider }}
|
||||||
|
AWS_ACCESS_KEY_ID: test
|
||||||
|
AWS_SECRET_ACCESS_KEY: test
|
||||||
|
AWS_S3_ENDPOINT: http://localhost:4566
|
||||||
|
AWS_ENDPOINT: http://localhost:4566
|
||||||
|
INPUT_AWSS3ENDPOINT: http://localhost:4566
|
||||||
|
INPUT_AWSENDPOINT: http://localhost:4566
|
||||||
|
AWS_S3_FORCE_PATH_STYLE: 'true'
|
||||||
|
AWS_EC2_METADATA_DISABLED: 'true'
|
||||||
|
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
localstack:
|
||||||
|
name: Cloud Runner Tests (LocalStack)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
services:
|
||||||
|
localstack:
|
||||||
|
image: localstack/localstack
|
||||||
|
ports:
|
||||||
|
- 4566:4566
|
||||||
|
env:
|
||||||
|
SERVICES: cloudformation,ecs,kinesis,cloudwatch,s3,logs
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
test:
|
||||||
|
- 'cloud-runner-end2end-locking'
|
||||||
|
- 'cloud-runner-end2end-caching'
|
||||||
|
- 'cloud-runner-end2end-retaining'
|
||||||
|
- 'cloud-runner-caching'
|
||||||
|
- 'cloud-runner-environment'
|
||||||
|
- 'cloud-runner-image'
|
||||||
|
- 'cloud-runner-hooks'
|
||||||
|
- 'cloud-runner-local-persistence'
|
||||||
|
- 'cloud-runner-locking-core'
|
||||||
|
- 'cloud-runner-locking-get-locked'
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
lfs: false
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: 'yarn'
|
||||||
|
- run: yarn install --frozen-lockfile
|
||||||
|
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||||
|
timeout-minutes: 60
|
||||||
|
env:
|
||||||
|
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||||
|
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||||
|
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
TARGET_PLATFORM: StandaloneWindows64
|
||||||
|
cloudRunnerTests: true
|
||||||
|
versioning: None
|
||||||
|
KUBE_STORAGE_CLASS: local-path
|
||||||
|
PROVIDER_STRATEGY: aws
|
||||||
|
AWS_ACCESS_KEY_ID: test
|
||||||
|
AWS_SECRET_ACCESS_KEY: test
|
||||||
|
AWS_ENDPOINT: http://localhost:4566
|
||||||
|
AWS_ENDPOINT_URL: http://localhost:4566
|
||||||
|
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
aws:
|
||||||
|
name: Cloud Runner Tests (AWS)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [k8s, localstack]
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
test:
|
||||||
|
- 'cloud-runner-end2end-caching'
|
||||||
|
- 'cloud-runner-end2end-retaining'
|
||||||
|
- 'cloud-runner-hooks'
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
lfs: false
|
||||||
|
- name: Configure AWS Credentials
|
||||||
|
uses: aws-actions/configure-aws-credentials@v1
|
||||||
|
with:
|
||||||
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||||
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||||
|
aws-region: ${{ env.AWS_REGION }}
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: 'yarn'
|
||||||
|
- run: yarn install --frozen-lockfile
|
||||||
|
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
||||||
|
timeout-minutes: 60
|
||||||
|
env:
|
||||||
|
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||||
|
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||||
|
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||||
|
PROJECT_PATH: test-project
|
||||||
|
TARGET_PLATFORM: StandaloneWindows64
|
||||||
|
cloudRunnerTests: true
|
||||||
|
versioning: None
|
||||||
|
PROVIDER_STRATEGY: aws
|
||||||
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||||
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||||
|
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
@@ -4,11 +4,6 @@ on:
|
|||||||
push: { branches: [main] }
|
push: { branches: [main] }
|
||||||
pull_request: {}
|
pull_request: {}
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
checks: write
|
|
||||||
statuses: write
|
|
||||||
|
|
||||||
env:
|
env:
|
||||||
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
|
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
|
||||||
|
|
||||||
@@ -32,7 +27,8 @@ jobs:
|
|||||||
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
|
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
|
||||||
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
|
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
|
||||||
|
|
||||||
orchestrator:
|
cloud-runner:
|
||||||
name: Orchestrator Integrity
|
name: Cloud Runner Integrity
|
||||||
uses: ./.github/workflows/orchestrator-integrity.yml
|
uses: ./.github/workflows/cloud-runner-integrity.yml
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -5,5 +5,3 @@ lib/
|
|||||||
.vsconfig
|
.vsconfig
|
||||||
yarn-error.log
|
yarn-error.log
|
||||||
.orig
|
.orig
|
||||||
$LOG_FILE
|
|
||||||
temp/
|
|
||||||
|
|||||||
+22
-53
@@ -104,11 +104,11 @@ inputs:
|
|||||||
gitPrivateToken:
|
gitPrivateToken:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[Orchestrator] Github private token to pull from github'
|
description: '[CloudRunner] Github private token to pull from github'
|
||||||
githubOwner:
|
githubOwner:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[Orchestrator] GitHub owner name or organization/team name'
|
description: '[CloudRunner] GitHub owner name or organization/team name'
|
||||||
runAsHostUser:
|
runAsHostUser:
|
||||||
required: false
|
required: false
|
||||||
default: 'false'
|
default: 'false'
|
||||||
@@ -149,101 +149,97 @@ inputs:
|
|||||||
allowDirtyBuild:
|
allowDirtyBuild:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[Orchestrator] Allows the branch of the build to be dirty, and still generate the build.'
|
description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.'
|
||||||
postBuildSteps:
|
postBuildSteps:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] run a post build job in yaml format with the keys image, secrets (name, value object array),
|
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array),
|
||||||
command string'
|
command string'
|
||||||
preBuildSteps:
|
preBuildSteps:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Run a pre build job after the repository setup but before the build job (in yaml format with the
|
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
|
||||||
keys image, secrets (name, value object array), command line string)'
|
keys image, secrets (name, value object array), command line string)'
|
||||||
containerHookFiles:
|
containerHookFiles:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Specify the names (by file name) of custom steps to run before or after orchestrator jobs, must
|
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must
|
||||||
match a yaml step file inside your repo in the folder .game-ci/steps/'
|
match a yaml step file inside your repo in the folder .game-ci/steps/'
|
||||||
customHookFiles:
|
customHookFiles:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Specify the names (by file name) of custom hooks to run before or after orchestrator jobs, must
|
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
|
||||||
match a yaml step file inside your repo in the folder .game-ci/hooks/'
|
match a yaml step file inside your repo in the folder .game-ci/hooks/'
|
||||||
customCommandHooks:
|
customCommandHooks:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[Orchestrator] Specify custom commands and trigger hooks (injects commands into jobs)'
|
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
|
||||||
customJob:
|
customJob:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
|
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
|
||||||
the keys image, secrets (name, value object array), command line string)'
|
keys image, secrets (name, value object array), command line string)'
|
||||||
awsStackName:
|
awsStackName:
|
||||||
default: 'game-ci'
|
default: 'game-ci'
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] The Cloud Formation stack name that must be setup before using this option.'
|
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
|
||||||
providerStrategy:
|
providerStrategy:
|
||||||
default: 'local'
|
default: 'local'
|
||||||
required: false
|
required: false
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
|
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
|
||||||
be configured.'
|
be configured.'
|
||||||
resourceTracking:
|
|
||||||
default: 'false'
|
|
||||||
required: false
|
|
||||||
description: '[Orchestrator] Enable resource tracking logs for disk usage and allocation summaries.'
|
|
||||||
containerCpu:
|
containerCpu:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Amount of CPU time to assign the remote build container'
|
description: '[CloudRunner] Amount of CPU time to assign the remote build container'
|
||||||
containerMemory:
|
containerMemory:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Amount of memory to assign the remote build container'
|
description: '[CloudRunner] Amount of memory to assign the remote build container'
|
||||||
readInputFromOverrideList:
|
readInputFromOverrideList:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Comma separated list of input value names to read from "input override command"'
|
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"'
|
||||||
readInputOverrideCommand:
|
readInputOverrideCommand:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
|
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
|
||||||
provider secret managers'
|
provider secret managers'
|
||||||
kubeConfig:
|
kubeConfig:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
|
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
|
||||||
completion.'
|
completion.'
|
||||||
kubeVolume:
|
kubeVolume:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Supply a Persistent Volume Claim name to use for the Unity build.'
|
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.'
|
||||||
kubeStorageClass:
|
kubeStorageClass:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Kubernetes storage class to use for orchestrator jobs, leave empty to install rook cluster.'
|
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
|
||||||
kubeVolumeSize:
|
kubeVolumeSize:
|
||||||
default: '5Gi'
|
default: '5Gi'
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Amount of disc space to assign the Kubernetes Persistent Volume'
|
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume'
|
||||||
cacheKey:
|
cacheKey:
|
||||||
default: ''
|
default: ''
|
||||||
required: false
|
required: false
|
||||||
description: '[Orchestrator] Cache key to indicate bucket for cache'
|
description: '[CloudRunner] Cache key to indicate bucket for cache'
|
||||||
watchToEnd:
|
watchToEnd:
|
||||||
default: 'true'
|
default: 'true'
|
||||||
required: false
|
required: false
|
||||||
description:
|
description:
|
||||||
'[Orchestrator] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
|
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
|
||||||
imports or self-hosted ephemeral runners.'
|
imports or self-hosted ephemeral runners.'
|
||||||
cacheUnityInstallationOnMac:
|
cacheUnityInstallationOnMac:
|
||||||
default: 'false'
|
default: 'false'
|
||||||
@@ -269,33 +265,6 @@ inputs:
|
|||||||
default: 'false'
|
default: 'false'
|
||||||
required: false
|
required: false
|
||||||
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
|
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
|
||||||
cloneDepth:
|
|
||||||
default: '50'
|
|
||||||
required: false
|
|
||||||
description: '[Orchestrator] Specifies the depth of the git clone for the repository. Use 0 for full clone.'
|
|
||||||
orchestratorRepoName:
|
|
||||||
default: 'game-ci/unity-builder'
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
|
|
||||||
fixes.'
|
|
||||||
testSuitePath:
|
|
||||||
description: 'Path to YAML test suite definition file'
|
|
||||||
required: false
|
|
||||||
testSuiteEvent:
|
|
||||||
description: 'CI event name for suite selection (pr, push, release)'
|
|
||||||
required: false
|
|
||||||
testTaxonomyPath:
|
|
||||||
description: 'Path to custom taxonomy definition YAML'
|
|
||||||
required: false
|
|
||||||
testResultFormat:
|
|
||||||
description: 'Test result output format: junit, json, or both'
|
|
||||||
required: false
|
|
||||||
default: 'junit'
|
|
||||||
testResultPath:
|
|
||||||
description: 'Directory for structured test result output'
|
|
||||||
required: false
|
|
||||||
default: './test-results'
|
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
volume:
|
volume:
|
||||||
|
|||||||
@@ -56,19 +56,14 @@ namespace UnityBuilderAction
|
|||||||
// of either `UnityEditor.BuildPlayerOptions` or `UnityEditor.BuildPlayerWithProfileOptions`
|
// of either `UnityEditor.BuildPlayerOptions` or `UnityEditor.BuildPlayerWithProfileOptions`
|
||||||
dynamic buildPlayerOptions;
|
dynamic buildPlayerOptions;
|
||||||
|
|
||||||
if (options.TryGetValue("activeBuildProfile", out var buildProfilePath)) {
|
if (options["customBuildProfile"] != "") {
|
||||||
if (string.IsNullOrEmpty(buildProfilePath)) {
|
|
||||||
throw new Exception("`-activeBuildProfile` is set but with an empty value; this shouldn't happen");
|
|
||||||
}
|
|
||||||
|
|
||||||
#if UNITY_6000_0_OR_NEWER
|
#if UNITY_6000_0_OR_NEWER
|
||||||
// Load build profile from Assets folder
|
// Load build profile from Assets folder
|
||||||
var buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(buildProfilePath)
|
BuildProfile buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(options["customBuildProfile"]);
|
||||||
?? throw new Exception("Build profile file not found at path: " + buildProfilePath);
|
|
||||||
|
|
||||||
// no need to set active profile, as already set by `-activeBuildProfile` CLI argument
|
// Set it as active
|
||||||
// BuildProfile.SetActiveBuildProfile(buildProfile);
|
BuildProfile.SetActiveBuildProfile(buildProfile);
|
||||||
Debug.Log($"build profile: {buildProfile.name}");
|
|
||||||
|
|
||||||
// Define BuildPlayerWithProfileOptions
|
// Define BuildPlayerWithProfileOptions
|
||||||
buildPlayerOptions = new BuildPlayerWithProfileOptions {
|
buildPlayerOptions = new BuildPlayerWithProfileOptions {
|
||||||
@@ -76,16 +71,12 @@ namespace UnityBuilderAction
|
|||||||
locationPathName = options["customBuildPath"],
|
locationPathName = options["customBuildPath"],
|
||||||
options = buildOptions,
|
options = buildOptions,
|
||||||
};
|
};
|
||||||
#else // UNITY_6000_0_OR_NEWER
|
#else
|
||||||
throw new Exception("Build profiles are not supported by this version of Unity (" + Application.unityVersion +")");
|
throw new Exception("Build profiles are not supported by this version of Unity (" + Application.unityVersion +")");
|
||||||
#endif // UNITY_6000_0_OR_NEWER
|
#endif
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
#if BUILD_PROFILE_LOADED
|
|
||||||
throw new Exception("Build profile's define symbol present; shouldn't happen");
|
|
||||||
#endif // BUILD_PROFILE_LOADED
|
|
||||||
|
|
||||||
// Gather values from project
|
// Gather values from project
|
||||||
var scenes = EditorBuildSettings.scenes.Where(scene => scene.enabled).Select(s => s.path).ToArray();
|
var scenes = EditorBuildSettings.scenes.Where(scene => scene.enabled).Select(s => s.path).ToArray();
|
||||||
|
|
||||||
|
|||||||
-2
@@ -115,7 +115,6 @@ namespace UnityBuilderAction.Input
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#if UNITY_6000_0_OR_NEWER
|
|
||||||
private static void SetDebugSymbols(string enumValueName)
|
private static void SetDebugSymbols(string enumValueName)
|
||||||
{
|
{
|
||||||
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
|
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
|
||||||
@@ -145,6 +144,5 @@ namespace UnityBuilderAction.Input
|
|||||||
}
|
}
|
||||||
levelProp.SetValue(null, enumValue);
|
levelProp.SetValue(null, enumValue);
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
-14
@@ -21,19 +21,6 @@ namespace UnityBuilderAction.Input
|
|||||||
EditorApplication.Exit(110);
|
EditorApplication.Exit(110);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if UNITY_6000_0_OR_NEWER
|
|
||||||
var buildProfileSupport = true;
|
|
||||||
#else
|
|
||||||
var buildProfileSupport = false;
|
|
||||||
#endif // UNITY_6000_0_OR_NEWER
|
|
||||||
|
|
||||||
string buildProfile;
|
|
||||||
if (buildProfileSupport && validatedOptions.TryGetValue("activeBuildProfile", out buildProfile)) {
|
|
||||||
if (validatedOptions.ContainsKey("buildTarget")) {
|
|
||||||
Console.WriteLine("Extra argument -buildTarget");
|
|
||||||
EditorApplication.Exit(122);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
string buildTarget;
|
string buildTarget;
|
||||||
if (!validatedOptions.TryGetValue("buildTarget", out buildTarget)) {
|
if (!validatedOptions.TryGetValue("buildTarget", out buildTarget)) {
|
||||||
Console.WriteLine("Missing argument -buildTarget");
|
Console.WriteLine("Missing argument -buildTarget");
|
||||||
@@ -44,7 +31,6 @@ namespace UnityBuilderAction.Input
|
|||||||
Console.WriteLine(buildTarget + " is not a defined " + typeof(BuildTarget).Name);
|
Console.WriteLine(buildTarget + " is not a defined " + typeof(BuildTarget).Name);
|
||||||
EditorApplication.Exit(121);
|
EditorApplication.Exit(121);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
string customBuildPath;
|
string customBuildPath;
|
||||||
if (!validatedOptions.TryGetValue("customBuildPath", out customBuildPath)) {
|
if (!validatedOptions.TryGetValue("customBuildPath", out customBuildPath)) {
|
||||||
|
|||||||
+30608
-30525
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+95
-102
@@ -6865,6 +6865,76 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
|
@deno/shim-deno
|
||||||
|
MIT
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright 2021-2022 the Deno authors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
|
@deno/shim-deno-test
|
||||||
|
MIT
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright 2021-2022 the Deno authors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
|
@fastify/busboy
|
||||||
|
MIT
|
||||||
|
Copyright Brian White. All rights reserved.
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to
|
||||||
|
deal in the Software without restriction, including without limitation the
|
||||||
|
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||||
|
sell copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
|
IN THE SOFTWARE.
|
||||||
|
|
||||||
@kubernetes/client-node
|
@kubernetes/client-node
|
||||||
Apache-2.0
|
Apache-2.0
|
||||||
Apache License
|
Apache License
|
||||||
@@ -17052,31 +17122,6 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|||||||
IN THE SOFTWARE.
|
IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
cross-fetch
|
|
||||||
MIT
|
|
||||||
The MIT License (MIT)
|
|
||||||
|
|
||||||
Copyright (c) 2017 Leonardo Quixadá
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
|
|
||||||
|
|
||||||
cross-spawn
|
cross-spawn
|
||||||
MIT
|
MIT
|
||||||
The MIT License (MIT)
|
The MIT License (MIT)
|
||||||
@@ -17569,56 +17614,6 @@ The above copyright notice and this permission notice shall be included in all c
|
|||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
graphql
|
|
||||||
MIT
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) GraphQL Contributors
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
|
|
||||||
|
|
||||||
graphql-request
|
|
||||||
MIT
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2022 Jason Kuhrt
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
|
|
||||||
|
|
||||||
har-schema
|
har-schema
|
||||||
ISC
|
ISC
|
||||||
Copyright (c) 2015, Ahmad Nassri <ahmad@ahmadnassri.com>
|
Copyright (c) 2015, Ahmad Nassri <ahmad@ahmadnassri.com>
|
||||||
@@ -19412,33 +19407,6 @@ The above copyright notice and this permission notice shall be included in all c
|
|||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
shell-quote
|
|
||||||
MIT
|
|
||||||
The MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2013 James Halliday (mail@substack.net)
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge,
|
|
||||||
to any person obtaining a copy of this software and
|
|
||||||
associated documentation files (the "Software"), to
|
|
||||||
deal in the Software without restriction, including
|
|
||||||
without limitation the rights to use, copy, modify,
|
|
||||||
merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom
|
|
||||||
the Software is furnished to do so,
|
|
||||||
subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice
|
|
||||||
shall be included in all copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
||||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
||||||
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
||||||
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
|
|
||||||
ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
||||||
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
||||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
|
|
||||||
shelljs
|
shelljs
|
||||||
BSD-3-Clause
|
BSD-3-Clause
|
||||||
Copyright (c) 2012, Artur Adib <arturadib@gmail.com>
|
Copyright (c) 2012, Artur Adib <arturadib@gmail.com>
|
||||||
@@ -19773,6 +19741,31 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|||||||
OTHER DEALINGS IN THE SOFTWARE.
|
OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
|
undici
|
||||||
|
MIT
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) Matteo Collina and Undici contributors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
unity-changeset
|
unity-changeset
|
||||||
MIT
|
MIT
|
||||||
MIT License
|
MIT License
|
||||||
|
|||||||
Vendored
+13
-61
@@ -4,69 +4,21 @@
|
|||||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||||
pushd "$ACTIVATE_LICENSE_PATH"
|
pushd "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
if [[ -n "$UNITY_SERIAL" && -n "$UNITY_EMAIL" && -n "$UNITY_PASSWORD" ]]; then
|
echo "Requesting activation"
|
||||||
#
|
|
||||||
# SERIAL LICENSE MODE
|
|
||||||
#
|
|
||||||
# This will activate unity, using the serial activation process.
|
|
||||||
#
|
|
||||||
|
|
||||||
echo "Requesting activation"
|
# Activate license
|
||||||
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||||
|
-logFile - \
|
||||||
|
-batchmode \
|
||||||
|
-nographics \
|
||||||
|
-quit \
|
||||||
|
-serial "$UNITY_SERIAL" \
|
||||||
|
-username "$UNITY_EMAIL" \
|
||||||
|
-password "$UNITY_PASSWORD" \
|
||||||
|
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
# Activate license
|
# Store the exit code from the verify command
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
UNITY_EXIT_CODE=$?
|
||||||
-logFile - \
|
|
||||||
-batchmode \
|
|
||||||
-nographics \
|
|
||||||
-quit \
|
|
||||||
-serial "$UNITY_SERIAL" \
|
|
||||||
-username "$UNITY_EMAIL" \
|
|
||||||
-password "$UNITY_PASSWORD" \
|
|
||||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
|
||||||
|
|
||||||
# Store the exit code from the verify command
|
|
||||||
UNITY_EXIT_CODE=$?
|
|
||||||
|
|
||||||
elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
|
||||||
#
|
|
||||||
# Custom Unity License Server
|
|
||||||
#
|
|
||||||
echo "Adding licensing server config"
|
|
||||||
mkdir -p "$UNITY_LICENSE_PATH/config/"
|
|
||||||
cp "$ACTION_FOLDER/unity-config/services-config.json" "$UNITY_LICENSE_PATH/config/services-config.json"
|
|
||||||
|
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
|
||||||
--acquire-floating > license.txt
|
|
||||||
|
|
||||||
# Store the exit code from the verify command
|
|
||||||
UNITY_EXIT_CODE=$?
|
|
||||||
|
|
||||||
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
|
||||||
PARSEDFILE=$(grep -oE '\"[^"]*\"' < license.txt | tr -d '"')
|
|
||||||
export FLOATING_LICENSE
|
|
||||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
|
||||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
|
||||||
|
|
||||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
#
|
|
||||||
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
|
||||||
#
|
|
||||||
# This will exit since no activation strategies could be matched.
|
|
||||||
#
|
|
||||||
echo "License activation strategy could not be determined."
|
|
||||||
echo ""
|
|
||||||
echo "Visit https://game.ci/docs/github/activation for more"
|
|
||||||
echo "details on how to set up one of the possible activation strategies."
|
|
||||||
|
|
||||||
echo "::error ::No valid license activation strategy could be determined. Make sure to provide UNITY_EMAIL, UNITY_PASSWORD, and either a UNITY_SERIAL \
|
|
||||||
or UNITY_LICENSE. Otherwise please use UNITY_LICENSING_SERVER. See more info at https://game.ci/docs/github/activation"
|
|
||||||
|
|
||||||
# Immediately exit as no UNITY_EXIT_CODE can be derived.
|
|
||||||
exit 1;
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# Display information about the result
|
# Display information about the result
|
||||||
|
|||||||
Vendored
+3
-2
@@ -149,13 +149,14 @@ echo ""
|
|||||||
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
||||||
-batchmode \
|
-batchmode \
|
||||||
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
|
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
|
||||||
|
-username "$UNITY_EMAIL" \
|
||||||
|
-password "$UNITY_PASSWORD" \
|
||||||
-customBuildName "$BUILD_NAME" \
|
-customBuildName "$BUILD_NAME" \
|
||||||
-projectPath "$UNITY_PROJECT_PATH" \
|
-projectPath "$UNITY_PROJECT_PATH" \
|
||||||
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET") \
|
-buildTarget "$BUILD_TARGET" \
|
||||||
-customBuildTarget "$BUILD_TARGET" \
|
-customBuildTarget "$BUILD_TARGET" \
|
||||||
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
||||||
-customBuildProfile "$BUILD_PROFILE" \
|
-customBuildProfile "$BUILD_PROFILE" \
|
||||||
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
|
|
||||||
-executeMethod "$BUILD_METHOD" \
|
-executeMethod "$BUILD_METHOD" \
|
||||||
-buildVersion "$VERSION" \
|
-buildVersion "$VERSION" \
|
||||||
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
||||||
|
|||||||
+9
-23
@@ -4,29 +4,15 @@
|
|||||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||||
pushd "$ACTIVATE_LICENSE_PATH"
|
pushd "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
if [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||||
#
|
-logFile - \
|
||||||
# Return any floating license used.
|
-batchmode \
|
||||||
#
|
-nographics \
|
||||||
echo "Returning floating license: \"$FLOATING_LICENSE\""
|
-quit \
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
-username "$UNITY_EMAIL" \
|
||||||
--return-floating "$FLOATING_LICENSE"
|
-password "$UNITY_PASSWORD" \
|
||||||
elif [[ -n "$UNITY_SERIAL" ]]; then
|
-returnlicense \
|
||||||
#
|
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||||
# SERIAL LICENSE MODE
|
|
||||||
#
|
|
||||||
# This will return the license that is currently in use.
|
|
||||||
#
|
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
|
||||||
-logFile - \
|
|
||||||
-batchmode \
|
|
||||||
-nographics \
|
|
||||||
-quit \
|
|
||||||
-username "$UNITY_EMAIL" \
|
|
||||||
-password "$UNITY_PASSWORD" \
|
|
||||||
-returnlicense \
|
|
||||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Return to previous working directory
|
# Return to previous working directory
|
||||||
popd
|
popd
|
||||||
|
|||||||
+5
-9
@@ -68,18 +68,14 @@ elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
|||||||
echo "Adding licensing server config"
|
echo "Adding licensing server config"
|
||||||
|
|
||||||
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
|
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
|
||||||
|
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
||||||
|
export FLOATING_LICENSE
|
||||||
|
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||||
|
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||||
|
|
||||||
|
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||||
# Store the exit code from the verify command
|
# Store the exit code from the verify command
|
||||||
UNITY_EXIT_CODE=$?
|
UNITY_EXIT_CODE=$?
|
||||||
|
|
||||||
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
|
||||||
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
|
||||||
export FLOATING_LICENSE
|
|
||||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
|
||||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
|
||||||
|
|
||||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
#
|
#
|
||||||
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
||||||
|
|||||||
Vendored
+1
-2
@@ -125,11 +125,10 @@ unity-editor \
|
|||||||
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
||||||
-customBuildName "$BUILD_NAME" \
|
-customBuildName "$BUILD_NAME" \
|
||||||
-projectPath "$UNITY_PROJECT_PATH" \
|
-projectPath "$UNITY_PROJECT_PATH" \
|
||||||
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET" ) \
|
-buildTarget "$BUILD_TARGET" \
|
||||||
-customBuildTarget "$BUILD_TARGET" \
|
-customBuildTarget "$BUILD_TARGET" \
|
||||||
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
-customBuildPath "$CUSTOM_BUILD_PATH" \
|
||||||
-customBuildProfile "$BUILD_PROFILE" \
|
-customBuildProfile "$BUILD_PROFILE" \
|
||||||
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
|
|
||||||
-executeMethod "$BUILD_METHOD" \
|
-executeMethod "$BUILD_METHOD" \
|
||||||
-buildVersion "$VERSION" \
|
-buildVersion "$VERSION" \
|
||||||
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
-androidVersionCode "$ANDROID_VERSION_CODE" \
|
||||||
|
|||||||
Vendored
+1
-7
@@ -166,6 +166,7 @@ $unityArgs = @(
|
|||||||
"-customBuildName", "`"$Env:BUILD_NAME`"",
|
"-customBuildName", "`"$Env:BUILD_NAME`"",
|
||||||
"-projectPath", "`"$Env:UNITY_PROJECT_PATH`"",
|
"-projectPath", "`"$Env:UNITY_PROJECT_PATH`"",
|
||||||
"-executeMethod", "`"$Env:BUILD_METHOD`"",
|
"-executeMethod", "`"$Env:BUILD_METHOD`"",
|
||||||
|
"-buildTarget", "`"$Env:BUILD_TARGET`"",
|
||||||
"-customBuildTarget", "`"$Env:BUILD_TARGET`"",
|
"-customBuildTarget", "`"$Env:BUILD_TARGET`"",
|
||||||
"-customBuildPath", "`"$Env:CUSTOM_BUILD_PATH`"",
|
"-customBuildPath", "`"$Env:CUSTOM_BUILD_PATH`"",
|
||||||
"-customBuildProfile", "`"$Env:BUILD_PROFILE`"",
|
"-customBuildProfile", "`"$Env:BUILD_PROFILE`"",
|
||||||
@@ -180,13 +181,6 @@ $unityArgs = @(
|
|||||||
"-logfile", "-"
|
"-logfile", "-"
|
||||||
) + $customParametersArray
|
) + $customParametersArray
|
||||||
|
|
||||||
if (-not $Env:BUILD_PROFILE) {
|
|
||||||
$unityArgs += @("-buildTarget", "`"$Env:BUILD_TARGET`"")
|
|
||||||
}
|
|
||||||
if ($Env:BUILD_PROFILE) {
|
|
||||||
$unityArgs += @("-activeBuildProfile", "`"$Env:BUILD_PROFILE`"")
|
|
||||||
}
|
|
||||||
|
|
||||||
# Remove null items as that will fail the Start-Process call
|
# Remove null items as that will fail the Start-Process call
|
||||||
$unityArgs = $unityArgs | Where-Object { $_ -ne $null }
|
$unityArgs = $unityArgs | Where-Object { $_ -ne $null }
|
||||||
|
|
||||||
|
|||||||
-3
@@ -18,9 +18,6 @@ regsvr32 C:\ProgramData\Microsoft\VisualStudio\Setup\x64\Microsoft.VisualStudio.
|
|||||||
# Kill the regsvr process
|
# Kill the regsvr process
|
||||||
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
|
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
|
||||||
|
|
||||||
# Install Visual C++ 2013 Redistributables
|
|
||||||
. "c:\steps\install_vcredist13.ps1"
|
|
||||||
|
|
||||||
# Setup Git Credentials
|
# Setup Git Credentials
|
||||||
. "c:\steps\set_gitcredential.ps1"
|
. "c:\steps\set_gitcredential.ps1"
|
||||||
|
|
||||||
|
|||||||
-11
@@ -1,11 +0,0 @@
|
|||||||
# For some reason, Unity is failing in github actions windows runners
|
|
||||||
# due to missing Visual C++ 2013 redistributables.
|
|
||||||
# This script downloads and installs the required redistributables.
|
|
||||||
Write-Output ""
|
|
||||||
Write-Output "#########################################################"
|
|
||||||
Write-Output "# Installing Visual C++ Redistributables (2013) #"
|
|
||||||
Write-Output "#########################################################"
|
|
||||||
Write-Output ""
|
|
||||||
|
|
||||||
|
|
||||||
choco install vcredist2013 -y --no-progress
|
|
||||||
+8
-9
@@ -12,18 +12,18 @@
|
|||||||
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
||||||
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
||||||
"cli": "yarn ts-node src/index.ts -m cli",
|
"cli": "yarn ts-node src/index.ts -m cli",
|
||||||
"gcp-secrets-tests": "cross-env providerStrategy=aws orchestratorTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"orchestrator\"",
|
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
|
||||||
"gcp-secrets-cli": "cross-env orchestratorTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
"gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||||
"aws-secrets-cli": "cross-env orchestratorTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
"aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||||
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
|
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
|
||||||
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
|
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
|
||||||
"test-cli": "cross-env orchestratorTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
||||||
"test": "jest",
|
"test": "jest",
|
||||||
"test:ci": "jest --config=jest.ci.config.js --runInBand",
|
"test:ci": "jest --config=jest.ci.config.js --runInBand",
|
||||||
"test-i": "cross-env orchestratorTests=true yarn test -i -t \"orchestrator\"",
|
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
|
||||||
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
|
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
|
||||||
"test-i-aws": "cross-env orchestratorTests=true providerStrategy=aws yarn test -i -t \"orchestrator\"",
|
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
|
||||||
"test-i-k8s": "cross-env orchestratorTests=true providerStrategy=k8s yarn test -i -t \"orchestrator\""
|
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18.x"
|
"node": ">=18.x"
|
||||||
@@ -50,9 +50,8 @@
|
|||||||
"nanoid": "^3.3.1",
|
"nanoid": "^3.3.1",
|
||||||
"reflect-metadata": "^0.1.13",
|
"reflect-metadata": "^0.1.13",
|
||||||
"semver": "^7.5.2",
|
"semver": "^7.5.2",
|
||||||
"shell-quote": "^1.8.3",
|
|
||||||
"ts-md5": "^1.3.1",
|
"ts-md5": "^1.3.1",
|
||||||
"unity-changeset": "^3.1.0",
|
"unity-changeset": "^2.0.0",
|
||||||
"uuid": "^9.0.0",
|
"uuid": "^9.0.0",
|
||||||
"yaml": "^2.2.2"
|
"yaml": "^2.2.2"
|
||||||
},
|
},
|
||||||
|
|||||||
+2
-20
@@ -1,9 +1,8 @@
|
|||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
|
import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model';
|
||||||
import { Cli } from './model/cli/cli';
|
import { Cli } from './model/cli/cli';
|
||||||
import MacBuilder from './model/mac-builder';
|
import MacBuilder from './model/mac-builder';
|
||||||
import PlatformSetup from './model/platform-setup';
|
import PlatformSetup from './model/platform-setup';
|
||||||
import { TestWorkflowService } from './model/orchestrator/services/test-workflow';
|
|
||||||
|
|
||||||
async function runMain() {
|
async function runMain() {
|
||||||
try {
|
try {
|
||||||
@@ -18,23 +17,6 @@ async function runMain() {
|
|||||||
const { workspace, actionFolder } = Action;
|
const { workspace, actionFolder } = Action;
|
||||||
|
|
||||||
const buildParameters = await BuildParameters.create();
|
const buildParameters = await BuildParameters.create();
|
||||||
|
|
||||||
// If a test suite path is provided, use the test workflow engine
|
|
||||||
// instead of the standard build execution path
|
|
||||||
if (buildParameters.testSuitePath) {
|
|
||||||
core.info('[TestWorkflow] Test suite path detected, using test workflow engine');
|
|
||||||
const results = await TestWorkflowService.executeTestSuite(buildParameters.testSuitePath, buildParameters);
|
|
||||||
|
|
||||||
const totalFailed = results.reduce((sum, r) => sum + r.failed, 0);
|
|
||||||
if (totalFailed > 0) {
|
|
||||||
core.setFailed(`Test workflow completed with ${totalFailed} failure(s)`);
|
|
||||||
} else {
|
|
||||||
core.info('[TestWorkflow] All test runs passed');
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const baseImage = new ImageTag(buildParameters);
|
const baseImage = new ImageTag(buildParameters);
|
||||||
|
|
||||||
let exitCode = -1;
|
let exitCode = -1;
|
||||||
@@ -51,7 +33,7 @@ async function runMain() {
|
|||||||
...buildParameters,
|
...buildParameters,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
await Orchestrator.run(buildParameters, baseImage.toString());
|
await CloudRunner.run(buildParameters, baseImage.toString());
|
||||||
exitCode = 0;
|
exitCode = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+5
-5
@@ -1,13 +1,13 @@
|
|||||||
// Integration test for exercising real GitHub check creation and updates.
|
// Integration test for exercising real GitHub check creation and updates.
|
||||||
import Orchestrator from '../model/orchestrator/orchestrator';
|
import CloudRunner from '../model/cloud-runner/cloud-runner';
|
||||||
import UnityVersioning from '../model/unity-versioning';
|
import UnityVersioning from '../model/unity-versioning';
|
||||||
import GitHub from '../model/github';
|
import GitHub from '../model/github';
|
||||||
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/orchestrator-test-helpers';
|
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/cloud-runner-test-helpers';
|
||||||
|
|
||||||
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
|
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
|
||||||
const describeOrSkip = runIntegration ? describe : describe.skip;
|
const describeOrSkip = runIntegration ? describe : describe.skip;
|
||||||
|
|
||||||
describeOrSkip('Orchestrator Github Checks Integration', () => {
|
describeOrSkip('Cloud Runner Github Checks Integration', () => {
|
||||||
it(
|
it(
|
||||||
'creates and updates a real GitHub check',
|
'creates and updates a real GitHub check',
|
||||||
async () => {
|
async () => {
|
||||||
@@ -15,10 +15,10 @@ describeOrSkip('Orchestrator Github Checks Integration', () => {
|
|||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
projectPath: 'test-project',
|
projectPath: 'test-project',
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
unityVersion: UnityVersioning.read('test-project'),
|
||||||
asyncOrchestrator: `true`,
|
asyncCloudRunner: `true`,
|
||||||
githubChecks: `true`,
|
githubChecks: `true`,
|
||||||
});
|
});
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
const checkId = await GitHub.createGitHubCheck(`integration create`);
|
const checkId = await GitHub.createGitHubCheck(`integration create`);
|
||||||
expect(checkId).not.toEqual('');
|
expect(checkId).not.toEqual('');
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);
|
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { customAlphabet } from 'nanoid';
|
import { customAlphabet } from 'nanoid';
|
||||||
import AndroidVersioning from './android-versioning';
|
import AndroidVersioning from './android-versioning';
|
||||||
import OrchestratorConstants from './orchestrator/options/orchestrator-constants';
|
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
|
||||||
import OrchestratorBuildGuid from './orchestrator/options/orchestrator-guid';
|
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
|
||||||
import Input from './input';
|
import Input from './input';
|
||||||
import Platform from './platform';
|
import Platform from './platform';
|
||||||
import UnityVersioning from './unity-versioning';
|
import UnityVersioning from './unity-versioning';
|
||||||
@@ -10,8 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
|
|||||||
import { GithubCliReader } from './input-readers/github-cli';
|
import { GithubCliReader } from './input-readers/github-cli';
|
||||||
import { Cli } from './cli/cli';
|
import { Cli } from './cli/cli';
|
||||||
import GitHub from './github';
|
import GitHub from './github';
|
||||||
import OrchestratorOptions from './orchestrator/options/orchestrator-options';
|
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
|
||||||
import Orchestrator from './orchestrator/orchestrator';
|
import CloudRunner from './cloud-runner/cloud-runner';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
|
|
||||||
class BuildParameters {
|
class BuildParameters {
|
||||||
@@ -67,7 +67,6 @@ class BuildParameters {
|
|||||||
public kubeConfig!: string;
|
public kubeConfig!: string;
|
||||||
public containerMemory!: string;
|
public containerMemory!: string;
|
||||||
public containerCpu!: string;
|
public containerCpu!: string;
|
||||||
public containerNamespace!: string;
|
|
||||||
public kubeVolumeSize!: string;
|
public kubeVolumeSize!: string;
|
||||||
public kubeVolume!: string;
|
public kubeVolume!: string;
|
||||||
public kubeStorageClass!: string;
|
public kubeStorageClass!: string;
|
||||||
@@ -84,13 +83,11 @@ class BuildParameters {
|
|||||||
public runNumber!: string;
|
public runNumber!: string;
|
||||||
public branch!: string;
|
public branch!: string;
|
||||||
public githubRepo!: string;
|
public githubRepo!: string;
|
||||||
public orchestratorRepoName!: string;
|
|
||||||
public cloneDepth!: number;
|
|
||||||
public gitSha!: string;
|
public gitSha!: string;
|
||||||
public logId!: string;
|
public logId!: string;
|
||||||
public buildGuid!: string;
|
public buildGuid!: string;
|
||||||
public orchestratorBranch!: string;
|
public cloudRunnerBranch!: string;
|
||||||
public orchestratorDebug!: boolean | undefined;
|
public cloudRunnerDebug!: boolean | undefined;
|
||||||
public buildPlatform!: string | undefined;
|
public buildPlatform!: string | undefined;
|
||||||
public isCliMode!: boolean;
|
public isCliMode!: boolean;
|
||||||
public maxRetainedWorkspaces!: number;
|
public maxRetainedWorkspaces!: number;
|
||||||
@@ -107,14 +104,8 @@ class BuildParameters {
|
|||||||
public unityHubVersionOnMac!: string;
|
public unityHubVersionOnMac!: string;
|
||||||
public dockerWorkspacePath!: string;
|
public dockerWorkspacePath!: string;
|
||||||
|
|
||||||
public testSuitePath!: string;
|
|
||||||
public testSuiteEvent!: string;
|
|
||||||
public testTaxonomyPath!: string;
|
|
||||||
public testResultFormat!: string;
|
|
||||||
public testResultPath!: string;
|
|
||||||
|
|
||||||
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
||||||
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
|
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
|
||||||
}
|
}
|
||||||
|
|
||||||
static async create(): Promise<BuildParameters> {
|
static async create(): Promise<BuildParameters> {
|
||||||
@@ -199,60 +190,52 @@ class BuildParameters {
|
|||||||
dockerIsolationMode: Input.dockerIsolationMode,
|
dockerIsolationMode: Input.dockerIsolationMode,
|
||||||
containerRegistryRepository: Input.containerRegistryRepository,
|
containerRegistryRepository: Input.containerRegistryRepository,
|
||||||
containerRegistryImageVersion: Input.containerRegistryImageVersion,
|
containerRegistryImageVersion: Input.containerRegistryImageVersion,
|
||||||
providerStrategy: OrchestratorOptions.providerStrategy,
|
providerStrategy: CloudRunnerOptions.providerStrategy,
|
||||||
buildPlatform: OrchestratorOptions.buildPlatform,
|
buildPlatform: CloudRunnerOptions.buildPlatform,
|
||||||
kubeConfig: OrchestratorOptions.kubeConfig,
|
kubeConfig: CloudRunnerOptions.kubeConfig,
|
||||||
containerMemory: OrchestratorOptions.containerMemory,
|
containerMemory: CloudRunnerOptions.containerMemory,
|
||||||
containerCpu: OrchestratorOptions.containerCpu,
|
containerCpu: CloudRunnerOptions.containerCpu,
|
||||||
containerNamespace: OrchestratorOptions.containerNamespace,
|
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
|
||||||
kubeVolumeSize: OrchestratorOptions.kubeVolumeSize,
|
kubeVolume: CloudRunnerOptions.kubeVolume,
|
||||||
kubeVolume: OrchestratorOptions.kubeVolume,
|
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
|
||||||
postBuildContainerHooks: OrchestratorOptions.postBuildContainerHooks,
|
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
|
||||||
preBuildContainerHooks: OrchestratorOptions.preBuildContainerHooks,
|
customJob: CloudRunnerOptions.customJob,
|
||||||
customJob: OrchestratorOptions.customJob,
|
|
||||||
runNumber: Input.runNumber,
|
runNumber: Input.runNumber,
|
||||||
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
||||||
orchestratorBranch: OrchestratorOptions.orchestratorBranch.split('/').reverse()[0],
|
cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0],
|
||||||
orchestratorDebug: OrchestratorOptions.orchestratorDebug,
|
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
|
||||||
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || OrchestratorOptions.orchestratorRepoName,
|
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
|
||||||
orchestratorRepoName: OrchestratorOptions.orchestratorRepoName,
|
|
||||||
cloneDepth: Number.parseInt(OrchestratorOptions.cloneDepth),
|
|
||||||
isCliMode: Cli.isCliMode,
|
isCliMode: Cli.isCliMode,
|
||||||
awsStackName: OrchestratorOptions.awsStackName,
|
awsStackName: CloudRunnerOptions.awsStackName,
|
||||||
awsEndpoint: OrchestratorOptions.awsEndpoint,
|
awsEndpoint: CloudRunnerOptions.awsEndpoint,
|
||||||
awsCloudFormationEndpoint: OrchestratorOptions.awsCloudFormationEndpoint,
|
awsCloudFormationEndpoint: CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||||
awsEcsEndpoint: OrchestratorOptions.awsEcsEndpoint,
|
awsEcsEndpoint: CloudRunnerOptions.awsEcsEndpoint,
|
||||||
awsKinesisEndpoint: OrchestratorOptions.awsKinesisEndpoint,
|
awsKinesisEndpoint: CloudRunnerOptions.awsKinesisEndpoint,
|
||||||
awsCloudWatchLogsEndpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
awsCloudWatchLogsEndpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||||
awsS3Endpoint: OrchestratorOptions.awsS3Endpoint,
|
awsS3Endpoint: CloudRunnerOptions.awsS3Endpoint,
|
||||||
storageProvider: OrchestratorOptions.storageProvider,
|
storageProvider: CloudRunnerOptions.storageProvider,
|
||||||
rcloneRemote: OrchestratorOptions.rcloneRemote,
|
rcloneRemote: CloudRunnerOptions.rcloneRemote,
|
||||||
gitSha: Input.gitSha,
|
gitSha: Input.gitSha,
|
||||||
logId: customAlphabet(OrchestratorConstants.alphabet, 9)(),
|
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
|
||||||
buildGuid: OrchestratorBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
||||||
commandHooks: OrchestratorOptions.commandHooks,
|
commandHooks: CloudRunnerOptions.commandHooks,
|
||||||
inputPullCommand: OrchestratorOptions.inputPullCommand,
|
inputPullCommand: CloudRunnerOptions.inputPullCommand,
|
||||||
pullInputList: OrchestratorOptions.pullInputList,
|
pullInputList: CloudRunnerOptions.pullInputList,
|
||||||
kubeStorageClass: OrchestratorOptions.kubeStorageClass,
|
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
|
||||||
cacheKey: OrchestratorOptions.cacheKey,
|
cacheKey: CloudRunnerOptions.cacheKey,
|
||||||
maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces),
|
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
|
||||||
useLargePackages: OrchestratorOptions.useLargePackages,
|
useLargePackages: CloudRunnerOptions.useLargePackages,
|
||||||
useCompressionStrategy: OrchestratorOptions.useCompressionStrategy,
|
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
|
||||||
garbageMaxAge: OrchestratorOptions.garbageMaxAge,
|
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
|
||||||
githubChecks: OrchestratorOptions.githubChecks,
|
githubChecks: CloudRunnerOptions.githubChecks,
|
||||||
asyncWorkflow: OrchestratorOptions.asyncOrchestrator,
|
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
|
||||||
githubCheckId: OrchestratorOptions.githubCheckId,
|
githubCheckId: CloudRunnerOptions.githubCheckId,
|
||||||
finalHooks: OrchestratorOptions.finalHooks,
|
finalHooks: CloudRunnerOptions.finalHooks,
|
||||||
skipLfs: OrchestratorOptions.skipLfs,
|
skipLfs: CloudRunnerOptions.skipLfs,
|
||||||
skipCache: OrchestratorOptions.skipCache,
|
skipCache: CloudRunnerOptions.skipCache,
|
||||||
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
||||||
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
||||||
dockerWorkspacePath: Input.dockerWorkspacePath,
|
dockerWorkspacePath: Input.dockerWorkspacePath,
|
||||||
testSuitePath: Input.testSuitePath,
|
|
||||||
testSuiteEvent: Input.testSuiteEvent,
|
|
||||||
testTaxonomyPath: Input.testTaxonomyPath,
|
|
||||||
testResultFormat: Input.testResultFormat,
|
|
||||||
testResultPath: Input.testResultPath,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+34
-34
@@ -1,14 +1,14 @@
|
|||||||
import { Command } from 'commander-ts';
|
import { Command } from 'commander-ts';
|
||||||
import { BuildParameters, Orchestrator, ImageTag, Input } from '..';
|
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import { ActionYamlReader } from '../input-readers/action-yaml';
|
import { ActionYamlReader } from '../input-readers/action-yaml';
|
||||||
import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
|
||||||
import OrchestratorQueryOverride from '../orchestrator/options/orchestrator-query-override';
|
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
|
||||||
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
||||||
import { Caching } from '../orchestrator/remote-client/caching';
|
import { Caching } from '../cloud-runner/remote-client/caching';
|
||||||
import { LfsHashing } from '../orchestrator/services/utility/lfs-hashing';
|
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
|
||||||
import { RemoteClient } from '../orchestrator/remote-client';
|
import { RemoteClient } from '../cloud-runner/remote-client';
|
||||||
import OrchestratorOptionsReader from '../orchestrator/options/orchestrator-options-reader';
|
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
|
||||||
import GitHub from '../github';
|
import GitHub from '../github';
|
||||||
import { OptionValues } from 'commander';
|
import { OptionValues } from 'commander';
|
||||||
import { InputKey } from '../input';
|
import { InputKey } from '../input';
|
||||||
@@ -36,7 +36,7 @@ export class Cli {
|
|||||||
const program = new Command();
|
const program = new Command();
|
||||||
program.version('0.0.1');
|
program.version('0.0.1');
|
||||||
|
|
||||||
const properties = OrchestratorOptionsReader.GetProperties();
|
const properties = CloudRunnerOptionsReader.GetProperties();
|
||||||
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
||||||
for (const element of properties) {
|
for (const element of properties) {
|
||||||
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
||||||
@@ -62,23 +62,23 @@ export class Cli {
|
|||||||
static async RunCli(): Promise<void> {
|
static async RunCli(): Promise<void> {
|
||||||
GitHub.githubInputEnabled = false;
|
GitHub.githubInputEnabled = false;
|
||||||
if (Cli.options!['populateOverride'] === `true`) {
|
if (Cli.options!['populateOverride'] === `true`) {
|
||||||
await OrchestratorQueryOverride.PopulateQueryOverrideInput();
|
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
|
||||||
}
|
}
|
||||||
if (Cli.options!['logInput']) {
|
if (Cli.options!['logInput']) {
|
||||||
Cli.logInput();
|
Cli.logInput();
|
||||||
}
|
}
|
||||||
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
|
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
|
||||||
OrchestratorLogger.log(`Entrypoint: ${results.key}`);
|
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
|
||||||
Cli.options!.versioning = 'None';
|
Cli.options!.versioning = 'None';
|
||||||
|
|
||||||
Orchestrator.buildParameters = await BuildParameters.create();
|
CloudRunner.buildParameters = await BuildParameters.create();
|
||||||
Orchestrator.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
|
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
|
||||||
OrchestratorLogger.log(`Build Params:
|
CloudRunnerLogger.log(`Build Params:
|
||||||
${JSON.stringify(Orchestrator.buildParameters, undefined, 4)}
|
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
|
||||||
`);
|
`);
|
||||||
Orchestrator.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
|
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
|
||||||
OrchestratorLogger.log(`Locked Workspace: ${Orchestrator.lockedWorkspace}`);
|
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
|
||||||
await Orchestrator.setup(Orchestrator.buildParameters);
|
await CloudRunner.setup(CloudRunner.buildParameters);
|
||||||
|
|
||||||
return await results.target[results.propertyKey](Cli.options);
|
return await results.target[results.propertyKey](Cli.options);
|
||||||
}
|
}
|
||||||
@@ -87,7 +87,7 @@ export class Cli {
|
|||||||
private static logInput() {
|
private static logInput() {
|
||||||
core.info(`\n`);
|
core.info(`\n`);
|
||||||
core.info(`INPUT:`);
|
core.info(`INPUT:`);
|
||||||
const properties = OrchestratorOptionsReader.GetProperties();
|
const properties = CloudRunnerOptionsReader.GetProperties();
|
||||||
for (const element of properties) {
|
for (const element of properties) {
|
||||||
if (
|
if (
|
||||||
element in Input &&
|
element in Input &&
|
||||||
@@ -104,28 +104,28 @@ export class Cli {
|
|||||||
core.info(`\n`);
|
core.info(`\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`cli-build`, `runs a orchestrator build`)
|
@CliFunction(`cli-build`, `runs a cloud runner build`)
|
||||||
public static async CLIBuild(): Promise<string> {
|
public static async CLIBuild(): Promise<string> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
const baseImage = new ImageTag(buildParameter);
|
const baseImage = new ImageTag(buildParameter);
|
||||||
|
|
||||||
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
|
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`async-workflow`, `runs a orchestrator build`)
|
@CliFunction(`async-workflow`, `runs a cloud runner build`)
|
||||||
public static async asyncronousWorkflow(): Promise<string> {
|
public static async asyncronousWorkflow(): Promise<string> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
const baseImage = new ImageTag(buildParameter);
|
const baseImage = new ImageTag(buildParameter);
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
|
|
||||||
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
|
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`checks-update`, `runs a orchestrator build`)
|
@CliFunction(`checks-update`, `runs a cloud runner build`)
|
||||||
public static async checksUpdate() {
|
public static async checksUpdate() {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
|
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
|
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
|
||||||
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
|
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
|
||||||
if (input.mode === `create`) {
|
if (input.mode === `create`) {
|
||||||
@@ -139,18 +139,18 @@ export class Cli {
|
|||||||
public static async GarbageCollect(): Promise<string> {
|
public static async GarbageCollect(): Promise<string> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
|
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
|
|
||||||
return await Orchestrator.Provider.garbageCollect(``, false, 0, false, false);
|
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`list-resources`, `lists active resources`)
|
@CliFunction(`list-resources`, `lists active resources`)
|
||||||
public static async ListResources(): Promise<string[]> {
|
public static async ListResources(): Promise<string[]> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
|
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
const result = await Orchestrator.Provider.listResources();
|
const result = await CloudRunner.Provider.listResources();
|
||||||
OrchestratorLogger.log(JSON.stringify(result, undefined, 4));
|
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4));
|
||||||
|
|
||||||
return result.map((x) => x.Name);
|
return result.map((x) => x.Name);
|
||||||
}
|
}
|
||||||
@@ -159,17 +159,17 @@ export class Cli {
|
|||||||
public static async ListWorfklow(): Promise<string[]> {
|
public static async ListWorfklow(): Promise<string[]> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
|
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
|
|
||||||
return (await Orchestrator.Provider.listWorkflow()).map((x) => x.Name);
|
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`watch`, `follows logs of a running workflow`)
|
@CliFunction(`watch`, `follows logs of a running workflow`)
|
||||||
public static async Watch(): Promise<string> {
|
public static async Watch(): Promise<string> {
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
|
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
|
|
||||||
return await Orchestrator.Provider.watchWorkflow();
|
return await CloudRunner.Provider.watchWorkflow();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
import AwsBuildPlatform from './providers/aws';
|
||||||
|
import { BuildParameters, Input } from '..';
|
||||||
|
import Kubernetes from './providers/k8s';
|
||||||
|
import CloudRunnerLogger from './services/core/cloud-runner-logger';
|
||||||
|
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
|
||||||
|
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
|
||||||
|
import { CloudRunnerError } from './error/cloud-runner-error';
|
||||||
|
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
|
||||||
|
import * as core from '@actions/core';
|
||||||
|
import CloudRunnerSecret from './options/cloud-runner-secret';
|
||||||
|
import { ProviderInterface } from './providers/provider-interface';
|
||||||
|
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
|
||||||
|
import TestCloudRunner from './providers/test';
|
||||||
|
import LocalCloudRunner from './providers/local';
|
||||||
|
import LocalDockerCloudRunner from './providers/docker';
|
||||||
|
import loadProvider from './providers/provider-loader';
|
||||||
|
import GitHub from '../github';
|
||||||
|
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
|
||||||
|
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
|
||||||
|
import CloudRunnerResult from './services/core/cloud-runner-result';
|
||||||
|
import CloudRunnerOptions from './options/cloud-runner-options';
|
||||||
|
|
||||||
|
class CloudRunner {
|
||||||
|
public static Provider: ProviderInterface;
|
||||||
|
public static buildParameters: BuildParameters;
|
||||||
|
private static defaultSecrets: CloudRunnerSecret[];
|
||||||
|
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
|
||||||
|
static lockedWorkspace: string = ``;
|
||||||
|
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
|
||||||
|
public static get isCloudRunnerEnvironment() {
|
||||||
|
return process.env[`GITHUB_ACTIONS`] !== `true`;
|
||||||
|
}
|
||||||
|
public static get isCloudRunnerAsyncEnvironment() {
|
||||||
|
return process.env[`ASYNC_WORKFLOW`] === `true`;
|
||||||
|
}
|
||||||
|
public static async setup(buildParameters: BuildParameters) {
|
||||||
|
CloudRunnerLogger.setup();
|
||||||
|
CloudRunnerLogger.log(`Setting up cloud runner`);
|
||||||
|
CloudRunner.buildParameters = buildParameters;
|
||||||
|
if (CloudRunner.buildParameters.githubCheckId === ``) {
|
||||||
|
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
|
||||||
|
}
|
||||||
|
await CloudRunner.setupSelectedBuildPlatform();
|
||||||
|
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
|
||||||
|
CloudRunner.cloudRunnerEnvironmentVariables =
|
||||||
|
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters);
|
||||||
|
if (GitHub.githubInputEnabled) {
|
||||||
|
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
|
||||||
|
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
|
||||||
|
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
|
||||||
|
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
|
||||||
|
}
|
||||||
|
for (const element of buildParameterPropertyNames) {
|
||||||
|
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
|
||||||
|
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
|
||||||
|
}
|
||||||
|
core.setOutput(
|
||||||
|
Input.ToEnvVarFormat(`buildArtifact`),
|
||||||
|
`build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||||
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
FollowLogStreamService.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async setupSelectedBuildPlatform() {
|
||||||
|
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
|
||||||
|
|
||||||
|
// Detect LocalStack endpoints and reroute AWS provider to local-docker for CI tests that only need S3
|
||||||
|
const endpointsToCheck = [
|
||||||
|
process.env.AWS_ENDPOINT,
|
||||||
|
process.env.AWS_S3_ENDPOINT,
|
||||||
|
process.env.AWS_CLOUD_FORMATION_ENDPOINT,
|
||||||
|
process.env.AWS_ECS_ENDPOINT,
|
||||||
|
process.env.AWS_KINESIS_ENDPOINT,
|
||||||
|
process.env.AWS_CLOUD_WATCH_LOGS_ENDPOINT,
|
||||||
|
CloudRunnerOptions.awsEndpoint,
|
||||||
|
CloudRunnerOptions.awsS3Endpoint,
|
||||||
|
CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||||
|
CloudRunnerOptions.awsEcsEndpoint,
|
||||||
|
CloudRunnerOptions.awsKinesisEndpoint,
|
||||||
|
CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||||
|
]
|
||||||
|
.filter((x) => typeof x === 'string')
|
||||||
|
.join(' ');
|
||||||
|
const isLocalStack = /localstack|localhost|127\.0\.0\.1/i.test(endpointsToCheck);
|
||||||
|
let provider = CloudRunner.buildParameters.providerStrategy;
|
||||||
|
if (provider === 'aws' && isLocalStack) {
|
||||||
|
CloudRunnerLogger.log('LocalStack endpoints detected; routing provider to local-docker for this run');
|
||||||
|
provider = 'local-docker';
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (provider) {
|
||||||
|
case 'k8s':
|
||||||
|
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
|
||||||
|
break;
|
||||||
|
case 'aws':
|
||||||
|
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
|
||||||
|
break;
|
||||||
|
case 'test':
|
||||||
|
CloudRunner.Provider = new TestCloudRunner();
|
||||||
|
break;
|
||||||
|
case 'local-docker':
|
||||||
|
CloudRunner.Provider = new LocalDockerCloudRunner();
|
||||||
|
break;
|
||||||
|
case 'local-system':
|
||||||
|
CloudRunner.Provider = new LocalCloudRunner();
|
||||||
|
break;
|
||||||
|
case 'local':
|
||||||
|
CloudRunner.Provider = new LocalCloudRunner();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
// Try to load provider using the dynamic loader for unknown providers
|
||||||
|
try {
|
||||||
|
CloudRunner.Provider = await loadProvider(provider, CloudRunner.buildParameters);
|
||||||
|
} catch (error: any) {
|
||||||
|
CloudRunnerLogger.log(`Failed to load provider '${provider}' using dynamic loader: ${error.message}`);
|
||||||
|
CloudRunnerLogger.log('Falling back to local provider...');
|
||||||
|
CloudRunner.Provider = new LocalCloudRunner();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async run(buildParameters: BuildParameters, baseImage: string) {
|
||||||
|
if (baseImage.includes(`undefined`)) {
|
||||||
|
throw new Error(`baseImage is undefined`);
|
||||||
|
}
|
||||||
|
await CloudRunner.setup(buildParameters);
|
||||||
|
await CloudRunner.Provider.setupWorkflow(
|
||||||
|
CloudRunner.buildParameters.buildGuid,
|
||||||
|
CloudRunner.buildParameters,
|
||||||
|
CloudRunner.buildParameters.branch,
|
||||||
|
CloudRunner.defaultSecrets,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
if (buildParameters.maxRetainedWorkspaces > 0) {
|
||||||
|
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
|
||||||
|
|
||||||
|
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
|
||||||
|
CloudRunner.lockedWorkspace,
|
||||||
|
CloudRunner.buildParameters.buildGuid,
|
||||||
|
CloudRunner.buildParameters,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result) {
|
||||||
|
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`);
|
||||||
|
CloudRunner.cloudRunnerEnvironmentVariables = [
|
||||||
|
...CloudRunner.cloudRunnerEnvironmentVariables,
|
||||||
|
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
|
||||||
|
buildParameters.maxRetainedWorkspaces = 0;
|
||||||
|
CloudRunner.lockedWorkspace = ``;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await CloudRunner.updateStatusWithBuildParameters();
|
||||||
|
const output = await new WorkflowCompositionRoot().run(
|
||||||
|
new CloudRunnerStepParameters(
|
||||||
|
baseImage,
|
||||||
|
CloudRunner.cloudRunnerEnvironmentVariables,
|
||||||
|
CloudRunner.defaultSecrets,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await CloudRunner.Provider.cleanupWorkflow(
|
||||||
|
CloudRunner.buildParameters,
|
||||||
|
CloudRunner.buildParameters.branch,
|
||||||
|
CloudRunner.defaultSecrets,
|
||||||
|
);
|
||||||
|
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||||
|
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) {
|
||||||
|
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
||||||
|
const workspace = CloudRunner.lockedWorkspace || ``;
|
||||||
|
await SharedWorkspaceLocking.ReleaseWorkspace(
|
||||||
|
workspace,
|
||||||
|
CloudRunner.buildParameters.buildGuid,
|
||||||
|
CloudRunner.buildParameters,
|
||||||
|
);
|
||||||
|
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
|
||||||
|
if (isLocked) {
|
||||||
|
throw new Error(
|
||||||
|
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
|
||||||
|
workspace,
|
||||||
|
buildParameters,
|
||||||
|
)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
CloudRunner.lockedWorkspace = ``;
|
||||||
|
}
|
||||||
|
|
||||||
|
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
|
||||||
|
|
||||||
|
if (buildParameters.constantGarbageCollection) {
|
||||||
|
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new CloudRunnerResult(buildParameters, output, true, true, false);
|
||||||
|
} catch (error: any) {
|
||||||
|
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
|
||||||
|
await GitHub.updateGitHubCheck(
|
||||||
|
CloudRunner.buildParameters.buildGuid,
|
||||||
|
`Failed - Error ${error?.message || error}`,
|
||||||
|
`failure`,
|
||||||
|
`completed`,
|
||||||
|
);
|
||||||
|
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||||
|
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async updateStatusWithBuildParameters() {
|
||||||
|
const content = { ...CloudRunner.buildParameters };
|
||||||
|
content.gitPrivateToken = ``;
|
||||||
|
content.unitySerial = ``;
|
||||||
|
content.unityEmail = ``;
|
||||||
|
content.unityPassword = ``;
|
||||||
|
const jsonContent = JSON.stringify(content, undefined, 4);
|
||||||
|
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export default CloudRunner;
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
import * as core from '@actions/core';
|
||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
||||||
|
import BuildParameters from '../../build-parameters';
|
||||||
|
|
||||||
|
export class CloudRunnerError {
|
||||||
|
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
|
||||||
|
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
|
||||||
|
core.setFailed('Cloud Runner failed');
|
||||||
|
if (CloudRunner.Provider !== undefined) {
|
||||||
|
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
class CloudRunnerConstants {
|
||||||
|
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
|
||||||
|
}
|
||||||
|
export default CloudRunnerConstants;
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
class CloudRunnerEnvironmentVariable {
|
||||||
|
public name!: string;
|
||||||
|
public value!: string;
|
||||||
|
}
|
||||||
|
export default CloudRunnerEnvironmentVariable;
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
import CloudRunnerOptions from './cloud-runner-options';
|
||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import BuildParameters from '../../build-parameters';
|
||||||
|
|
||||||
|
export class CloudRunnerFolders {
|
||||||
|
public static readonly repositoryFolder = 'repo';
|
||||||
|
|
||||||
|
public static ToLinuxFolder(folder: string) {
|
||||||
|
return folder.replace(/\\/g, `/`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||||
|
|
||||||
|
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
|
||||||
|
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
|
||||||
|
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
|
||||||
|
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get cacheFolderForAllFull(): string {
|
||||||
|
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get cacheFolderForCacheKeyFull(): string {
|
||||||
|
return path.join(
|
||||||
|
'/',
|
||||||
|
CloudRunnerFolders.buildVolumeFolder,
|
||||||
|
CloudRunnerFolders.cacheFolder,
|
||||||
|
CloudRunner.buildParameters.cacheKey,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get builderPathAbsolute(): string {
|
||||||
|
return path.join(
|
||||||
|
CloudRunnerOptions.useSharedBuilder
|
||||||
|
? `/${CloudRunnerFolders.buildVolumeFolder}`
|
||||||
|
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||||
|
`builder`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get repoPathAbsolute(): string {
|
||||||
|
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get projectPathAbsolute(): string {
|
||||||
|
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get libraryFolderAbsolute(): string {
|
||||||
|
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get projectBuildFolderAbsolute(): string {
|
||||||
|
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get lfsFolderAbsolute(): string {
|
||||||
|
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get purgeRemoteCaching(): boolean {
|
||||||
|
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get lfsCacheFolderFull() {
|
||||||
|
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get libraryCacheFolderFull() {
|
||||||
|
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get unityBuilderRepoUrl(): string {
|
||||||
|
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get targetBuildRepoUrl(): string {
|
||||||
|
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get buildVolumeFolder() {
|
||||||
|
return 'data';
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get cacheFolder() {
|
||||||
|
return 'cache';
|
||||||
|
}
|
||||||
|
}
|
||||||
+4
-4
@@ -1,11 +1,11 @@
|
|||||||
import { customAlphabet } from 'nanoid';
|
import { customAlphabet } from 'nanoid';
|
||||||
import OrchestratorConstants from './orchestrator-constants';
|
import CloudRunnerConstants from './cloud-runner-constants';
|
||||||
|
|
||||||
class OrchestratorNamespace {
|
class CloudRunnerNamespace {
|
||||||
static generateGuid(runNumber: string | number, platform: string) {
|
static generateGuid(runNumber: string | number, platform: string) {
|
||||||
const nanoid = customAlphabet(OrchestratorConstants.alphabet, 4);
|
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
|
||||||
|
|
||||||
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
|
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default OrchestratorNamespace;
|
export default CloudRunnerNamespace;
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import Input from '../../input';
|
||||||
|
import CloudRunnerOptions from './cloud-runner-options';
|
||||||
|
|
||||||
|
class CloudRunnerOptionsReader {
|
||||||
|
static GetProperties() {
|
||||||
|
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default CloudRunnerOptionsReader;
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
import { Cli } from '../../cli/cli';
|
||||||
|
import CloudRunnerQueryOverride from './cloud-runner-query-override';
|
||||||
|
import GitHub from '../../github';
|
||||||
|
import * as core from '@actions/core';
|
||||||
|
|
||||||
|
class CloudRunnerOptions {
|
||||||
|
// ### ### ###
|
||||||
|
// Input Handling
|
||||||
|
// ### ### ###
|
||||||
|
public static getInput(query: string): string | undefined {
|
||||||
|
if (GitHub.githubInputEnabled) {
|
||||||
|
const coreInput = core.getInput(query);
|
||||||
|
if (coreInput && coreInput !== '') {
|
||||||
|
return coreInput;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
|
||||||
|
|
||||||
|
// Query input sources
|
||||||
|
if (Cli.query(query, alternativeQuery)) {
|
||||||
|
return Cli.query(query, alternativeQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
|
||||||
|
return CloudRunnerQueryOverride.query(query, alternativeQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.env[query] !== undefined) {
|
||||||
|
return process.env[query];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
|
||||||
|
return process.env[alternativeQuery];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ToEnvVarFormat(input: string): string {
|
||||||
|
if (input.toUpperCase() === input) {
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
return input
|
||||||
|
.replace(/([A-Z])/g, ' $1')
|
||||||
|
.trim()
|
||||||
|
.toUpperCase()
|
||||||
|
.replace(/ /g, '_');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Provider parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get region(): string {
|
||||||
|
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// GitHub parameters
|
||||||
|
// ### ### ###
|
||||||
|
static get githubChecks(): boolean {
|
||||||
|
const value = CloudRunnerOptions.getInput('githubChecks');
|
||||||
|
|
||||||
|
return value === `true` || false;
|
||||||
|
}
|
||||||
|
static get githubCheckId(): string {
|
||||||
|
return CloudRunnerOptions.getInput('githubCheckId') || ``;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get githubOwner(): string {
|
||||||
|
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get githubRepoName(): string {
|
||||||
|
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get finalHooks(): string[] {
|
||||||
|
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Git syncronization parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get githubRepo(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
|
||||||
|
}
|
||||||
|
static get branch(): string {
|
||||||
|
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
|
||||||
|
return (
|
||||||
|
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
|
||||||
|
);
|
||||||
|
} else if (CloudRunnerOptions.getInput('branch')) {
|
||||||
|
return CloudRunnerOptions.getInput('branch') || ``;
|
||||||
|
} else {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Cloud Runner parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get buildPlatform(): string {
|
||||||
|
const input = CloudRunnerOptions.getInput('buildPlatform');
|
||||||
|
if (input && input !== '') {
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
if (CloudRunnerOptions.providerStrategy !== 'local') {
|
||||||
|
return 'linux';
|
||||||
|
}
|
||||||
|
|
||||||
|
return process.platform;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get cloudRunnerBranch(): string {
|
||||||
|
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get providerStrategy(): string {
|
||||||
|
const provider =
|
||||||
|
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
|
||||||
|
if (Cli.isCliMode) {
|
||||||
|
return provider || 'aws';
|
||||||
|
}
|
||||||
|
|
||||||
|
return provider || 'local';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get containerCpu(): string {
|
||||||
|
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get containerMemory(): string {
|
||||||
|
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get customJob(): string {
|
||||||
|
return CloudRunnerOptions.getInput('customJob') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Custom commands from files parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get containerHookFiles(): string[] {
|
||||||
|
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
|
||||||
|
}
|
||||||
|
|
||||||
|
static get commandHookFiles(): string[] {
|
||||||
|
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Custom commands from yaml parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get commandHooks(): string {
|
||||||
|
return CloudRunnerOptions.getInput('commandHooks') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get postBuildContainerHooks(): string {
|
||||||
|
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get preBuildContainerHooks(): string {
|
||||||
|
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Input override handling
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get pullInputList(): string[] {
|
||||||
|
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
|
||||||
|
}
|
||||||
|
|
||||||
|
static get inputPullCommand(): string {
|
||||||
|
const value = CloudRunnerOptions.getInput('inputPullCommand');
|
||||||
|
|
||||||
|
if (value === 'gcp-secret-manager') {
|
||||||
|
return 'gcloud secrets versions access 1 --secret="{0}"';
|
||||||
|
} else if (value === 'aws-secret-manager') {
|
||||||
|
return 'aws secretsmanager get-secret-value --secret-id {0}';
|
||||||
|
}
|
||||||
|
|
||||||
|
return value || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Aws
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get awsStackName() {
|
||||||
|
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsEndpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsEndpoint');
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsCloudFormationEndpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsCloudFormationEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsEcsEndpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsEcsEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsKinesisEndpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsKinesisEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsCloudWatchLogsEndpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsCloudWatchLogsEndpoint') || CloudRunnerOptions.awsEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
static get awsS3Endpoint(): string | undefined {
|
||||||
|
return CloudRunnerOptions.getInput('awsS3Endpoint') || CloudRunnerOptions.awsEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Storage
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get storageProvider(): string {
|
||||||
|
return CloudRunnerOptions.getInput('storageProvider') || 's3';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get rcloneRemote(): string {
|
||||||
|
return CloudRunnerOptions.getInput('rcloneRemote') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// K8s
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get kubeConfig(): string {
|
||||||
|
return CloudRunnerOptions.getInput('kubeConfig') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get kubeVolume(): string {
|
||||||
|
return CloudRunnerOptions.getInput('kubeVolume') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get kubeVolumeSize(): string {
|
||||||
|
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
|
||||||
|
}
|
||||||
|
|
||||||
|
static get kubeStorageClass(): string {
|
||||||
|
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Caching
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get cacheKey(): string {
|
||||||
|
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Utility Parameters
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get cloudRunnerDebug(): boolean {
|
||||||
|
return (
|
||||||
|
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
|
||||||
|
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
|
||||||
|
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
|
||||||
|
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
|
||||||
|
false
|
||||||
|
);
|
||||||
|
}
|
||||||
|
static get skipLfs(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
|
||||||
|
}
|
||||||
|
static get skipCache(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get asyncCloudRunner(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get useLargePackages(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get useSharedBuilder(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get useCompressionStrategy(): boolean {
|
||||||
|
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static get useCleanupCron(): boolean {
|
||||||
|
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Retained Workspace
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
public static get maxRetainedWorkspaces(): string {
|
||||||
|
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### ### ###
|
||||||
|
// Garbage Collection
|
||||||
|
// ### ### ###
|
||||||
|
|
||||||
|
static get garbageMaxAge(): number {
|
||||||
|
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default CloudRunnerOptions;
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import Input from '../../input';
|
||||||
|
import { GenericInputReader } from '../../input-readers/generic-input-reader';
|
||||||
|
import CloudRunnerOptions from './cloud-runner-options';
|
||||||
|
|
||||||
|
const formatFunction = (value: string, arguments_: any[]) => {
|
||||||
|
for (const element of arguments_) {
|
||||||
|
value = value.replace(`{${element.key}}`, element.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
class CloudRunnerQueryOverride {
|
||||||
|
static queryOverrides: { [key: string]: string } | undefined;
|
||||||
|
|
||||||
|
// TODO accept premade secret sources or custom secret source definition yamls
|
||||||
|
|
||||||
|
public static query(key: string, alternativeKey: string) {
|
||||||
|
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
|
||||||
|
return CloudRunnerQueryOverride.queryOverrides[key];
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
CloudRunnerQueryOverride.queryOverrides &&
|
||||||
|
alternativeKey &&
|
||||||
|
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
|
||||||
|
) {
|
||||||
|
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static shouldUseOverride(query: string) {
|
||||||
|
if (CloudRunnerOptions.inputPullCommand !== '') {
|
||||||
|
if (CloudRunnerOptions.pullInputList.length > 0) {
|
||||||
|
const doesInclude =
|
||||||
|
CloudRunnerOptions.pullInputList.includes(query) ||
|
||||||
|
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
|
||||||
|
|
||||||
|
return doesInclude ? true : false;
|
||||||
|
} else {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async queryOverride(query: string) {
|
||||||
|
if (!this.shouldUseOverride(query)) {
|
||||||
|
throw new Error(`Should not be trying to run override query on ${query}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await GenericInputReader.Run(
|
||||||
|
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async PopulateQueryOverrideInput() {
|
||||||
|
const queries = CloudRunnerOptions.pullInputList;
|
||||||
|
CloudRunnerQueryOverride.queryOverrides = {};
|
||||||
|
for (const element of queries) {
|
||||||
|
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
|
||||||
|
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export default CloudRunnerQueryOverride;
|
||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
class OrchestratorSecret {
|
class CloudRunnerSecret {
|
||||||
public ParameterKey!: string;
|
public ParameterKey!: string;
|
||||||
public EnvironmentVariable!: string;
|
public EnvironmentVariable!: string;
|
||||||
public ParameterValue!: string;
|
public ParameterValue!: string;
|
||||||
}
|
}
|
||||||
export default OrchestratorSecret;
|
export default CloudRunnerSecret;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
export class CloudRunnerStatics {
|
||||||
|
public static readonly logPrefix = `Cloud-Runner`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
|
||||||
|
import CloudRunnerSecret from './cloud-runner-secret';
|
||||||
|
|
||||||
|
export class CloudRunnerStepParameters {
|
||||||
|
public image: string;
|
||||||
|
public environment: CloudRunnerEnvironmentVariable[];
|
||||||
|
public secrets: CloudRunnerSecret[];
|
||||||
|
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
|
||||||
|
this.image = image;
|
||||||
|
this.environment = environmentVariables;
|
||||||
|
this.secrets = secrets;
|
||||||
|
}
|
||||||
|
}
|
||||||
+48
-20
@@ -1,14 +1,16 @@
|
|||||||
# Provider Loader Dynamic Imports
|
# Provider Loader Dynamic Imports
|
||||||
|
|
||||||
|
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub
|
||||||
|
repositories, and NPM packages.
|
||||||
|
|
||||||
## What is a Provider?
|
## What is a Provider?
|
||||||
|
|
||||||
A **provider** is a pluggable backend that Orchestrator uses to run builds and workflows. Examples include **AWS**, **Kubernetes**, or local execution. Each provider implements the [ProviderInterface](https://github.com/game-ci/unity-builder/blob/main/src/model/orchestrator/providers/provider-interface.ts), which defines the common lifecycle methods (setup, run, cleanup, garbage collection, etc.).
|
A provider is a pluggable backend that Cloud Runner uses to run builds and workflows. Examples include AWS, Kubernetes,
|
||||||
|
or local execution. Each provider implements the `ProviderInterface`, which defines the common lifecycle methods (setup,
|
||||||
|
run, cleanup, garbage collection, etc.).
|
||||||
|
|
||||||
This abstraction makes Orchestrator flexible: you can switch execution environments or add your own provider (via npm package, GitHub repo, or local path) without changing the rest of your pipeline.
|
This abstraction makes Cloud Runner flexible: you can switch execution environments or add your own provider (via npm
|
||||||
|
package, GitHub repo, or local path) without changing the rest of your pipeline.
|
||||||
## Dynamic Provider Loading
|
|
||||||
|
|
||||||
The provider loader now supports dynamic loading of providers from multiple sources including local file paths, GitHub repositories, and NPM packages.
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@@ -45,21 +47,18 @@ const absoluteProvider = await ProviderLoader.loadProvider('/path/to/provider',
|
|||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Load from GitHub URL
|
// Load from GitHub URL
|
||||||
const githubProvider = await ProviderLoader.loadProvider(
|
const githubProvider = await ProviderLoader.loadProvider('https://github.com/user/my-provider', buildParameters);
|
||||||
'https://github.com/user/my-provider',
|
|
||||||
buildParameters
|
|
||||||
);
|
|
||||||
|
|
||||||
// Load from specific branch
|
// Load from specific branch
|
||||||
const branchProvider = await ProviderLoader.loadProvider(
|
const branchProvider = await ProviderLoader.loadProvider(
|
||||||
'https://github.com/user/my-provider/tree/develop',
|
'https://github.com/user/my-provider/tree/develop',
|
||||||
buildParameters
|
buildParameters,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Load from specific path in repository
|
// Load from specific path in repository
|
||||||
const pathProvider = await ProviderLoader.loadProvider(
|
const pathProvider = await ProviderLoader.loadProvider(
|
||||||
'https://github.com/user/my-provider/tree/main/src/providers',
|
'https://github.com/user/my-provider/tree/main/src/providers',
|
||||||
buildParameters
|
buildParameters,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Shorthand notation
|
// Shorthand notation
|
||||||
@@ -84,8 +83,20 @@ All providers must implement the `ProviderInterface`:
|
|||||||
```typescript
|
```typescript
|
||||||
interface ProviderInterface {
|
interface ProviderInterface {
|
||||||
cleanupWorkflow(): Promise<void>;
|
cleanupWorkflow(): Promise<void>;
|
||||||
setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void>;
|
setupWorkflow(
|
||||||
runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string>;
|
buildGuid: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
branchName: string,
|
||||||
|
defaultSecretsArray: any[],
|
||||||
|
): Promise<void>;
|
||||||
|
runTaskInWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
task: string,
|
||||||
|
workingDirectory: string,
|
||||||
|
buildVolumeFolder: string,
|
||||||
|
environmentVariables: any[],
|
||||||
|
secrets: any[],
|
||||||
|
): Promise<string>;
|
||||||
garbageCollect(): Promise<void>;
|
garbageCollect(): Promise<void>;
|
||||||
listResources(): Promise<ProviderResource[]>;
|
listResources(): Promise<ProviderResource[]>;
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||||
@@ -107,11 +118,23 @@ export default class MyProvider implements ProviderInterface {
|
|||||||
// Cleanup logic
|
// Cleanup logic
|
||||||
}
|
}
|
||||||
|
|
||||||
async setupWorkflow(buildGuid: string, buildParameters: BuildParameters, branchName: string, defaultSecretsArray: any[]): Promise<void> {
|
async setupWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
branchName: string,
|
||||||
|
defaultSecretsArray: any[],
|
||||||
|
): Promise<void> {
|
||||||
// Setup logic
|
// Setup logic
|
||||||
}
|
}
|
||||||
|
|
||||||
async runTaskInWorkflow(buildGuid: string, task: string, workingDirectory: string, buildVolumeFolder: string, environmentVariables: any[], secrets: any[]): Promise<string> {
|
async runTaskInWorkflow(
|
||||||
|
buildGuid: string,
|
||||||
|
task: string,
|
||||||
|
workingDirectory: string,
|
||||||
|
buildVolumeFolder: string,
|
||||||
|
environmentVariables: any[],
|
||||||
|
secrets: any[],
|
||||||
|
): Promise<string> {
|
||||||
// Task execution logic
|
// Task execution logic
|
||||||
return 'Task completed';
|
return 'Task completed';
|
||||||
}
|
}
|
||||||
@@ -167,6 +190,7 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
|||||||
## Supported URL Formats
|
## Supported URL Formats
|
||||||
|
|
||||||
### GitHub URLs
|
### GitHub URLs
|
||||||
|
|
||||||
- `https://github.com/user/repo`
|
- `https://github.com/user/repo`
|
||||||
- `https://github.com/user/repo.git`
|
- `https://github.com/user/repo.git`
|
||||||
- `https://github.com/user/repo/tree/branch`
|
- `https://github.com/user/repo/tree/branch`
|
||||||
@@ -174,23 +198,27 @@ console.log(providers); // ['aws', 'k8s', 'test', 'local-docker', 'local-system'
|
|||||||
- `git@github.com:user/repo.git`
|
- `git@github.com:user/repo.git`
|
||||||
|
|
||||||
### Shorthand GitHub References
|
### Shorthand GitHub References
|
||||||
|
|
||||||
- `user/repo`
|
- `user/repo`
|
||||||
- `user/repo@branch`
|
- `user/repo@branch`
|
||||||
- `user/repo@branch/path/to/provider`
|
- `user/repo@branch/path/to/provider`
|
||||||
|
|
||||||
### Local Paths
|
### Local Paths
|
||||||
|
|
||||||
- `./relative/path`
|
- `./relative/path`
|
||||||
- `../relative/path`
|
- `../relative/path`
|
||||||
- `/absolute/path`
|
- `/absolute/path`
|
||||||
- `C:\\path\\to\\provider` (Windows)
|
- `C:\\path\\to\\provider` (Windows)
|
||||||
|
|
||||||
### NPM Packages
|
### NPM Packages
|
||||||
|
|
||||||
- `package-name`
|
- `package-name`
|
||||||
- `@scope/package-name`
|
- `@scope/package-name`
|
||||||
|
|
||||||
## Caching
|
## Caching
|
||||||
|
|
||||||
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the repository owner, name, and branch. This ensures that:
|
GitHub repositories are automatically cached in the `.provider-cache` directory. The cache key is generated based on the
|
||||||
|
repository owner, name, and branch. This ensures that:
|
||||||
|
|
||||||
1. Repositories are only cloned once
|
1. Repositories are only cloned once
|
||||||
2. Updates are checked and applied automatically
|
2. Updates are checked and applied automatically
|
||||||
@@ -215,7 +243,7 @@ The provider loader can be configured through environment variables:
|
|||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
1. **Use specific branches or tags**: Always specify the branch or specific tag when loading from GitHub
|
1. **Use specific branches or versions**: Always specify the branch or specific tag when loading from GitHub
|
||||||
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
2. **Implement proper error handling**: Wrap provider loading in try-catch blocks
|
||||||
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
3. **Clean up regularly**: Use the cleanup utility to manage cache size
|
||||||
4. **Test locally first**: Test providers locally before deploying
|
4. **Test locally first**: Test providers locally before deploying
|
||||||
+16
-59
@@ -1,18 +1,14 @@
|
|||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import {
|
import {
|
||||||
CloudFormation,
|
CloudFormation,
|
||||||
CreateStackCommand,
|
CreateStackCommand,
|
||||||
// eslint-disable-next-line import/named
|
|
||||||
CreateStackCommandInput,
|
CreateStackCommandInput,
|
||||||
DescribeStacksCommand,
|
DescribeStacksCommand,
|
||||||
// eslint-disable-next-line import/named
|
|
||||||
DescribeStacksCommandInput,
|
DescribeStacksCommandInput,
|
||||||
ListStacksCommand,
|
ListStacksCommand,
|
||||||
// eslint-disable-next-line import/named
|
|
||||||
Parameter,
|
Parameter,
|
||||||
UpdateStackCommand,
|
UpdateStackCommand,
|
||||||
// eslint-disable-next-line import/named
|
|
||||||
UpdateStackCommandInput,
|
UpdateStackCommandInput,
|
||||||
waitUntilStackCreateComplete,
|
waitUntilStackCreateComplete,
|
||||||
waitUntilStackUpdateComplete,
|
waitUntilStackUpdateComplete,
|
||||||
@@ -20,17 +16,6 @@ import {
|
|||||||
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
|
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
|
||||||
import crypto from 'node:crypto';
|
import crypto from 'node:crypto';
|
||||||
|
|
||||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
|
||||||
|
|
||||||
function getStackWaitTime(): number {
|
|
||||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
|
||||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
|
||||||
return overrideValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class AWSBaseStack {
|
export class AWSBaseStack {
|
||||||
constructor(baseStackName: string) {
|
constructor(baseStackName: string) {
|
||||||
this.baseStackName = baseStackName;
|
this.baseStackName = baseStackName;
|
||||||
@@ -39,7 +24,6 @@ export class AWSBaseStack {
|
|||||||
|
|
||||||
async setupBaseStack(CF: CloudFormation) {
|
async setupBaseStack(CF: CloudFormation) {
|
||||||
const baseStackName = this.baseStackName;
|
const baseStackName = this.baseStackName;
|
||||||
const stackWaitTimeSeconds = getStackWaitTime();
|
|
||||||
|
|
||||||
const baseStack = BaseStackFormation.formation;
|
const baseStack = BaseStackFormation.formation;
|
||||||
|
|
||||||
@@ -70,39 +54,18 @@ export class AWSBaseStack {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const stacks = await CF.send(
|
const stacks = await CF.send(
|
||||||
new ListStacksCommand({
|
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
|
||||||
StackStatusFilter: [
|
|
||||||
'CREATE_IN_PROGRESS',
|
|
||||||
'UPDATE_IN_PROGRESS',
|
|
||||||
'UPDATE_COMPLETE',
|
|
||||||
'CREATE_COMPLETE',
|
|
||||||
'ROLLBACK_COMPLETE',
|
|
||||||
],
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
|
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
|
||||||
const stackExists: boolean = stackNames.includes(baseStackName);
|
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
|
||||||
const describeStack = async () => {
|
const describeStack = async () => {
|
||||||
return await CF.send(new DescribeStacksCommand(describeStackInput));
|
return await CF.send(new DescribeStacksCommand(describeStackInput));
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
if (!stackExists) {
|
if (!stackExists) {
|
||||||
OrchestratorLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
||||||
let created = false;
|
await CF.send(new CreateStackCommand(createStackInput));
|
||||||
try {
|
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
|
||||||
await CF.send(new CreateStackCommand(createStackInput));
|
|
||||||
created = true;
|
|
||||||
} catch (error: any) {
|
|
||||||
const message = `${error?.name ?? ''} ${error?.message ?? ''}`;
|
|
||||||
if (message.includes('AlreadyExistsException')) {
|
|
||||||
OrchestratorLogger.log(`Base stack already exists, continuing with describe`);
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (created) {
|
|
||||||
OrchestratorLogger.log(`created stack (version: ${parametersHash})`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
const CFState = await describeStack();
|
const CFState = await describeStack();
|
||||||
let stack = CFState.Stacks?.[0];
|
let stack = CFState.Stacks?.[0];
|
||||||
@@ -112,35 +75,32 @@ export class AWSBaseStack {
|
|||||||
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
|
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
|
||||||
|
|
||||||
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
|
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
|
||||||
OrchestratorLogger.log(
|
|
||||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`,
|
|
||||||
);
|
|
||||||
await waitUntilStackCreateComplete(
|
await waitUntilStackCreateComplete(
|
||||||
{
|
{
|
||||||
client: CF,
|
client: CF,
|
||||||
maxWaitTime: stackWaitTimeSeconds,
|
maxWaitTime: 200,
|
||||||
},
|
},
|
||||||
describeStackInput,
|
describeStackInput,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (stackExists) {
|
if (stackExists) {
|
||||||
OrchestratorLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
||||||
if (parametersHash !== stackVersion) {
|
if (parametersHash !== stackVersion) {
|
||||||
OrchestratorLogger.log(`Attempting update of base stack`);
|
CloudRunnerLogger.log(`Attempting update of base stack`);
|
||||||
try {
|
try {
|
||||||
await CF.send(new UpdateStackCommand(updateInput));
|
await CF.send(new UpdateStackCommand(updateInput));
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
if (error['message'].includes('No updates are to be performed')) {
|
if (error['message'].includes('No updates are to be performed')) {
|
||||||
OrchestratorLogger.log(`No updates are to be performed`);
|
CloudRunnerLogger.log(`No updates are to be performed`);
|
||||||
} else {
|
} else {
|
||||||
OrchestratorLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
||||||
OrchestratorLogger.log(error['message']);
|
CloudRunnerLogger.log(error['message']);
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`Continuing...`);
|
CloudRunnerLogger.log(`Continuing...`);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
OrchestratorLogger.log(`No update required`);
|
CloudRunnerLogger.log(`No update required`);
|
||||||
}
|
}
|
||||||
stack = (await describeStack()).Stacks?.[0];
|
stack = (await describeStack()).Stacks?.[0];
|
||||||
if (!stack) {
|
if (!stack) {
|
||||||
@@ -149,19 +109,16 @@ export class AWSBaseStack {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
|
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
|
||||||
OrchestratorLogger.log(
|
|
||||||
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`,
|
|
||||||
);
|
|
||||||
await waitUntilStackUpdateComplete(
|
await waitUntilStackUpdateComplete(
|
||||||
{
|
{
|
||||||
client: CF,
|
client: CF,
|
||||||
maxWaitTime: stackWaitTimeSeconds,
|
maxWaitTime: 200,
|
||||||
},
|
},
|
||||||
describeStackInput,
|
describeStackInput,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log('base stack is now ready');
|
CloudRunnerLogger.log('base stack is now ready');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.error(JSON.stringify(await describeStack(), undefined, 4));
|
core.error(JSON.stringify(await describeStack(), undefined, 4));
|
||||||
throw error;
|
throw error;
|
||||||
+6
-28
@@ -4,7 +4,7 @@ import { Kinesis } from '@aws-sdk/client-kinesis';
|
|||||||
import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs';
|
import { CloudWatchLogs } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
import { S3 } from '@aws-sdk/client-s3';
|
import { S3 } from '@aws-sdk/client-s3';
|
||||||
import { Input } from '../../..';
|
import { Input } from '../../..';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
|
|
||||||
export class AwsClientFactory {
|
export class AwsClientFactory {
|
||||||
private static cloudFormation: CloudFormation;
|
private static cloudFormation: CloudFormation;
|
||||||
@@ -13,29 +13,11 @@ export class AwsClientFactory {
|
|||||||
private static cloudWatchLogs: CloudWatchLogs;
|
private static cloudWatchLogs: CloudWatchLogs;
|
||||||
private static s3: S3;
|
private static s3: S3;
|
||||||
|
|
||||||
private static getCredentials() {
|
|
||||||
// Explicitly provide credentials from environment variables for LocalStack compatibility
|
|
||||||
// LocalStack accepts any credentials, but the AWS SDK needs them to be explicitly set
|
|
||||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
|
||||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
|
||||||
|
|
||||||
if (accessKeyId && secretAccessKey) {
|
|
||||||
return {
|
|
||||||
accessKeyId,
|
|
||||||
secretAccessKey,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return undefined to let AWS SDK use default credential chain
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
static getCloudFormation(): CloudFormation {
|
static getCloudFormation(): CloudFormation {
|
||||||
if (!this.cloudFormation) {
|
if (!this.cloudFormation) {
|
||||||
this.cloudFormation = new CloudFormation({
|
this.cloudFormation = new CloudFormation({
|
||||||
region: Input.region,
|
region: Input.region,
|
||||||
endpoint: OrchestratorOptions.awsCloudFormationEndpoint,
|
endpoint: CloudRunnerOptions.awsCloudFormationEndpoint,
|
||||||
credentials: AwsClientFactory.getCredentials(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,8 +28,7 @@ export class AwsClientFactory {
|
|||||||
if (!this.ecs) {
|
if (!this.ecs) {
|
||||||
this.ecs = new ECS({
|
this.ecs = new ECS({
|
||||||
region: Input.region,
|
region: Input.region,
|
||||||
endpoint: OrchestratorOptions.awsEcsEndpoint,
|
endpoint: CloudRunnerOptions.awsEcsEndpoint,
|
||||||
credentials: AwsClientFactory.getCredentials(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,8 +39,7 @@ export class AwsClientFactory {
|
|||||||
if (!this.kinesis) {
|
if (!this.kinesis) {
|
||||||
this.kinesis = new Kinesis({
|
this.kinesis = new Kinesis({
|
||||||
region: Input.region,
|
region: Input.region,
|
||||||
endpoint: OrchestratorOptions.awsKinesisEndpoint,
|
endpoint: CloudRunnerOptions.awsKinesisEndpoint,
|
||||||
credentials: AwsClientFactory.getCredentials(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,8 +50,7 @@ export class AwsClientFactory {
|
|||||||
if (!this.cloudWatchLogs) {
|
if (!this.cloudWatchLogs) {
|
||||||
this.cloudWatchLogs = new CloudWatchLogs({
|
this.cloudWatchLogs = new CloudWatchLogs({
|
||||||
region: Input.region,
|
region: Input.region,
|
||||||
endpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
|
endpoint: CloudRunnerOptions.awsCloudWatchLogsEndpoint,
|
||||||
credentials: AwsClientFactory.getCredentials(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,9 +61,8 @@ export class AwsClientFactory {
|
|||||||
if (!this.s3) {
|
if (!this.s3) {
|
||||||
this.s3 = new S3({
|
this.s3 = new S3({
|
||||||
region: Input.region,
|
region: Input.region,
|
||||||
endpoint: OrchestratorOptions.awsS3Endpoint,
|
endpoint: CloudRunnerOptions.awsS3Endpoint,
|
||||||
forcePathStyle: true,
|
forcePathStyle: true,
|
||||||
credentials: AwsClientFactory.getCredentials(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
-1
@@ -21,7 +21,6 @@ export class AWSCloudFormationTemplates {
|
|||||||
|
|
||||||
public static getSecretDefinitionTemplate(p1: string, p2: string) {
|
public static getSecretDefinitionTemplate(p1: string, p2: string) {
|
||||||
return `
|
return `
|
||||||
Secrets:
|
|
||||||
- Name: '${p1}'
|
- Name: '${p1}'
|
||||||
ValueFrom: !Ref ${p2}Secret
|
ValueFrom: !Ref ${p2}Secret
|
||||||
`;
|
`;
|
||||||
+6
-6
@@ -1,16 +1,16 @@
|
|||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
|
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
|
|
||||||
export class AWSError {
|
export class AWSError {
|
||||||
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
|
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
|
||||||
OrchestratorLogger.log('aws error: ');
|
CloudRunnerLogger.log('aws error: ');
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
core.error(JSON.stringify(error, undefined, 4));
|
||||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||||
OrchestratorLogger.log('Getting events and resources for task stack');
|
CloudRunnerLogger.log('Getting events and resources for task stack');
|
||||||
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
|
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
|
||||||
OrchestratorLogger.log(JSON.stringify(events, undefined, 4));
|
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+18
-33
@@ -1,34 +1,22 @@
|
|||||||
import {
|
import {
|
||||||
CloudFormation,
|
CloudFormation,
|
||||||
CreateStackCommand,
|
CreateStackCommand,
|
||||||
// eslint-disable-next-line import/named
|
|
||||||
CreateStackCommandInput,
|
CreateStackCommandInput,
|
||||||
DescribeStackResourcesCommand,
|
DescribeStackResourcesCommand,
|
||||||
DescribeStacksCommand,
|
DescribeStacksCommand,
|
||||||
ListStacksCommand,
|
ListStacksCommand,
|
||||||
waitUntilStackCreateComplete,
|
waitUntilStackCreateComplete,
|
||||||
} from '@aws-sdk/client-cloudformation';
|
} from '@aws-sdk/client-cloudformation';
|
||||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { AWSError } from './aws-error';
|
import { AWSError } from './aws-error';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
|
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
||||||
|
|
||||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
|
||||||
|
|
||||||
function getStackWaitTime(): number {
|
|
||||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
|
||||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
|
||||||
return overrideValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class AWSJobStack {
|
export class AWSJobStack {
|
||||||
private baseStackName: string;
|
private baseStackName: string;
|
||||||
constructor(baseStackName: string) {
|
constructor(baseStackName: string) {
|
||||||
@@ -43,23 +31,23 @@ export class AWSJobStack {
|
|||||||
commands: string,
|
commands: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<OrchestratorAWSTaskDef> {
|
): Promise<CloudRunnerAWSTaskDef> {
|
||||||
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
||||||
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
||||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||||
`ContainerCpu:
|
`ContainerCpu:
|
||||||
Default: 1024`,
|
Default: 1024`,
|
||||||
`ContainerCpu:
|
`ContainerCpu:
|
||||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerCpu)}`,
|
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
|
||||||
);
|
);
|
||||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
taskDefCloudFormation = taskDefCloudFormation.replace(
|
||||||
`ContainerMemory:
|
`ContainerMemory:
|
||||||
Default: 2048`,
|
Default: 2048`,
|
||||||
`ContainerMemory:
|
`ContainerMemory:
|
||||||
Default: ${Number.parseInt(Orchestrator.buildParameters.containerMemory)}`,
|
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
|
||||||
);
|
);
|
||||||
if (!OrchestratorOptions.asyncOrchestrator) {
|
if (!CloudRunnerOptions.asyncCloudRunner) {
|
||||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||||
taskDefCloudFormation,
|
taskDefCloudFormation,
|
||||||
'# template resources logstream',
|
'# template resources logstream',
|
||||||
@@ -133,8 +121,8 @@ export class AWSJobStack {
|
|||||||
},
|
},
|
||||||
...secretsMappedToCloudFormationParameters,
|
...secretsMappedToCloudFormationParameters,
|
||||||
];
|
];
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Starting AWS job with memory: ${Orchestrator.buildParameters.containerMemory} cpu: ${Orchestrator.buildParameters.containerCpu}`,
|
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
|
||||||
);
|
);
|
||||||
let previousStackExists = true;
|
let previousStackExists = true;
|
||||||
while (previousStackExists) {
|
while (previousStackExists) {
|
||||||
@@ -147,7 +135,7 @@ export class AWSJobStack {
|
|||||||
const element = stacks.StackSummaries[index];
|
const element = stacks.StackSummaries[index];
|
||||||
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
||||||
previousStackExists = true;
|
previousStackExists = true;
|
||||||
OrchestratorLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
||||||
await new Promise((promise) => setTimeout(promise, 5000));
|
await new Promise((promise) => setTimeout(promise, 5000));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -159,15 +147,12 @@ export class AWSJobStack {
|
|||||||
Parameters: parameters,
|
Parameters: parameters,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
const stackWaitTimeSeconds = getStackWaitTime();
|
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
|
||||||
OrchestratorLogger.log(
|
|
||||||
`Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`,
|
|
||||||
);
|
|
||||||
await CF.send(new CreateStackCommand(createStackInput));
|
await CF.send(new CreateStackCommand(createStackInput));
|
||||||
await waitUntilStackCreateComplete(
|
await waitUntilStackCreateComplete(
|
||||||
{
|
{
|
||||||
client: CF,
|
client: CF,
|
||||||
maxWaitTime: stackWaitTimeSeconds,
|
maxWaitTime: 200,
|
||||||
},
|
},
|
||||||
{ StackName: taskDefStackName },
|
{ StackName: taskDefStackName },
|
||||||
);
|
);
|
||||||
@@ -201,7 +186,7 @@ export class AWSJobStack {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
ParameterKey: 'BUILDGUID',
|
ParameterKey: 'BUILDGUID',
|
||||||
ParameterValue: Orchestrator.buildParameters.buildGuid,
|
ParameterValue: CloudRunner.buildParameters.buildGuid,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
ParameterKey: 'EnvironmentName',
|
ParameterKey: 'EnvironmentName',
|
||||||
@@ -209,9 +194,9 @@ export class AWSJobStack {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
if (OrchestratorOptions.useCleanupCron) {
|
if (CloudRunnerOptions.useCleanupCron) {
|
||||||
try {
|
try {
|
||||||
OrchestratorLogger.log(`Creating job cleanup formation`);
|
CloudRunnerLogger.log(`Creating job cleanup formation`);
|
||||||
await CF.send(new CreateStackCommand(createCleanupStackInput));
|
await CF.send(new CreateStackCommand(createCleanupStackInput));
|
||||||
|
|
||||||
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
|
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
|
||||||
+30
-72
@@ -1,61 +1,23 @@
|
|||||||
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
|
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
|
||||||
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
|
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||||
import * as zlib from 'node:zlib';
|
import * as zlib from 'node:zlib';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { Input } from '../../..';
|
import { Input } from '../../..';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import GitHub from '../../../github';
|
import GitHub from '../../../github';
|
||||||
import { AwsClientFactory } from './aws-client-factory';
|
import { AwsClientFactory } from './aws-client-factory';
|
||||||
|
|
||||||
class AWSTaskRunner {
|
class AWSTaskRunner {
|
||||||
private static readonly encodedUnderscore = `$252F`;
|
private static readonly encodedUnderscore = `$252F`;
|
||||||
|
|
||||||
/**
|
|
||||||
* Transform localhost endpoints to host.docker.internal for container environments.
|
|
||||||
* When LocalStack is used, ECS tasks run in Docker containers that need to reach
|
|
||||||
* LocalStack on the host machine via host.docker.internal.
|
|
||||||
*/
|
|
||||||
private static transformEndpointsForContainer(
|
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
|
||||||
): OrchestratorEnvironmentVariable[] {
|
|
||||||
const endpointEnvironmentNames = new Set([
|
|
||||||
'AWS_S3_ENDPOINT',
|
|
||||||
'AWS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
|
||||||
'AWS_ECS_ENDPOINT',
|
|
||||||
'AWS_KINESIS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
|
||||||
'INPUT_AWSS3ENDPOINT',
|
|
||||||
'INPUT_AWSENDPOINT',
|
|
||||||
]);
|
|
||||||
|
|
||||||
return environment.map((x) => {
|
|
||||||
let value = x.value;
|
|
||||||
if (
|
|
||||||
typeof value === 'string' &&
|
|
||||||
endpointEnvironmentNames.has(x.name) &&
|
|
||||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
|
||||||
) {
|
|
||||||
// Replace localhost with host.docker.internal so ECS containers can access host services
|
|
||||||
value = value
|
|
||||||
.replace('http://localhost', 'http://host.docker.internal')
|
|
||||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
|
||||||
OrchestratorLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return { name: x.name, value };
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
static async runTask(
|
static async runTask(
|
||||||
taskDef: OrchestratorAWSTaskDef,
|
taskDef: CloudRunnerAWSTaskDef,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
commands: string,
|
commands: string,
|
||||||
): Promise<{ output: string; shouldCleanup: boolean }> {
|
): Promise<{ output: string; shouldCleanup: boolean }> {
|
||||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||||
@@ -70,9 +32,6 @@ class AWSTaskRunner {
|
|||||||
const streamName =
|
const streamName =
|
||||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||||
|
|
||||||
// Transform localhost endpoints for container environment
|
|
||||||
const transformedEnvironment = AWSTaskRunner.transformEndpointsForContainer(environment);
|
|
||||||
|
|
||||||
const runParameters = {
|
const runParameters = {
|
||||||
cluster,
|
cluster,
|
||||||
taskDefinition,
|
taskDefinition,
|
||||||
@@ -81,8 +40,8 @@ class AWSTaskRunner {
|
|||||||
containerOverrides: [
|
containerOverrides: [
|
||||||
{
|
{
|
||||||
name: taskDef.taskDefStackName,
|
name: taskDef.taskDefStackName,
|
||||||
environment: transformedEnvironment,
|
environment,
|
||||||
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, Orchestrator.buildParameters)],
|
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -97,28 +56,28 @@ class AWSTaskRunner {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
|
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
|
||||||
OrchestratorLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
||||||
throw new Error(`Container Overrides length must be at most 8192`);
|
throw new Error(`Container Overrides length must be at most 8192`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
|
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
|
||||||
const taskArn = task.tasks?.[0].taskArn || '';
|
const taskArn = task.tasks?.[0].taskArn || '';
|
||||||
OrchestratorLogger.log('Orchestrator job is starting');
|
CloudRunnerLogger.log('Cloud runner job is starting');
|
||||||
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Orchestrator job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
||||||
OrchestratorOptions.asyncOrchestrator
|
CloudRunnerOptions.asyncCloudRunner
|
||||||
}`,
|
}`,
|
||||||
);
|
);
|
||||||
if (OrchestratorOptions.asyncOrchestrator) {
|
if (CloudRunnerOptions.asyncCloudRunner) {
|
||||||
const shouldCleanup: boolean = false;
|
const shouldCleanup: boolean = false;
|
||||||
const output: string = '';
|
const output: string = '';
|
||||||
OrchestratorLogger.log(`Watch Orchestrator To End: false`);
|
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
|
||||||
|
|
||||||
return { output, shouldCleanup };
|
return { output, shouldCleanup };
|
||||||
}
|
}
|
||||||
|
|
||||||
OrchestratorLogger.log(`Streaming...`);
|
CloudRunnerLogger.log(`Streaming...`);
|
||||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
||||||
let exitCode;
|
let exitCode;
|
||||||
let containerState;
|
let containerState;
|
||||||
@@ -133,13 +92,13 @@ class AWSTaskRunner {
|
|||||||
containerState = containers[0];
|
containerState = containers[0];
|
||||||
exitCode = containerState?.exitCode;
|
exitCode = containerState?.exitCode;
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||||
if (exitCode === undefined) {
|
if (exitCode === undefined) {
|
||||||
OrchestratorLogger.logWarning(`Undefined exitcode for container`);
|
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
|
||||||
}
|
}
|
||||||
const wasSuccessful = exitCode === 0;
|
const wasSuccessful = exitCode === 0;
|
||||||
if (wasSuccessful) {
|
if (wasSuccessful) {
|
||||||
OrchestratorLogger.log(`Orchestrator job has finished successfully`);
|
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
||||||
|
|
||||||
return { output, shouldCleanup };
|
return { output, shouldCleanup };
|
||||||
}
|
}
|
||||||
@@ -166,7 +125,7 @@ class AWSTaskRunner {
|
|||||||
const error = error_ as Error;
|
const error = error_ as Error;
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||||
OrchestratorLogger.log(`Orchestrator job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
|
CloudRunnerLogger.log(`Cloud runner job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
|
||||||
|
|
||||||
core.setFailed(error);
|
core.setFailed(error);
|
||||||
core.error(error);
|
core.error(error);
|
||||||
@@ -193,7 +152,7 @@ class AWSTaskRunner {
|
|||||||
}
|
}
|
||||||
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
|
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
|
||||||
const sleepMs = delayMs + jitterMs;
|
const sleepMs = delayMs + jitterMs;
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
|
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
|
||||||
);
|
);
|
||||||
await new Promise((r) => setTimeout(r, sleepMs));
|
await new Promise((r) => setTimeout(r, sleepMs));
|
||||||
@@ -204,12 +163,12 @@ class AWSTaskRunner {
|
|||||||
|
|
||||||
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
OrchestratorLogger.log(`Streaming...`);
|
CloudRunnerLogger.log(`Streaming...`);
|
||||||
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
||||||
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
||||||
|
|
||||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${Orchestrator.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${Orchestrator.buildParameters.awsStackName}-${Orchestrator.buildParameters.buildGuid}`;
|
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
|
||||||
OrchestratorLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
||||||
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
|
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
|
||||||
let shouldReadLogs = true;
|
let shouldReadLogs = true;
|
||||||
let shouldCleanup = true;
|
let shouldCleanup = true;
|
||||||
@@ -248,9 +207,8 @@ class AWSTaskRunner {
|
|||||||
const baseBackoffMs = 1000;
|
const baseBackoffMs = 1000;
|
||||||
const jitterMs = Math.floor(Math.random() * 1000);
|
const jitterMs = Math.floor(Math.random() * 1000);
|
||||||
const sleepMs = baseBackoffMs + jitterMs;
|
const sleepMs = baseBackoffMs + jitterMs;
|
||||||
OrchestratorLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
|
CloudRunnerLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
|
||||||
await new Promise((r) => setTimeout(r, sleepMs));
|
await new Promise((r) => setTimeout(r, sleepMs));
|
||||||
|
|
||||||
return { iterator, shouldReadLogs, output, shouldCleanup };
|
return { iterator, shouldReadLogs, output, shouldCleanup };
|
||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
@@ -269,18 +227,18 @@ class AWSTaskRunner {
|
|||||||
|
|
||||||
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
|
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
|
||||||
if (taskData?.lastStatus === 'UNKNOWN') {
|
if (taskData?.lastStatus === 'UNKNOWN') {
|
||||||
OrchestratorLogger.log('## Orchestrator job unknwon');
|
CloudRunnerLogger.log('## Cloud runner job unknwon');
|
||||||
}
|
}
|
||||||
if (taskData?.lastStatus !== 'RUNNING') {
|
if (taskData?.lastStatus !== 'RUNNING') {
|
||||||
if (timestamp === 0) {
|
if (timestamp === 0) {
|
||||||
OrchestratorLogger.log('## Orchestrator job stopped, streaming end of logs');
|
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
|
||||||
timestamp = Date.now();
|
timestamp = Date.now();
|
||||||
}
|
}
|
||||||
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
|
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
|
||||||
OrchestratorLogger.log('## Orchestrator status is not RUNNING for 30 seconds, last query for logs');
|
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs');
|
||||||
shouldReadLogs = false;
|
shouldReadLogs = false;
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { timestamp, shouldReadLogs };
|
return { timestamp, shouldReadLogs };
|
||||||
+6
-5
@@ -1,7 +1,7 @@
|
|||||||
import Orchestrator from '../../../orchestrator';
|
import CloudRunner from '../../../cloud-runner';
|
||||||
|
|
||||||
export class TaskDefinitionFormation {
|
export class TaskDefinitionFormation {
|
||||||
public static readonly description: string = `Game CI Orchestrator Task Stack`;
|
public static readonly description: string = `Game CI Cloud Runner Task Stack`;
|
||||||
public static get formation(): string {
|
public static get formation(): string {
|
||||||
return `AWSTemplateFormatVersion: 2010-09-09
|
return `AWSTemplateFormatVersion: 2010-09-09
|
||||||
Description: ${TaskDefinitionFormation.description}
|
Description: ${TaskDefinitionFormation.description}
|
||||||
@@ -29,11 +29,11 @@ Parameters:
|
|||||||
Default: 80
|
Default: 80
|
||||||
Description: What port number the application inside the docker container is binding to
|
Description: What port number the application inside the docker container is binding to
|
||||||
ContainerCpu:
|
ContainerCpu:
|
||||||
Default: ${Orchestrator.buildParameters.containerCpu}
|
Default: ${CloudRunner.buildParameters.containerCpu}
|
||||||
Type: Number
|
Type: Number
|
||||||
Description: How much CPU to give the container. 1024 is 1 CPU
|
Description: How much CPU to give the container. 1024 is 1 CPU
|
||||||
ContainerMemory:
|
ContainerMemory:
|
||||||
Default: ${Orchestrator.buildParameters.containerMemory}
|
Default: ${CloudRunner.buildParameters.containerMemory}
|
||||||
Type: Number
|
Type: Number
|
||||||
Description: How much memory in megabytes to give the container
|
Description: How much memory in megabytes to give the container
|
||||||
BUILDGUID:
|
BUILDGUID:
|
||||||
@@ -127,7 +127,8 @@ Resources:
|
|||||||
- SourceVolume: efs-data
|
- SourceVolume: efs-data
|
||||||
ContainerPath: !Ref EFSMountDirectory
|
ContainerPath: !Ref EFSMountDirectory
|
||||||
ReadOnly: false
|
ReadOnly: false
|
||||||
# template secrets p3 - container def
|
Secrets:
|
||||||
|
# template secrets p3 - container def
|
||||||
LogConfiguration:
|
LogConfiguration:
|
||||||
LogDriver: awslogs
|
LogDriver: awslogs
|
||||||
Options:
|
Options:
|
||||||
+2
-3
@@ -1,10 +1,9 @@
|
|||||||
// eslint-disable-next-line import/named
|
|
||||||
import { StackResource } from '@aws-sdk/client-cloudformation';
|
import { StackResource } from '@aws-sdk/client-cloudformation';
|
||||||
|
|
||||||
class OrchestratorAWSTaskDef {
|
class CloudRunnerAWSTaskDef {
|
||||||
public taskDefStackName!: string;
|
public taskDefStackName!: string;
|
||||||
public taskDefCloudFormation!: string;
|
public taskDefCloudFormation!: string;
|
||||||
public taskDefResources: StackResource[] | undefined;
|
public taskDefResources: StackResource[] | undefined;
|
||||||
public baseResources: StackResource[] | undefined;
|
public baseResources: StackResource[] | undefined;
|
||||||
}
|
}
|
||||||
export default OrchestratorAWSTaskDef;
|
export default CloudRunnerAWSTaskDef;
|
||||||
+19
-34
@@ -1,11 +1,11 @@
|
|||||||
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
|
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
|
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||||
import AwsTaskRunner from './aws-task-runner';
|
import AwsTaskRunner from './aws-task-runner';
|
||||||
import { ProviderInterface } from '../provider-interface';
|
import { ProviderInterface } from '../provider-interface';
|
||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
||||||
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
||||||
import { Input } from '../../..';
|
import { Input } from '../../..';
|
||||||
@@ -13,20 +13,8 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
|
|||||||
import { ProviderResource } from '../provider-resource';
|
import { ProviderResource } from '../provider-resource';
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
import { ProviderWorkflow } from '../provider-workflow';
|
||||||
import { TaskService } from './services/task-service';
|
import { TaskService } from './services/task-service';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import { AwsClientFactory } from './aws-client-factory';
|
import { AwsClientFactory } from './aws-client-factory';
|
||||||
import ResourceTracking from '../../services/core/resource-tracking';
|
|
||||||
|
|
||||||
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
|
|
||||||
|
|
||||||
function getStackWaitTime(): number {
|
|
||||||
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
|
|
||||||
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
|
|
||||||
return overrideValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
return DEFAULT_STACK_WAIT_TIME_SECONDS;
|
|
||||||
}
|
|
||||||
|
|
||||||
class AWSBuildEnvironment implements ProviderInterface {
|
class AWSBuildEnvironment implements ProviderInterface {
|
||||||
private baseStackName: string;
|
private baseStackName: string;
|
||||||
@@ -98,16 +86,14 @@ class AWSBuildEnvironment implements ProviderInterface {
|
|||||||
commands: string,
|
commands: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
ResourceTracking.logAllocationSummary('aws workflow');
|
|
||||||
await ResourceTracking.logDiskUsageSnapshot('aws workflow (host)');
|
|
||||||
AwsClientFactory.getECS();
|
AwsClientFactory.getECS();
|
||||||
const CF = AwsClientFactory.getCloudFormation();
|
const CF = AwsClientFactory.getCloudFormation();
|
||||||
AwsClientFactory.getKinesis();
|
AwsClientFactory.getKinesis();
|
||||||
OrchestratorLogger.log(`AWS Region: ${CF.config.region}`);
|
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
|
||||||
const entrypoint = ['/bin/sh'];
|
const entrypoint = ['/bin/sh'];
|
||||||
const startTimeMs = Date.now();
|
const startTimeMs = Date.now();
|
||||||
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
||||||
@@ -124,37 +110,36 @@ class AWSBuildEnvironment implements ProviderInterface {
|
|||||||
let postRunTaskTimeMs;
|
let postRunTaskTimeMs;
|
||||||
try {
|
try {
|
||||||
const postSetupStacksTimeMs = Date.now();
|
const postSetupStacksTimeMs = Date.now();
|
||||||
OrchestratorLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
||||||
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
||||||
postRunTaskTimeMs = Date.now();
|
postRunTaskTimeMs = Date.now();
|
||||||
OrchestratorLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
||||||
if (shouldCleanup) {
|
if (shouldCleanup) {
|
||||||
await this.cleanupResources(CF, taskDef);
|
await this.cleanupResources(CF, taskDef);
|
||||||
}
|
}
|
||||||
const postCleanupTimeMs = Date.now();
|
const postCleanupTimeMs = Date.now();
|
||||||
if (postRunTaskTimeMs !== undefined)
|
if (postRunTaskTimeMs !== undefined)
|
||||||
OrchestratorLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
||||||
|
|
||||||
return output;
|
return output;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
OrchestratorLogger.log(`error running task ${error}`);
|
CloudRunnerLogger.log(`error running task ${error}`);
|
||||||
await this.cleanupResources(CF, taskDef);
|
await this.cleanupResources(CF, taskDef);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async cleanupResources(CF: CloudFormation, taskDef: OrchestratorAWSTaskDef) {
|
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
|
||||||
const stackWaitTimeSeconds = getStackWaitTime();
|
CloudRunnerLogger.log('Cleanup starting');
|
||||||
OrchestratorLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`);
|
|
||||||
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
|
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
|
||||||
if (OrchestratorOptions.useCleanupCron) {
|
if (CloudRunnerOptions.useCleanupCron) {
|
||||||
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
|
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
|
||||||
}
|
}
|
||||||
|
|
||||||
await waitUntilStackDeleteComplete(
|
await waitUntilStackDeleteComplete(
|
||||||
{
|
{
|
||||||
client: CF,
|
client: CF,
|
||||||
maxWaitTime: stackWaitTimeSeconds,
|
maxWaitTime: 200,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
StackName: taskDef.taskDefStackName,
|
StackName: taskDef.taskDefStackName,
|
||||||
@@ -163,14 +148,14 @@ class AWSBuildEnvironment implements ProviderInterface {
|
|||||||
await waitUntilStackDeleteComplete(
|
await waitUntilStackDeleteComplete(
|
||||||
{
|
{
|
||||||
client: CF,
|
client: CF,
|
||||||
maxWaitTime: stackWaitTimeSeconds,
|
maxWaitTime: 200,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
StackName: `${taskDef.taskDefStackName}-cleanup`,
|
StackName: `${taskDef.taskDefStackName}-cleanup`,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
OrchestratorLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
||||||
OrchestratorLogger.log('Cleanup complete');
|
CloudRunnerLogger.log('Cleanup complete');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default AWSBuildEnvironment;
|
export default AWSBuildEnvironment;
|
||||||
+7
-7
@@ -2,7 +2,7 @@ import { DeleteStackCommand, DescribeStackResourcesCommand } from '@aws-sdk/clie
|
|||||||
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
|
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
import { StopTaskCommand } from '@aws-sdk/client-ecs';
|
import { StopTaskCommand } from '@aws-sdk/client-ecs';
|
||||||
import Input from '../../../../input';
|
import Input from '../../../../input';
|
||||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
||||||
import { TaskService } from './task-service';
|
import { TaskService } from './task-service';
|
||||||
import { AwsClientFactory } from '../aws-client-factory';
|
import { AwsClientFactory } from '../aws-client-factory';
|
||||||
|
|
||||||
@@ -25,7 +25,7 @@ export class GarbageCollectionService {
|
|||||||
const { taskElement, element } = task;
|
const { taskElement, element } = task;
|
||||||
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
||||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
|
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
|
||||||
OrchestratorLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||||
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
|
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -37,7 +37,7 @@ export class GarbageCollectionService {
|
|||||||
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
OrchestratorLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -47,12 +47,12 @@ export class GarbageCollectionService {
|
|||||||
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
|
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
|
||||||
) {
|
) {
|
||||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
||||||
OrchestratorLogger.log(`Skipping ${element.StackName} ignore list`);
|
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
OrchestratorLogger.log(`Deleting ${element.StackName}`);
|
CloudRunnerLogger.log(`Deleting ${element.StackName}`);
|
||||||
await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
|
await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -62,14 +62,14 @@ export class GarbageCollectionService {
|
|||||||
deleteResources &&
|
deleteResources &&
|
||||||
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
|
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
|
||||||
) {
|
) {
|
||||||
OrchestratorLogger.log(`Deleting ${element.logGroupName}`);
|
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||||
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
|
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const locks = await TaskService.getLocks();
|
const locks = await TaskService.getLocks();
|
||||||
for (const element of locks) {
|
for (const element of locks) {
|
||||||
OrchestratorLogger.log(`Lock: ${element.Key}`);
|
CloudRunnerLogger.log(`Lock: ${element.Key}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+66
-73
@@ -3,21 +3,24 @@ import {
|
|||||||
DescribeStacksCommand,
|
DescribeStacksCommand,
|
||||||
ListStacksCommand,
|
ListStacksCommand,
|
||||||
} from '@aws-sdk/client-cloudformation';
|
} from '@aws-sdk/client-cloudformation';
|
||||||
import type { StackSummary } from '@aws-sdk/client-cloudformation';
|
import type { ListStacksCommandOutput } from '@aws-sdk/client-cloudformation';
|
||||||
// eslint-disable-next-line import/named
|
import { DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
import { DescribeLogGroupsCommand, DescribeLogGroupsCommandInput } from '@aws-sdk/client-cloudwatch-logs';
|
import type { DescribeLogGroupsCommandInput, DescribeLogGroupsCommandOutput } from '@aws-sdk/client-cloudwatch-logs';
|
||||||
import type { LogGroup } from '@aws-sdk/client-cloudwatch-logs';
|
|
||||||
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
|
||||||
import type { Task } from '@aws-sdk/client-ecs';
|
import type { DescribeTasksCommandOutput } from '@aws-sdk/client-ecs';
|
||||||
import { ListObjectsV2Command } from '@aws-sdk/client-s3';
|
import { ListObjectsCommand } from '@aws-sdk/client-s3';
|
||||||
import Input from '../../../../input';
|
import Input from '../../../../input';
|
||||||
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
||||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
||||||
import AwsTaskRunner from '../aws-task-runner';
|
import AwsTaskRunner from '../aws-task-runner';
|
||||||
import Orchestrator from '../../../orchestrator';
|
import CloudRunner from '../../../cloud-runner';
|
||||||
import { AwsClientFactory } from '../aws-client-factory';
|
import { AwsClientFactory } from '../aws-client-factory';
|
||||||
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
|
||||||
|
|
||||||
|
type StackSummary = NonNullable<ListStacksCommandOutput['StackSummaries']>[number];
|
||||||
|
type LogGroup = NonNullable<DescribeLogGroupsCommandOutput['logGroups']>[number];
|
||||||
|
type Task = NonNullable<DescribeTasksCommandOutput['tasks']>[number];
|
||||||
|
|
||||||
export class TaskService {
|
export class TaskService {
|
||||||
static async watch() {
|
static async watch() {
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
@@ -29,10 +32,10 @@ export class TaskService {
|
|||||||
|
|
||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
public static async getCloudFormationJobStacks(): Promise<StackSummary[]> {
|
public static async getCloudFormationJobStacks() {
|
||||||
const result: StackSummary[] = [];
|
const result: StackSummary[] = [];
|
||||||
OrchestratorLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
OrchestratorLogger.log(`List Cloud Formation Stacks`);
|
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
const CF = AwsClientFactory.getCloudFormation();
|
const CF = AwsClientFactory.getCloudFormation();
|
||||||
const stacks =
|
const stacks =
|
||||||
@@ -40,16 +43,16 @@ export class TaskService {
|
|||||||
(_x) =>
|
(_x) =>
|
||||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
||||||
) || [];
|
) || [];
|
||||||
OrchestratorLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
OrchestratorLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
||||||
for (const element of stacks) {
|
for (const element of stacks) {
|
||||||
if (!element.CreationTime) {
|
if (!element.CreationTime) {
|
||||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||||
ageDate.getHours() / 24,
|
ageDate.getHours() / 24,
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||||
@@ -61,73 +64,68 @@ export class TaskService {
|
|||||||
(_x) =>
|
(_x) =>
|
||||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
||||||
) || [];
|
) || [];
|
||||||
OrchestratorLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
OrchestratorLogger.log(`Base Stacks ${baseStacks.length}`);
|
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||||
for (const element of baseStacks) {
|
for (const element of baseStacks) {
|
||||||
if (!element.CreationTime) {
|
if (!element.CreationTime) {
|
||||||
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
|
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||||
ageDate.getHours() / 24,
|
ageDate.getHours() / 24,
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||||
);
|
);
|
||||||
result.push(element);
|
result.push(element);
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> {
|
public static async getTasks() {
|
||||||
const result: { taskElement: Task; element: string }[] = [];
|
// Extended Task type to include custom properties added in this method
|
||||||
OrchestratorLogger.log(``);
|
type ExtendedTask = Task & {
|
||||||
OrchestratorLogger.log(`List Tasks`);
|
overrides?: Record<string, unknown>;
|
||||||
|
attachments?: unknown[];
|
||||||
|
};
|
||||||
|
const result: { taskElement: ExtendedTask; element: string }[] = [];
|
||||||
|
CloudRunnerLogger.log(``);
|
||||||
|
CloudRunnerLogger.log(`List Tasks`);
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
const ecs = AwsClientFactory.getECS();
|
const ecs = AwsClientFactory.getECS();
|
||||||
const clusters: string[] = [];
|
const clusters = (await ecs.send(new ListClustersCommand({}))).clusterArns || [];
|
||||||
{
|
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
|
||||||
let nextToken: string | undefined;
|
|
||||||
do {
|
|
||||||
const clusterResponse = await ecs.send(new ListClustersCommand({ nextToken }));
|
|
||||||
clusters.push(...(clusterResponse.clusterArns ?? []));
|
|
||||||
nextToken = clusterResponse.nextToken;
|
|
||||||
} while (nextToken);
|
|
||||||
}
|
|
||||||
OrchestratorLogger.log(`Task Clusters ${clusters.length}`);
|
|
||||||
for (const element of clusters) {
|
for (const element of clusters) {
|
||||||
const taskArns: string[] = [];
|
const input = {
|
||||||
{
|
cluster: element,
|
||||||
let nextToken: string | undefined;
|
};
|
||||||
do {
|
const list = (await ecs.send(new ListTasksCommand(input))).taskArns || [];
|
||||||
const taskResponse = await ecs.send(new ListTasksCommand({ cluster: element, nextToken }));
|
if (list.length > 0) {
|
||||||
taskArns.push(...(taskResponse.taskArns ?? []));
|
const describeInput = { tasks: list, cluster: element };
|
||||||
nextToken = taskResponse.nextToken;
|
|
||||||
} while (nextToken);
|
|
||||||
}
|
|
||||||
if (taskArns.length > 0) {
|
|
||||||
const describeInput = { tasks: taskArns, cluster: element };
|
|
||||||
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
|
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
|
||||||
if (describeList.length === 0) {
|
if (describeList.length === 0) {
|
||||||
OrchestratorLogger.log(`No Tasks`);
|
CloudRunnerLogger.log(`No Tasks`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`Tasks ${describeList.length}`);
|
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
|
||||||
for (const taskElement of describeList) {
|
for (const taskElement of describeList) {
|
||||||
if (taskElement === undefined) {
|
if (taskElement === undefined) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (taskElement.createdAt === undefined) {
|
const extendedTask = taskElement as ExtendedTask;
|
||||||
OrchestratorLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
extendedTask.overrides = {};
|
||||||
|
extendedTask.attachments = [];
|
||||||
|
if (extendedTask.createdAt === undefined) {
|
||||||
|
CloudRunnerLogger.log(`Skipping ${extendedTask.taskDefinitionArn} no createdAt date`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
result.push({ taskElement, element });
|
result.push({ taskElement: extendedTask, element });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(``);
|
CloudRunnerLogger.log(``);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -143,7 +141,7 @@ export class TaskService {
|
|||||||
throw new Error('stack not defined');
|
throw new Error('stack not defined');
|
||||||
}
|
}
|
||||||
if (!stack.CreationTime) {
|
if (!stack.CreationTime) {
|
||||||
OrchestratorLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
||||||
}
|
}
|
||||||
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
|
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
|
||||||
const message = `
|
const message = `
|
||||||
@@ -153,43 +151,40 @@ export class TaskService {
|
|||||||
${JSON.stringify(stackInfo, undefined, 4)}
|
${JSON.stringify(stackInfo, undefined, 4)}
|
||||||
${JSON.stringify(stackInfo2, undefined, 4)}
|
${JSON.stringify(stackInfo2, undefined, 4)}
|
||||||
`;
|
`;
|
||||||
OrchestratorLogger.log(message);
|
CloudRunnerLogger.log(message);
|
||||||
|
|
||||||
return message;
|
return message;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
OrchestratorLogger.error(
|
CloudRunnerLogger.error(
|
||||||
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
|
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
|
||||||
);
|
);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
public static async getLogGroups(): Promise<LogGroup[]> {
|
public static async getLogGroups() {
|
||||||
const result: LogGroup[] = [];
|
const result: LogGroup[] = [];
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
const cwl = AwsClientFactory.getCloudWatchLogs();
|
const ecs = AwsClientFactory.getCloudWatchLogs();
|
||||||
let logStreamInput: DescribeLogGroupsCommandInput = {
|
let logStreamInput: DescribeLogGroupsCommandInput = {
|
||||||
/* logGroupNamePrefix: 'game-ci' */
|
/* logGroupNamePrefix: 'game-ci' */
|
||||||
};
|
};
|
||||||
let logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||||
const logGroups = logGroupsDescribe.logGroups || [];
|
const logGroups = logGroupsDescribe.logGroups || [];
|
||||||
while (logGroupsDescribe.nextToken) {
|
while (logGroupsDescribe.nextToken) {
|
||||||
logStreamInput = {
|
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
|
||||||
/* logGroupNamePrefix: 'game-ci',*/
|
logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
||||||
nextToken: logGroupsDescribe.nextToken,
|
|
||||||
};
|
|
||||||
logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
|
|
||||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
||||||
}
|
}
|
||||||
|
|
||||||
OrchestratorLogger.log(`Log Groups ${logGroups.length}`);
|
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
|
||||||
for (const element of logGroups) {
|
for (const element of logGroups) {
|
||||||
if (element.creationTime === undefined) {
|
if (element.creationTime === undefined) {
|
||||||
OrchestratorLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
||||||
ageDate.getHours() / 24,
|
ageDate.getHours() / 24,
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||||
@@ -199,22 +194,20 @@ export class TaskService {
|
|||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
public static async getLocks(): Promise<Array<{ Key: string }>> {
|
public static async getLocks() {
|
||||||
process.env.AWS_REGION = Input.region;
|
process.env.AWS_REGION = Input.region;
|
||||||
if (Orchestrator.buildParameters.storageProvider === 'rclone') {
|
if (CloudRunner.buildParameters.storageProvider === 'rclone') {
|
||||||
// eslint-disable-next-line no-unused-vars
|
const objects = await (SharedWorkspaceLocking as any).listObjects('');
|
||||||
type ListObjectsFunction = (prefix: string) => Promise<string[]>;
|
|
||||||
const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects('');
|
|
||||||
|
|
||||||
return objects.map((x: string) => ({ Key: x }));
|
return objects.map((x: string) => ({ Key: x }));
|
||||||
}
|
}
|
||||||
const s3 = AwsClientFactory.getS3();
|
const s3 = AwsClientFactory.getS3();
|
||||||
const listRequest = {
|
const listRequest = {
|
||||||
Bucket: Orchestrator.buildParameters.awsStackName,
|
Bucket: CloudRunner.buildParameters.awsStackName,
|
||||||
};
|
};
|
||||||
|
|
||||||
const results = await s3.send(new ListObjectsV2Command(listRequest));
|
const results = await s3.send(new ListObjectsCommand(listRequest));
|
||||||
|
|
||||||
return (results.Contents || []).map((object) => ({ Key: object.Key || '' }));
|
return results.Contents || [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+27
-60
@@ -1,20 +1,20 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { ProviderInterface } from '../provider-interface';
|
import { ProviderInterface } from '../provider-interface';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import Docker from '../../../docker';
|
import Docker from '../../../docker';
|
||||||
import { Action } from '../../..';
|
import { Action } from '../../..';
|
||||||
import { writeFileSync } from 'node:fs';
|
import { writeFileSync } from 'node:fs';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { ProviderResource } from '../provider-resource';
|
import { ProviderResource } from '../provider-resource';
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
import { ProviderWorkflow } from '../provider-workflow';
|
||||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||||
import { StringKeyValuePair } from '../../../shared-types';
|
import { StringKeyValuePair } from '../../../shared-types';
|
||||||
|
|
||||||
class LocalDockerOrchestrator implements ProviderInterface {
|
class LocalDockerCloudRunner implements ProviderInterface {
|
||||||
public buildParameters!: BuildParameters;
|
public buildParameters!: BuildParameters;
|
||||||
|
|
||||||
listResources(): Promise<ProviderResource[]> {
|
listResources(): Promise<ProviderResource[]> {
|
||||||
@@ -50,15 +50,15 @@ class LocalDockerOrchestrator implements ProviderInterface {
|
|||||||
const { workspace } = Action;
|
const { workspace } = Action;
|
||||||
if (
|
if (
|
||||||
fs.existsSync(
|
fs.existsSync(
|
||||||
`${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
}`,
|
}`,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache/cache/build/`);
|
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
|
||||||
await OrchestratorSystem.Run(
|
await CloudRunnerSystem.Run(
|
||||||
`rm -r ${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
}`,
|
}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -80,47 +80,22 @@ class LocalDockerOrchestrator implements ProviderInterface {
|
|||||||
commands: string,
|
commands: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
OrchestratorLogger.log(buildGuid);
|
CloudRunnerLogger.log(buildGuid);
|
||||||
OrchestratorLogger.log(commands);
|
CloudRunnerLogger.log(commands);
|
||||||
|
|
||||||
const { workspace, actionFolder } = Action;
|
const { workspace, actionFolder } = Action;
|
||||||
const content: StringKeyValuePair[] = [];
|
const content: StringKeyValuePair[] = [];
|
||||||
for (const x of secrets) {
|
for (const x of secrets) {
|
||||||
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Replace localhost with host.docker.internal for LocalStack endpoints (similar to K8s)
|
|
||||||
// This allows Docker containers to access LocalStack running on the host
|
|
||||||
const endpointEnvironmentNames = new Set([
|
|
||||||
'AWS_S3_ENDPOINT',
|
|
||||||
'AWS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
|
||||||
'AWS_ECS_ENDPOINT',
|
|
||||||
'AWS_KINESIS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
|
||||||
'INPUT_AWSS3ENDPOINT',
|
|
||||||
'INPUT_AWSENDPOINT',
|
|
||||||
]);
|
|
||||||
for (const x of environment) {
|
for (const x of environment) {
|
||||||
let value = x.value;
|
content.push({ name: x.name, value: x.value });
|
||||||
if (
|
|
||||||
typeof value === 'string' &&
|
|
||||||
endpointEnvironmentNames.has(x.name) &&
|
|
||||||
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
|
||||||
) {
|
|
||||||
// Replace localhost with host.docker.internal so containers can access host services
|
|
||||||
value = value
|
|
||||||
.replace('http://localhost', 'http://host.docker.internal')
|
|
||||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
|
||||||
OrchestratorLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
|
|
||||||
}
|
|
||||||
content.push({ name: x.name, value });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// if (this.buildParameters?.orchestratorIntegrationTests) {
|
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
|
||||||
// core.info(JSON.stringify(content, undefined, 4));
|
// core.info(JSON.stringify(content, undefined, 4));
|
||||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||||
// core.info(JSON.stringify(environment, undefined, 4));
|
// core.info(JSON.stringify(environment, undefined, 4));
|
||||||
@@ -137,33 +112,25 @@ class LocalDockerOrchestrator implements ProviderInterface {
|
|||||||
|
|
||||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||||
const entrypointFilePath = `start.sh`;
|
const entrypointFilePath = `start.sh`;
|
||||||
|
const fileContents = `#!/bin/bash
|
||||||
// Use #!/bin/sh for POSIX compatibility (Alpine-based images like rclone/rclone don't have bash)
|
|
||||||
const fileContents = `#!/bin/sh
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
mkdir -p /github/workspace/orchestrator-cache
|
mkdir -p /github/workspace/cloud-runner-cache
|
||||||
mkdir -p /data/cache
|
mkdir -p /data/cache
|
||||||
cp -a /github/workspace/orchestrator-cache/. ${sharedFolder}
|
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
||||||
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
|
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
|
||||||
# Only copy cache directory, exclude retained workspaces to avoid running out of disk space
|
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||||
if [ -d "${sharedFolder}cache" ]; then
|
|
||||||
cp -a ${sharedFolder}cache/. /github/workspace/orchestrator-cache/cache/ || true
|
|
||||||
fi
|
|
||||||
# Copy test files from /data/ root to workspace for test assertions
|
|
||||||
# This allows tests to write files to /data/ and have them available in the workspace
|
|
||||||
find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/orchestrator-cache/ \\; || true
|
|
||||||
`;
|
`;
|
||||||
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
||||||
flag: 'w',
|
flag: 'w',
|
||||||
});
|
});
|
||||||
|
|
||||||
if (Orchestrator.buildParameters.orchestratorDebug) {
|
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||||
OrchestratorLogger.log(`Running local-docker: \n ${fileContents}`);
|
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fs.existsSync(`${workspace}/orchestrator-cache`)) {
|
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
|
||||||
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache && du -sh ${workspace}/orchestrator-cache`);
|
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
|
||||||
}
|
}
|
||||||
const exitCode = await Docker.run(
|
const exitCode = await Docker.run(
|
||||||
image,
|
image,
|
||||||
@@ -193,4 +160,4 @@ find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/w
|
|||||||
return myOutput;
|
return myOutput;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default LocalDockerOrchestrator;
|
export default LocalDockerCloudRunner;
|
||||||
+36
-159
@@ -2,22 +2,21 @@ import * as k8s from '@kubernetes/client-node';
|
|||||||
import { BuildParameters } from '../../..';
|
import { BuildParameters } from '../../..';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import { ProviderInterface } from '../provider-interface';
|
import { ProviderInterface } from '../provider-interface';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import KubernetesStorage from './kubernetes-storage';
|
import KubernetesStorage from './kubernetes-storage';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
import KubernetesTaskRunner from './kubernetes-task-runner';
|
||||||
import KubernetesSecret from './kubernetes-secret';
|
import KubernetesSecret from './kubernetes-secret';
|
||||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
||||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
import KubernetesServiceAccount from './kubernetes-service-account';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
import { CoreV1Api } from '@kubernetes/client-node';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { ProviderResource } from '../provider-resource';
|
import { ProviderResource } from '../provider-resource';
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
import { ProviderWorkflow } from '../provider-workflow';
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||||
import { KubernetesRole } from './kubernetes-role';
|
import { KubernetesRole } from './kubernetes-role';
|
||||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||||
import ResourceTracking from '../../services/core/resource-tracking';
|
|
||||||
|
|
||||||
class Kubernetes implements ProviderInterface {
|
class Kubernetes implements ProviderInterface {
|
||||||
public static Instance: Kubernetes;
|
public static Instance: Kubernetes;
|
||||||
@@ -38,6 +37,7 @@ class Kubernetes implements ProviderInterface {
|
|||||||
public serviceAccountName: string = '';
|
public serviceAccountName: string = '';
|
||||||
public ip: string = '';
|
public ip: string = '';
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-unused-vars
|
||||||
constructor(buildParameters: BuildParameters) {
|
constructor(buildParameters: BuildParameters) {
|
||||||
Kubernetes.Instance = this;
|
Kubernetes.Instance = this;
|
||||||
this.kubeConfig = new k8s.KubeConfig();
|
this.kubeConfig = new k8s.KubeConfig();
|
||||||
@@ -46,8 +46,8 @@ class Kubernetes implements ProviderInterface {
|
|||||||
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
|
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
|
||||||
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
||||||
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
|
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
|
||||||
this.namespace = buildParameters.containerNamespace ? buildParameters.containerNamespace : 'default';
|
this.namespace = 'default';
|
||||||
OrchestratorLogger.log('Loaded default Kubernetes configuration for this environment');
|
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||||
}
|
}
|
||||||
|
|
||||||
async PushLogUpdate(logs: string) {
|
async PushLogUpdate(logs: string) {
|
||||||
@@ -63,7 +63,7 @@ class Kubernetes implements ProviderInterface {
|
|||||||
|
|
||||||
// logs to base64
|
// logs to base64
|
||||||
logs = Buffer.from(logs).toString('base64');
|
logs = Buffer.from(logs).toString('base64');
|
||||||
const response = await OrchestratorSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
|
const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
|
||||||
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
|
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,19 +133,16 @@ class Kubernetes implements ProviderInterface {
|
|||||||
commands: string,
|
commands: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
try {
|
try {
|
||||||
OrchestratorLogger.log('Orchestrator K8s workflow!');
|
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
|
||||||
ResourceTracking.logAllocationSummary('k8s workflow');
|
|
||||||
await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)');
|
|
||||||
await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)');
|
|
||||||
|
|
||||||
// Setup
|
// Setup
|
||||||
const id =
|
const id =
|
||||||
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
|
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
|
||||||
? Orchestrator.lockedWorkspace
|
? CloudRunner.lockedWorkspace
|
||||||
: this.buildParameters.buildGuid;
|
: this.buildParameters.buildGuid;
|
||||||
this.pvcName = `unity-builder-pvc-${id}`;
|
this.pvcName = `unity-builder-pvc-${id}`;
|
||||||
await KubernetesStorage.createPersistentVolumeClaim(
|
await KubernetesStorage.createPersistentVolumeClaim(
|
||||||
@@ -159,134 +156,14 @@ class Kubernetes implements ProviderInterface {
|
|||||||
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
||||||
this.containerName = `main`;
|
this.containerName = `main`;
|
||||||
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
||||||
|
|
||||||
// For tests, clean up old images before creating job to free space for image pull
|
|
||||||
// IMPORTANT: Preserve the Unity image to avoid re-pulling it
|
|
||||||
if (process.env['orchestratorTests'] === 'true') {
|
|
||||||
try {
|
|
||||||
OrchestratorLogger.log('Cleaning up old images in k3d node before pulling new image...');
|
|
||||||
const { OrchestratorSystem: OrchestratorSystemModule } = await import(
|
|
||||||
'../../services/core/orchestrator-system'
|
|
||||||
);
|
|
||||||
|
|
||||||
// Aggressive cleanup: remove stopped containers and non-Unity images
|
|
||||||
// IMPORTANT: Preserve Unity images (unityci/editor) to avoid re-pulling the 3.9GB image
|
|
||||||
const K3D_NODE_CONTAINERS = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
|
|
||||||
const cleanupCommands: string[] = [];
|
|
||||||
|
|
||||||
for (const NODE of K3D_NODE_CONTAINERS) {
|
|
||||||
// Remove all stopped containers (this frees runtime space but keeps images)
|
|
||||||
cleanupCommands.push(
|
|
||||||
`docker exec ${NODE} sh -c "crictl rm --all 2>/dev/null || true" || true`,
|
|
||||||
`docker exec ${NODE} sh -c "for img in $(crictl images -q 2>/dev/null); do repo=$(crictl inspecti $img --format '{{.repo}}' 2>/dev/null || echo ''); if echo "$repo" | grep -qvE 'unityci/editor|unity'; then crictl rmi $img 2>/dev/null || true; fi; done" || true`,
|
|
||||||
`docker exec ${NODE} sh -c "crictl rmi --prune 2>/dev/null || true" || true`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const cmd of cleanupCommands) {
|
|
||||||
try {
|
|
||||||
await OrchestratorSystemModule.Run(cmd, true, true);
|
|
||||||
} catch (cmdError) {
|
|
||||||
// Ignore individual command failures - cleanup is best effort
|
|
||||||
OrchestratorLogger.log(`Cleanup command failed (non-fatal): ${cmdError}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
OrchestratorLogger.log('Cleanup completed (containers and non-Unity images removed, Unity images preserved)');
|
|
||||||
} catch (cleanupError) {
|
|
||||||
OrchestratorLogger.logWarning(`Failed to cleanup images before job creation: ${cleanupError}`);
|
|
||||||
|
|
||||||
// Continue anyway - image might already be cached
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let output = '';
|
let output = '';
|
||||||
try {
|
try {
|
||||||
// Before creating the job, verify we have the Unity image cached on the agent node
|
CloudRunnerLogger.log('Job does not exist');
|
||||||
// If not cached, try to ensure it's available to avoid disk pressure during pull
|
|
||||||
if (process.env['orchestratorTests'] === 'true' && image.includes('unityci/editor')) {
|
|
||||||
try {
|
|
||||||
const { OrchestratorSystem: OrchestratorSystemModule2 } = await import(
|
|
||||||
'../../services/core/orchestrator-system'
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check if image is cached on agent node (where pods run)
|
|
||||||
const agentImageCheck = await OrchestratorSystemModule2.Run(
|
|
||||||
`docker exec k3d-unity-builder-agent-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
|
|
||||||
true,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (agentImageCheck.includes('not_cached')) {
|
|
||||||
// Check if image is on server node
|
|
||||||
const serverImageCheck = await OrchestratorSystemModule2.Run(
|
|
||||||
`docker exec k3d-unity-builder-server-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
|
|
||||||
true,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check available disk space on agent node
|
|
||||||
const diskInfo = await OrchestratorSystemModule2.Run(
|
|
||||||
'docker exec k3d-unity-builder-agent-0 sh -c "df -h /var/lib/rancher/k3s 2>/dev/null | tail -1 || df -h / 2>/dev/null | tail -1 || echo unknown" || echo unknown',
|
|
||||||
true,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
OrchestratorLogger.logWarning(
|
|
||||||
`Unity image not cached on agent node (where pods run). Server node: ${
|
|
||||||
serverImageCheck.includes('cached') ? 'has image' : 'no image'
|
|
||||||
}. Disk info: ${diskInfo.trim()}. Pod will attempt to pull image (3.9GB) which may fail due to disk pressure.`,
|
|
||||||
);
|
|
||||||
|
|
||||||
// If image is on server but not agent, log a warning
|
|
||||||
// NOTE: We don't attempt to pull here because:
|
|
||||||
// 1. Pulling a 3.9GB image can take several minutes and block the test
|
|
||||||
// 2. If there's not enough disk space, the pull will hang indefinitely
|
|
||||||
// 3. The pod will attempt to pull during scheduling anyway
|
|
||||||
// 4. If the pull fails, Kubernetes will provide proper error messages
|
|
||||||
if (serverImageCheck.includes('cached')) {
|
|
||||||
OrchestratorLogger.logWarning(
|
|
||||||
'Unity image exists on server node but not agent node. Pod will attempt to pull during scheduling. If pull fails due to disk pressure, ensure cleanup runs before this test.',
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
// Image not on either node - check if we have enough space to pull
|
|
||||||
// Extract available space from disk info
|
|
||||||
const availableSpaceMatch = diskInfo.match(/(\d+(?:\.\d+)?)\s*([gkm]?i?b)/i);
|
|
||||||
if (availableSpaceMatch) {
|
|
||||||
const availableValue = Number.parseFloat(availableSpaceMatch[1]);
|
|
||||||
const availableUnit = availableSpaceMatch[2].toUpperCase();
|
|
||||||
let availableGB = availableValue;
|
|
||||||
|
|
||||||
if (availableUnit.includes('M')) {
|
|
||||||
availableGB = availableValue / 1024;
|
|
||||||
} else if (availableUnit.includes('K')) {
|
|
||||||
availableGB = availableValue / (1024 * 1024);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unity image is ~3.9GB, need at least 4.5GB to be safe
|
|
||||||
if (availableGB < 4.5) {
|
|
||||||
OrchestratorLogger.logWarning(
|
|
||||||
`CRITICAL: Unity image not cached and only ${availableGB.toFixed(
|
|
||||||
2,
|
|
||||||
)}GB available. Image pull (3.9GB) will likely fail. Consider running cleanup or ensuring pre-pull step succeeds.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
OrchestratorLogger.log('Unity image is cached on agent node - pod should start without pulling');
|
|
||||||
}
|
|
||||||
} catch (checkError) {
|
|
||||||
// Ignore check errors - continue with job creation
|
|
||||||
OrchestratorLogger.logWarning(`Failed to verify Unity image cache: ${checkError}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
OrchestratorLogger.log('Job does not exist');
|
|
||||||
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
|
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||||
OrchestratorLogger.log('Watching pod until running');
|
CloudRunnerLogger.log('Watching pod until running');
|
||||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||||
|
|
||||||
OrchestratorLogger.log('Pod is running');
|
CloudRunnerLogger.log('Pod is running');
|
||||||
output += await KubernetesTaskRunner.runTask(
|
output += await KubernetesTaskRunner.runTask(
|
||||||
this.kubeConfig,
|
this.kubeConfig,
|
||||||
this.kubeClient,
|
this.kubeClient,
|
||||||
@@ -296,9 +173,9 @@ class Kubernetes implements ProviderInterface {
|
|||||||
this.namespace,
|
this.namespace,
|
||||||
);
|
);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
OrchestratorLogger.log(`error running k8s workflow ${error}`);
|
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
|
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
|
||||||
.map((x) => {
|
.map((x) => {
|
||||||
@@ -321,7 +198,7 @@ class Kubernetes implements ProviderInterface {
|
|||||||
|
|
||||||
return output;
|
return output;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
OrchestratorLogger.log('Running job failed');
|
CloudRunnerLogger.log('Running job failed');
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
core.error(JSON.stringify(error, undefined, 4));
|
||||||
|
|
||||||
// await this.cleanupTaskResources();
|
// await this.cleanupTaskResources();
|
||||||
@@ -334,8 +211,8 @@ class Kubernetes implements ProviderInterface {
|
|||||||
image: string,
|
image: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
) {
|
) {
|
||||||
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||||
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
|
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
|
||||||
@@ -359,8 +236,8 @@ class Kubernetes implements ProviderInterface {
|
|||||||
image: string,
|
image: string,
|
||||||
mountdir: string,
|
mountdir: string,
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
) {
|
) {
|
||||||
for (let index = 0; index < 3; index++) {
|
for (let index = 0; index < 3; index++) {
|
||||||
try {
|
try {
|
||||||
@@ -385,13 +262,13 @@ class Kubernetes implements ProviderInterface {
|
|||||||
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
||||||
|
|
||||||
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||||
OrchestratorLogger.log(`Build job created`);
|
CloudRunnerLogger.log(`Build job created`);
|
||||||
await new Promise((promise) => setTimeout(promise, 5000));
|
await new Promise((promise) => setTimeout(promise, 5000));
|
||||||
OrchestratorLogger.log('Job created');
|
CloudRunnerLogger.log('Job created');
|
||||||
|
|
||||||
return result.body.metadata?.name;
|
return result.body.metadata?.name;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
OrchestratorLogger.log(`Error occured creating job: ${error}`);
|
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -403,26 +280,26 @@ class Kubernetes implements ProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async cleanupTaskResources() {
|
async cleanupTaskResources() {
|
||||||
OrchestratorLogger.log('cleaning up');
|
CloudRunnerLogger.log('cleaning up');
|
||||||
try {
|
try {
|
||||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||||
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
||||||
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
OrchestratorLogger.log(`Failed to cleanup`);
|
CloudRunnerLogger.log(`Failed to cleanup`);
|
||||||
if (error.response.body.reason !== `NotFound`) {
|
if (error.response.body.reason !== `NotFound`) {
|
||||||
OrchestratorLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
OrchestratorLogger.log(`Failed to cleanup secret`);
|
CloudRunnerLogger.log(`Failed to cleanup secret`);
|
||||||
OrchestratorLogger.log(error.response.body.reason);
|
CloudRunnerLogger.log(error.response.body.reason);
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log('cleaned up Secret, Job and Pod');
|
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
|
||||||
OrchestratorLogger.log('cleaning up finished');
|
CloudRunnerLogger.log('cleaning up finished');
|
||||||
}
|
}
|
||||||
|
|
||||||
async cleanupWorkflow(
|
async cleanupWorkflow(
|
||||||
@@ -435,14 +312,14 @@ class Kubernetes implements ProviderInterface {
|
|||||||
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`deleting PVC`);
|
CloudRunnerLogger.log(`deleting PVC`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||||
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
||||||
OrchestratorLogger.log('cleaned up PVC and Service Account');
|
CloudRunnerLogger.log('cleaned up PVC and Service Account');
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
OrchestratorLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
||||||
|
import BuildParameters from '../../../build-parameters';
|
||||||
|
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
||||||
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
|
import CloudRunner from '../../cloud-runner';
|
||||||
|
|
||||||
|
class KubernetesJobSpecFactory {
|
||||||
|
static getJobSpec(
|
||||||
|
command: string,
|
||||||
|
image: string,
|
||||||
|
mountdir: string,
|
||||||
|
workingDirectory: string,
|
||||||
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
|
secrets: CloudRunnerSecret[],
|
||||||
|
buildGuid: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
secretName: string,
|
||||||
|
pvcName: string,
|
||||||
|
jobName: string,
|
||||||
|
k8s: any,
|
||||||
|
containerName: string,
|
||||||
|
ip: string = '',
|
||||||
|
) {
|
||||||
|
const endpointEnvNames = new Set([
|
||||||
|
'AWS_S3_ENDPOINT',
|
||||||
|
'AWS_ENDPOINT',
|
||||||
|
'AWS_CLOUD_FORMATION_ENDPOINT',
|
||||||
|
'AWS_ECS_ENDPOINT',
|
||||||
|
'AWS_KINESIS_ENDPOINT',
|
||||||
|
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||||
|
'INPUT_AWSS3ENDPOINT',
|
||||||
|
'INPUT_AWSENDPOINT',
|
||||||
|
]);
|
||||||
|
const adjustedEnvironment = environment.map((x) => {
|
||||||
|
let value = x.value;
|
||||||
|
if (
|
||||||
|
typeof value === 'string' &&
|
||||||
|
endpointEnvNames.has(x.name) &&
|
||||||
|
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
|
||||||
|
) {
|
||||||
|
value = value
|
||||||
|
.replace('http://localhost', 'http://host.k3d.internal')
|
||||||
|
.replace('http://127.0.0.1', 'http://host.k3d.internal');
|
||||||
|
}
|
||||||
|
return { name: x.name, value } as CloudRunnerEnvironmentVariable;
|
||||||
|
});
|
||||||
|
|
||||||
|
const job = new k8s.V1Job();
|
||||||
|
job.apiVersion = 'batch/v1';
|
||||||
|
job.kind = 'Job';
|
||||||
|
job.metadata = {
|
||||||
|
name: jobName,
|
||||||
|
labels: {
|
||||||
|
app: 'unity-builder',
|
||||||
|
buildGuid,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
job.spec = {
|
||||||
|
ttlSecondsAfterFinished: 9999,
|
||||||
|
backoffLimit: 0,
|
||||||
|
template: {
|
||||||
|
spec: {
|
||||||
|
volumes: [
|
||||||
|
{
|
||||||
|
name: 'build-mount',
|
||||||
|
persistentVolumeClaim: {
|
||||||
|
claimName: pvcName,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
containers: [
|
||||||
|
{
|
||||||
|
ttlSecondsAfterFinished: 9999,
|
||||||
|
name: containerName,
|
||||||
|
image,
|
||||||
|
command: ['/bin/sh'],
|
||||||
|
args: [
|
||||||
|
'-c',
|
||||||
|
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
|
||||||
|
],
|
||||||
|
|
||||||
|
workingDir: `${workingDirectory}`,
|
||||||
|
resources: {
|
||||||
|
requests: {
|
||||||
|
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
|
||||||
|
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
env: [
|
||||||
|
...adjustedEnvironment.map((x) => {
|
||||||
|
const environmentVariable = new V1EnvVar();
|
||||||
|
environmentVariable.name = x.name;
|
||||||
|
environmentVariable.value = x.value;
|
||||||
|
|
||||||
|
return environmentVariable;
|
||||||
|
}),
|
||||||
|
...secrets.map((x) => {
|
||||||
|
const secret = new V1EnvVarSource();
|
||||||
|
secret.secretKeyRef = new V1SecretKeySelector();
|
||||||
|
secret.secretKeyRef.key = x.ParameterKey;
|
||||||
|
secret.secretKeyRef.name = secretName;
|
||||||
|
const environmentVariable = new V1EnvVar();
|
||||||
|
environmentVariable.name = x.EnvironmentVariable;
|
||||||
|
environmentVariable.valueFrom = secret;
|
||||||
|
|
||||||
|
return environmentVariable;
|
||||||
|
}),
|
||||||
|
{ name: 'LOG_SERVICE_IP', value: ip },
|
||||||
|
],
|
||||||
|
volumeMounts: [
|
||||||
|
{
|
||||||
|
name: 'build-mount',
|
||||||
|
mountPath: `${mountdir}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
lifecycle: {
|
||||||
|
preStop: {
|
||||||
|
exec: {
|
||||||
|
command: [
|
||||||
|
`wait 60s;
|
||||||
|
cd /data/builder/action/steps;
|
||||||
|
chmod +x /return_license.sh;
|
||||||
|
/return_license.sh;`,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
restartPolicy: 'Never',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
if (process.env['CLOUD_RUNNER_MINIKUBE']) {
|
||||||
|
job.spec.template.spec.volumes[0] = {
|
||||||
|
name: 'build-mount',
|
||||||
|
hostPath: {
|
||||||
|
path: `/data`,
|
||||||
|
type: `Directory`,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
|
||||||
|
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export default KubernetesJobSpecFactory;
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
|
import { CoreV1Api } from '@kubernetes/client-node';
|
||||||
|
class KubernetesPods {
|
||||||
|
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||||
|
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
|
||||||
|
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
|
||||||
|
const phase = pods[0]?.status?.phase || 'undefined status';
|
||||||
|
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
|
||||||
|
if (phase === `Failed`) {
|
||||||
|
throw new Error(`K8s pod failed`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return running;
|
||||||
|
}
|
||||||
|
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||||
|
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
|
||||||
|
const phase = pods?.status?.phase || 'undefined status';
|
||||||
|
|
||||||
|
return phase;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default KubernetesPods;
|
||||||
+7
-7
@@ -1,12 +1,12 @@
|
|||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
import { CoreV1Api } from '@kubernetes/client-node';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import * as k8s from '@kubernetes/client-node';
|
import * as k8s from '@kubernetes/client-node';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import * as base64 from 'base-64';
|
import * as base64 from 'base-64';
|
||||||
|
|
||||||
class KubernetesSecret {
|
class KubernetesSecret {
|
||||||
static async createSecret(
|
static async createSecret(
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
secretName: string,
|
secretName: string,
|
||||||
namespace: string,
|
namespace: string,
|
||||||
kubeClient: CoreV1Api,
|
kubeClient: CoreV1Api,
|
||||||
@@ -23,20 +23,20 @@ class KubernetesSecret {
|
|||||||
for (const buildSecret of secrets) {
|
for (const buildSecret of secrets) {
|
||||||
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
||||||
}
|
}
|
||||||
OrchestratorLogger.log(`Creating secret: ${secretName}`);
|
CloudRunnerLogger.log(`Creating secret: ${secretName}`);
|
||||||
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
|
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
|
||||||
const mappedSecrets = existingSecrets.body.items.map((x) => {
|
const mappedSecrets = existingSecrets.body.items.map((x) => {
|
||||||
return x.metadata?.name || `no name`;
|
return x.metadata?.name || `no name`;
|
||||||
});
|
});
|
||||||
|
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
|
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
|
||||||
);
|
);
|
||||||
await new Promise((promise) => setTimeout(promise, 15000));
|
await new Promise((promise) => setTimeout(promise, 15000));
|
||||||
await kubeClient.createNamespacedSecret(namespace, secret);
|
await kubeClient.createNamespacedSecret(namespace, secret);
|
||||||
OrchestratorLogger.log('Created secret');
|
CloudRunnerLogger.log('Created secret');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
OrchestratorLogger.log(`Created secret failed ${error}`);
|
CloudRunnerLogger.log(`Created secret failed ${error}`);
|
||||||
throw new Error(`Failed to create kubernetes secret`);
|
throw new Error(`Failed to create kubernetes secret`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import { waitUntil } from 'async-wait-until';
|
||||||
|
import * as core from '@actions/core';
|
||||||
|
import * as k8s from '@kubernetes/client-node';
|
||||||
|
import BuildParameters from '../../../build-parameters';
|
||||||
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
|
import { IncomingMessage } from 'node:http';
|
||||||
|
import GitHub from '../../../github';
|
||||||
|
|
||||||
|
class KubernetesStorage {
|
||||||
|
public static async createPersistentVolumeClaim(
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
pvcName: string,
|
||||||
|
kubeClient: k8s.CoreV1Api,
|
||||||
|
namespace: string,
|
||||||
|
) {
|
||||||
|
if (buildParameters.kubeVolume !== ``) {
|
||||||
|
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
|
||||||
|
pvcName = buildParameters.kubeVolume;
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
|
||||||
|
const pvcList = allPvc.map((x) => x.metadata?.name);
|
||||||
|
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
|
||||||
|
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
|
||||||
|
if (pvcList.includes(pvcName)) {
|
||||||
|
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
|
||||||
|
if (GitHub.githubInputEnabled) {
|
||||||
|
core.setOutput('volume', pvcName);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
|
||||||
|
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
|
||||||
|
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||||
|
try {
|
||||||
|
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
|
||||||
|
} catch (error) {
|
||||||
|
core.error('Failed to get PVC phase');
|
||||||
|
core.error(JSON.stringify(error, undefined, 4));
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
||||||
|
try {
|
||||||
|
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
||||||
|
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
|
||||||
|
await waitUntil(
|
||||||
|
async () => {
|
||||||
|
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeout: 750000,
|
||||||
|
intervalBetweenAttempts: 15000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch (error: any) {
|
||||||
|
core.error('Failed to watch PVC');
|
||||||
|
core.error(error.toString());
|
||||||
|
core.error(
|
||||||
|
`PVC Body: ${JSON.stringify(
|
||||||
|
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
|
||||||
|
undefined,
|
||||||
|
4,
|
||||||
|
)}`,
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async createPVC(
|
||||||
|
pvcName: string,
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
kubeClient: k8s.CoreV1Api,
|
||||||
|
namespace: string,
|
||||||
|
) {
|
||||||
|
const pvc = new k8s.V1PersistentVolumeClaim();
|
||||||
|
pvc.apiVersion = 'v1';
|
||||||
|
pvc.kind = 'PersistentVolumeClaim';
|
||||||
|
pvc.metadata = {
|
||||||
|
name: pvcName,
|
||||||
|
};
|
||||||
|
pvc.spec = {
|
||||||
|
accessModes: ['ReadWriteOnce'],
|
||||||
|
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
|
||||||
|
resources: {
|
||||||
|
requests: {
|
||||||
|
storage: buildParameters.kubeVolumeSize,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async handleResult(
|
||||||
|
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
|
||||||
|
kubeClient: k8s.CoreV1Api,
|
||||||
|
namespace: string,
|
||||||
|
pvcName: string,
|
||||||
|
) {
|
||||||
|
const name = result.body.metadata?.name || '';
|
||||||
|
CloudRunnerLogger.log(`PVC ${name} created`);
|
||||||
|
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
|
||||||
|
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`);
|
||||||
|
core.setOutput('volume', pvcName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default KubernetesStorage;
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
|
||||||
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
|
import { waitUntil } from 'async-wait-until';
|
||||||
|
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||||
|
import CloudRunner from '../../cloud-runner';
|
||||||
|
import KubernetesPods from './kubernetes-pods';
|
||||||
|
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
||||||
|
|
||||||
|
class KubernetesTaskRunner {
|
||||||
|
static readonly maxRetry: number = 3;
|
||||||
|
static lastReceivedMessage: string = ``;
|
||||||
|
|
||||||
|
static async runTask(
|
||||||
|
kubeConfig: KubeConfig,
|
||||||
|
kubeClient: CoreV1Api,
|
||||||
|
jobName: string,
|
||||||
|
podName: string,
|
||||||
|
containerName: string,
|
||||||
|
namespace: string,
|
||||||
|
) {
|
||||||
|
let output = '';
|
||||||
|
let shouldReadLogs = true;
|
||||||
|
let shouldCleanup = true;
|
||||||
|
let retriesAfterFinish = 0;
|
||||||
|
// eslint-disable-next-line no-constant-condition
|
||||||
|
while (true) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
|
CloudRunnerLogger.log(
|
||||||
|
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
|
||||||
|
);
|
||||||
|
let extraFlags = ``;
|
||||||
|
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
|
||||||
|
? ` -f -c ${containerName}`
|
||||||
|
: ` --previous`;
|
||||||
|
|
||||||
|
const callback = (outputChunk: string) => {
|
||||||
|
output += outputChunk;
|
||||||
|
|
||||||
|
// split output chunk and handle per line
|
||||||
|
for (const chunk of outputChunk.split(`\n`)) {
|
||||||
|
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
||||||
|
chunk,
|
||||||
|
shouldReadLogs,
|
||||||
|
shouldCleanup,
|
||||||
|
output,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
|
||||||
|
} catch (error: any) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
|
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
||||||
|
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
|
||||||
|
if (continueStreaming) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
||||||
|
retriesAfterFinish++;
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
||||||
|
CloudRunnerLogger.log('end of log stream');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||||
|
let waitComplete: boolean = false;
|
||||||
|
let message = ``;
|
||||||
|
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
||||||
|
await waitUntil(
|
||||||
|
async () => {
|
||||||
|
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||||
|
const phase = status?.body.status?.phase;
|
||||||
|
waitComplete = phase !== 'Pending';
|
||||||
|
message = `Phase:${status.body.status?.phase} \n Reason:${
|
||||||
|
status.body.status?.conditions?.[0].reason || ''
|
||||||
|
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
|
||||||
|
|
||||||
|
// CloudRunnerLogger.log(
|
||||||
|
// JSON.stringify(
|
||||||
|
// (await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||||
|
// .map((x) => {
|
||||||
|
// return {
|
||||||
|
// message: x.message || ``,
|
||||||
|
// name: x.metadata.name || ``,
|
||||||
|
// reason: x.reason || ``,
|
||||||
|
// };
|
||||||
|
// })
|
||||||
|
// .filter((x) => x.name.includes(podName)),
|
||||||
|
// undefined,
|
||||||
|
// 4,
|
||||||
|
// ),
|
||||||
|
// );
|
||||||
|
if (waitComplete || phase !== 'Pending') return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeout: 2000000,
|
||||||
|
intervalBetweenAttempts: 15000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if (!waitComplete) {
|
||||||
|
CloudRunnerLogger.log(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
return waitComplete;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default KubernetesTaskRunner;
|
||||||
+40
-18
@@ -1,14 +1,13 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
|
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { ProviderInterface } from '../provider-interface';
|
import { ProviderInterface } from '../provider-interface';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import { ProviderResource } from '../provider-resource';
|
import { ProviderResource } from '../provider-resource';
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
import { ProviderWorkflow } from '../provider-workflow';
|
||||||
import { quote } from 'shell-quote';
|
|
||||||
|
|
||||||
class LocalOrchestrator implements ProviderInterface {
|
class LocalCloudRunner implements ProviderInterface {
|
||||||
listResources(): Promise<ProviderResource[]> {
|
listResources(): Promise<ProviderResource[]> {
|
||||||
throw new Error('Method not implemented.');
|
throw new Error('Method not implemented.');
|
||||||
}
|
}
|
||||||
@@ -59,29 +58,52 @@ class LocalOrchestrator implements ProviderInterface {
|
|||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
OrchestratorLogger.log(image);
|
CloudRunnerLogger.log(image);
|
||||||
OrchestratorLogger.log(buildGuid);
|
CloudRunnerLogger.log(buildGuid);
|
||||||
OrchestratorLogger.log(commands);
|
CloudRunnerLogger.log(commands);
|
||||||
|
|
||||||
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
|
||||||
if (process.platform === 'win32') {
|
if (process.platform === 'win32') {
|
||||||
const inline = commands
|
// Properly escape the command string for embedding in a double-quoted bash string.
|
||||||
|
// Order matters: backslashes must be escaped first to avoid double-escaping.
|
||||||
|
const escapeForBashDoubleQuotes = (stringValue: string): string => {
|
||||||
|
return stringValue
|
||||||
|
.replace(/\\/g, '\\\\') // Escape backslashes first
|
||||||
|
.replace(/\$/g, '\\$') // Escape dollar signs to prevent variable expansion
|
||||||
|
.replace(/`/g, '\\`') // Escape backticks to prevent command substitution
|
||||||
|
.replace(/"/g, '\\"'); // Escape double quotes
|
||||||
|
};
|
||||||
|
|
||||||
|
// Split commands by newlines and escape each line
|
||||||
|
const lines = commands
|
||||||
.replace(/\r/g, '')
|
.replace(/\r/g, '')
|
||||||
.split('\n')
|
.split('\n')
|
||||||
.filter((x) => x.trim().length > 0)
|
.filter((x) => x.trim().length > 0)
|
||||||
.join(' ; ');
|
.map((line) => escapeForBashDoubleQuotes(line));
|
||||||
|
|
||||||
// Use shell-quote to properly escape the command string, preventing command injection
|
// Join with semicolons, but don't add semicolon after control flow keywords
|
||||||
const bashWrapped = `bash -lc ${quote([inline])}`;
|
// Control flow keywords that shouldn't be followed by semicolons: then, else, do, fi, done, esac
|
||||||
|
const controlFlowKeywords = /\b(then|else|do|fi|done|esac)\s*$/;
|
||||||
|
const inline = lines
|
||||||
|
.map((line, index) => {
|
||||||
|
// Don't add semicolon if this line ends with a control flow keyword
|
||||||
|
if (controlFlowKeywords.test(line.trim()) || index === lines.length - 1) {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
return await OrchestratorSystem.Run(bashWrapped);
|
return `${line} ;`;
|
||||||
|
})
|
||||||
|
.join(' ');
|
||||||
|
const bashWrapped = `bash -lc "${inline}"`;
|
||||||
|
|
||||||
|
return await CloudRunnerSystem.Run(bashWrapped);
|
||||||
}
|
}
|
||||||
|
|
||||||
return await OrchestratorSystem.Run(commands);
|
return await CloudRunnerSystem.Run(commands);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default LocalOrchestrator;
|
export default LocalCloudRunner;
|
||||||
+20
-20
@@ -2,7 +2,7 @@ import { exec } from 'child_process';
|
|||||||
import { promisify } from 'util';
|
import { promisify } from 'util';
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser';
|
import { GitHubUrlInfo, generateCacheKey } from './provider-url-parser';
|
||||||
|
|
||||||
const execAsync = promisify(exec);
|
const execAsync = promisify(exec);
|
||||||
@@ -32,7 +32,7 @@ export class ProviderGitManager {
|
|||||||
private static ensureCacheDir(): void {
|
private static ensureCacheDir(): void {
|
||||||
if (!fs.existsSync(this.CACHE_DIR)) {
|
if (!fs.existsSync(this.CACHE_DIR)) {
|
||||||
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
|
fs.mkdirSync(this.CACHE_DIR, { recursive: true });
|
||||||
OrchestratorLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`);
|
CloudRunnerLogger.log(`Created provider cache directory: ${this.CACHE_DIR}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,15 +69,15 @@ export class ProviderGitManager {
|
|||||||
|
|
||||||
// Remove existing directory if it exists
|
// Remove existing directory if it exists
|
||||||
if (fs.existsSync(localPath)) {
|
if (fs.existsSync(localPath)) {
|
||||||
OrchestratorLogger.log(`Removing existing directory: ${localPath}`);
|
CloudRunnerLogger.log(`Removing existing directory: ${localPath}`);
|
||||||
fs.rmSync(localPath, { recursive: true, force: true });
|
fs.rmSync(localPath, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
OrchestratorLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
|
CloudRunnerLogger.log(`Cloning repository: ${urlInfo.url} to ${localPath}`);
|
||||||
|
|
||||||
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
|
const cloneCommand = `git clone --depth 1 --branch ${urlInfo.branch} ${urlInfo.url} "${localPath}"`;
|
||||||
OrchestratorLogger.log(`Executing: ${cloneCommand}`);
|
CloudRunnerLogger.log(`Executing: ${cloneCommand}`);
|
||||||
|
|
||||||
const { stderr } = await execAsync(cloneCommand, {
|
const { stderr } = await execAsync(cloneCommand, {
|
||||||
timeout: this.GIT_TIMEOUT,
|
timeout: this.GIT_TIMEOUT,
|
||||||
@@ -85,10 +85,10 @@ export class ProviderGitManager {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (stderr && !stderr.includes('warning')) {
|
if (stderr && !stderr.includes('warning')) {
|
||||||
OrchestratorLogger.log(`Git clone stderr: ${stderr}`);
|
CloudRunnerLogger.log(`Git clone stderr: ${stderr}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
OrchestratorLogger.log(`Successfully cloned repository to: ${localPath}`);
|
CloudRunnerLogger.log(`Successfully cloned repository to: ${localPath}`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
@@ -96,7 +96,7 @@ export class ProviderGitManager {
|
|||||||
};
|
};
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
|
const errorMessage = `Failed to clone repository ${urlInfo.url}: ${error.message}`;
|
||||||
OrchestratorLogger.log(`Error: ${errorMessage}`);
|
CloudRunnerLogger.log(`Error: ${errorMessage}`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
@@ -123,7 +123,7 @@ export class ProviderGitManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
OrchestratorLogger.log(`Updating repository: ${localPath}`);
|
CloudRunnerLogger.log(`Updating repository: ${localPath}`);
|
||||||
|
|
||||||
// Fetch latest changes
|
// Fetch latest changes
|
||||||
await execAsync('git fetch origin', {
|
await execAsync('git fetch origin', {
|
||||||
@@ -141,7 +141,7 @@ export class ProviderGitManager {
|
|||||||
statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
|
statusOutput.includes('Your branch is behind') || statusOutput.includes('can be fast-forwarded');
|
||||||
|
|
||||||
if (hasUpdates) {
|
if (hasUpdates) {
|
||||||
OrchestratorLogger.log(`Updates available, pulling latest changes...`);
|
CloudRunnerLogger.log(`Updates available, pulling latest changes...`);
|
||||||
|
|
||||||
// Reset to origin/branch to get latest changes
|
// Reset to origin/branch to get latest changes
|
||||||
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
|
await execAsync(`git reset --hard origin/${urlInfo.branch}`, {
|
||||||
@@ -149,14 +149,14 @@ export class ProviderGitManager {
|
|||||||
cwd: localPath,
|
cwd: localPath,
|
||||||
});
|
});
|
||||||
|
|
||||||
OrchestratorLogger.log(`Repository updated successfully`);
|
CloudRunnerLogger.log(`Repository updated successfully`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
updated: true,
|
updated: true,
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
OrchestratorLogger.log(`Repository is already up to date`);
|
CloudRunnerLogger.log(`Repository is already up to date`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
@@ -165,7 +165,7 @@ export class ProviderGitManager {
|
|||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
|
const errorMessage = `Failed to update repository ${localPath}: ${error.message}`;
|
||||||
OrchestratorLogger.log(`Error: ${errorMessage}`);
|
CloudRunnerLogger.log(`Error: ${errorMessage}`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
@@ -184,11 +184,11 @@ export class ProviderGitManager {
|
|||||||
this.ensureCacheDir();
|
this.ensureCacheDir();
|
||||||
|
|
||||||
if (this.isRepositoryCloned(urlInfo)) {
|
if (this.isRepositoryCloned(urlInfo)) {
|
||||||
OrchestratorLogger.log(`Repository already exists locally, checking for updates...`);
|
CloudRunnerLogger.log(`Repository already exists locally, checking for updates...`);
|
||||||
const updateResult = await this.updateRepository(urlInfo);
|
const updateResult = await this.updateRepository(urlInfo);
|
||||||
|
|
||||||
if (!updateResult.success) {
|
if (!updateResult.success) {
|
||||||
OrchestratorLogger.log(`Failed to update repository, attempting fresh clone...`);
|
CloudRunnerLogger.log(`Failed to update repository, attempting fresh clone...`);
|
||||||
const cloneResult = await this.cloneRepository(urlInfo);
|
const cloneResult = await this.cloneRepository(urlInfo);
|
||||||
if (!cloneResult.success) {
|
if (!cloneResult.success) {
|
||||||
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
|
throw new Error(`Failed to ensure repository availability: ${cloneResult.error}`);
|
||||||
@@ -199,7 +199,7 @@ export class ProviderGitManager {
|
|||||||
|
|
||||||
return this.getLocalPath(urlInfo);
|
return this.getLocalPath(urlInfo);
|
||||||
} else {
|
} else {
|
||||||
OrchestratorLogger.log(`Repository not found locally, cloning...`);
|
CloudRunnerLogger.log(`Repository not found locally, cloning...`);
|
||||||
const cloneResult = await this.cloneRepository(urlInfo);
|
const cloneResult = await this.cloneRepository(urlInfo);
|
||||||
|
|
||||||
if (!cloneResult.success) {
|
if (!cloneResult.success) {
|
||||||
@@ -236,14 +236,14 @@ export class ProviderGitManager {
|
|||||||
for (const entryPoint of commonEntryPoints) {
|
for (const entryPoint of commonEntryPoints) {
|
||||||
const fullPath = path.join(localPath, entryPoint);
|
const fullPath = path.join(localPath, entryPoint);
|
||||||
if (fs.existsSync(fullPath)) {
|
if (fs.existsSync(fullPath)) {
|
||||||
OrchestratorLogger.log(`Found provider entry point: ${entryPoint}`);
|
CloudRunnerLogger.log(`Found provider entry point: ${entryPoint}`);
|
||||||
|
|
||||||
return fullPath;
|
return fullPath;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default to repository root
|
// Default to repository root
|
||||||
OrchestratorLogger.log(`No specific entry point found, using repository root`);
|
CloudRunnerLogger.log(`No specific entry point found, using repository root`);
|
||||||
|
|
||||||
return localPath;
|
return localPath;
|
||||||
}
|
}
|
||||||
@@ -266,13 +266,13 @@ export class ProviderGitManager {
|
|||||||
const stats = fs.statSync(entryPath);
|
const stats = fs.statSync(entryPath);
|
||||||
|
|
||||||
if (now - stats.mtime.getTime() > maxAge) {
|
if (now - stats.mtime.getTime() > maxAge) {
|
||||||
OrchestratorLogger.log(`Cleaning up old repository: ${entry.name}`);
|
CloudRunnerLogger.log(`Cleaning up old repository: ${entry.name}`);
|
||||||
fs.rmSync(entryPath, { recursive: true, force: true });
|
fs.rmSync(entryPath, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
OrchestratorLogger.log(`Error during cleanup: ${error.message}`);
|
CloudRunnerLogger.log(`Error during cleanup: ${error.message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+4
-4
@@ -1,6 +1,6 @@
|
|||||||
import BuildParameters from '../../build-parameters';
|
import BuildParameters from '../../build-parameters';
|
||||||
import OrchestratorEnvironmentVariable from '../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorSecret from '../options/orchestrator-secret';
|
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
||||||
import { ProviderResource } from './provider-resource';
|
import { ProviderResource } from './provider-resource';
|
||||||
import { ProviderWorkflow } from './provider-workflow';
|
import { ProviderWorkflow } from './provider-workflow';
|
||||||
|
|
||||||
@@ -35,9 +35,9 @@ export interface ProviderInterface {
|
|||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string>;
|
): Promise<string>;
|
||||||
garbageCollect(
|
garbageCollect(
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
+8
-8
@@ -1,6 +1,6 @@
|
|||||||
import { ProviderInterface } from './provider-interface';
|
import { ProviderInterface } from './provider-interface';
|
||||||
import BuildParameters from '../../build-parameters';
|
import BuildParameters from '../../build-parameters';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser';
|
import { parseProviderSource, logProviderSource, ProviderSourceInfo } from './provider-url-parser';
|
||||||
import { ProviderGitManager } from './provider-git-manager';
|
import { ProviderGitManager } from './provider-git-manager';
|
||||||
|
|
||||||
@@ -16,7 +16,7 @@ export default async function loadProvider(
|
|||||||
providerSource: string,
|
providerSource: string,
|
||||||
buildParameters: BuildParameters,
|
buildParameters: BuildParameters,
|
||||||
): Promise<ProviderInterface> {
|
): Promise<ProviderInterface> {
|
||||||
OrchestratorLogger.log(`Loading provider: ${providerSource}`);
|
CloudRunnerLogger.log(`Loading provider: ${providerSource}`);
|
||||||
|
|
||||||
// Parse the provider source to determine its type
|
// Parse the provider source to determine its type
|
||||||
const sourceInfo = parseProviderSource(providerSource);
|
const sourceInfo = parseProviderSource(providerSource);
|
||||||
@@ -29,7 +29,7 @@ export default async function loadProvider(
|
|||||||
// Handle different source types
|
// Handle different source types
|
||||||
switch (sourceInfo.type) {
|
switch (sourceInfo.type) {
|
||||||
case 'github': {
|
case 'github': {
|
||||||
OrchestratorLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
|
CloudRunnerLogger.log(`Processing GitHub repository: ${sourceInfo.owner}/${sourceInfo.repo}`);
|
||||||
|
|
||||||
// Ensure the repository is available locally
|
// Ensure the repository is available locally
|
||||||
const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
|
const localRepoPath = await ProviderGitManager.ensureRepositoryAvailable(sourceInfo);
|
||||||
@@ -37,19 +37,19 @@ export default async function loadProvider(
|
|||||||
// Get the path to the provider module within the repository
|
// Get the path to the provider module within the repository
|
||||||
modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
|
modulePath = ProviderGitManager.getProviderModulePath(sourceInfo, localRepoPath);
|
||||||
|
|
||||||
OrchestratorLogger.log(`Loading provider from: ${modulePath}`);
|
CloudRunnerLogger.log(`Loading provider from: ${modulePath}`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'local': {
|
case 'local': {
|
||||||
modulePath = sourceInfo.path;
|
modulePath = sourceInfo.path;
|
||||||
OrchestratorLogger.log(`Loading provider from local path: ${modulePath}`);
|
CloudRunnerLogger.log(`Loading provider from local path: ${modulePath}`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'npm': {
|
case 'npm': {
|
||||||
modulePath = sourceInfo.packageName;
|
modulePath = sourceInfo.packageName;
|
||||||
OrchestratorLogger.log(`Loading provider from NPM package: ${modulePath}`);
|
CloudRunnerLogger.log(`Loading provider from NPM package: ${modulePath}`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ export default async function loadProvider(
|
|||||||
};
|
};
|
||||||
|
|
||||||
modulePath = providerModuleMap[providerSource] || providerSource;
|
modulePath = providerModuleMap[providerSource] || providerSource;
|
||||||
OrchestratorLogger.log(`Loading provider from module path: ${modulePath}`);
|
CloudRunnerLogger.log(`Loading provider from module path: ${modulePath}`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,7 +111,7 @@ export default async function loadProvider(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
OrchestratorLogger.log(`Successfully loaded provider: ${providerSource}`);
|
CloudRunnerLogger.log(`Successfully loaded provider: ${providerSource}`);
|
||||||
|
|
||||||
return instance as ProviderInterface;
|
return instance as ProviderInterface;
|
||||||
}
|
}
|
||||||
+11
-11
@@ -1,4 +1,4 @@
|
|||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
|
||||||
export interface GitHubUrlInfo {
|
export interface GitHubUrlInfo {
|
||||||
type: 'github';
|
type: 'github';
|
||||||
@@ -115,24 +115,24 @@ export function isGitHubSource(source: string): boolean {
|
|||||||
* @param parsed The parsed source information
|
* @param parsed The parsed source information
|
||||||
*/
|
*/
|
||||||
export function logProviderSource(source: string, parsed: ProviderSourceInfo): void {
|
export function logProviderSource(source: string, parsed: ProviderSourceInfo): void {
|
||||||
OrchestratorLogger.log(`Provider source: ${source}`);
|
CloudRunnerLogger.log(`Provider source: ${source}`);
|
||||||
switch (parsed.type) {
|
switch (parsed.type) {
|
||||||
case 'github':
|
case 'github':
|
||||||
OrchestratorLogger.log(` Type: GitHub repository`);
|
CloudRunnerLogger.log(` Type: GitHub repository`);
|
||||||
OrchestratorLogger.log(` Owner: ${parsed.owner}`);
|
CloudRunnerLogger.log(` Owner: ${parsed.owner}`);
|
||||||
OrchestratorLogger.log(` Repository: ${parsed.repo}`);
|
CloudRunnerLogger.log(` Repository: ${parsed.repo}`);
|
||||||
OrchestratorLogger.log(` Branch: ${parsed.branch}`);
|
CloudRunnerLogger.log(` Branch: ${parsed.branch}`);
|
||||||
if (parsed.path) {
|
if (parsed.path) {
|
||||||
OrchestratorLogger.log(` Path: ${parsed.path}`);
|
CloudRunnerLogger.log(` Path: ${parsed.path}`);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'local':
|
case 'local':
|
||||||
OrchestratorLogger.log(` Type: Local path`);
|
CloudRunnerLogger.log(` Type: Local path`);
|
||||||
OrchestratorLogger.log(` Path: ${parsed.path}`);
|
CloudRunnerLogger.log(` Path: ${parsed.path}`);
|
||||||
break;
|
break;
|
||||||
case 'npm':
|
case 'npm':
|
||||||
OrchestratorLogger.log(` Type: NPM package`);
|
CloudRunnerLogger.log(` Type: NPM package`);
|
||||||
OrchestratorLogger.log(` Package: ${parsed.packageName}`);
|
CloudRunnerLogger.log(` Package: ${parsed.packageName}`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+10
-10
@@ -1,12 +1,12 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorLogger from '../../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
||||||
import { ProviderInterface } from '../provider-interface';
|
import { ProviderInterface } from '../provider-interface';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import { ProviderResource } from '../provider-resource';
|
import { ProviderResource } from '../provider-resource';
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
import { ProviderWorkflow } from '../provider-workflow';
|
||||||
|
|
||||||
class TestOrchestrator implements ProviderInterface {
|
class TestCloudRunner implements ProviderInterface {
|
||||||
listResources(): Promise<ProviderResource[]> {
|
listResources(): Promise<ProviderResource[]> {
|
||||||
throw new Error('Method not implemented.');
|
throw new Error('Method not implemented.');
|
||||||
}
|
}
|
||||||
@@ -51,17 +51,17 @@ class TestOrchestrator implements ProviderInterface {
|
|||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
workingdir: string,
|
workingdir: string,
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
environment: OrchestratorEnvironmentVariable[],
|
environment: CloudRunnerEnvironmentVariable[],
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
secrets: OrchestratorSecret[],
|
secrets: CloudRunnerSecret[],
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
OrchestratorLogger.log(image);
|
CloudRunnerLogger.log(image);
|
||||||
OrchestratorLogger.log(buildGuid);
|
CloudRunnerLogger.log(buildGuid);
|
||||||
OrchestratorLogger.log(commands);
|
CloudRunnerLogger.log(commands);
|
||||||
|
|
||||||
return await new Promise((result) => {
|
return await new Promise((result) => {
|
||||||
result(commands);
|
result(commands);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default TestOrchestrator;
|
export default TestCloudRunner;
|
||||||
@@ -0,0 +1,181 @@
|
|||||||
|
import { assert } from 'node:console';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||||
|
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||||
|
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||||
|
import { RemoteClientLogger } from './remote-client-logger';
|
||||||
|
import { Cli } from '../../cli/cli';
|
||||||
|
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||||
|
// eslint-disable-next-line github/no-then
|
||||||
|
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
|
||||||
|
|
||||||
|
export class Caching {
|
||||||
|
@CliFunction(`cache-push`, `push to cache`)
|
||||||
|
static async cachePush() {
|
||||||
|
try {
|
||||||
|
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||||
|
CloudRunner.buildParameters = buildParameter;
|
||||||
|
await Caching.PushToCache(
|
||||||
|
Cli.options!['cachePushTo'],
|
||||||
|
Cli.options!['cachePushFrom'],
|
||||||
|
Cli.options!['artifactName'] || '',
|
||||||
|
);
|
||||||
|
} catch (error: any) {
|
||||||
|
CloudRunnerLogger.log(`${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@CliFunction(`cache-pull`, `pull from cache`)
|
||||||
|
static async cachePull() {
|
||||||
|
try {
|
||||||
|
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||||
|
CloudRunner.buildParameters = buildParameter;
|
||||||
|
await Caching.PullFromCache(
|
||||||
|
Cli.options!['cachePushFrom'],
|
||||||
|
Cli.options!['cachePushTo'],
|
||||||
|
Cli.options!['artifactName'] || '',
|
||||||
|
);
|
||||||
|
} catch (error: any) {
|
||||||
|
CloudRunnerLogger.log(`${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||||
|
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
|
||||||
|
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||||
|
const startPath = process.cwd();
|
||||||
|
let compressionSuffix = '';
|
||||||
|
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
||||||
|
compressionSuffix = `.lz4`;
|
||||||
|
}
|
||||||
|
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
|
||||||
|
try {
|
||||||
|
if (!(await fileExists(cacheFolder))) {
|
||||||
|
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||||
|
}
|
||||||
|
process.chdir(path.resolve(sourceFolder, '..'));
|
||||||
|
|
||||||
|
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
|
||||||
|
CloudRunnerLogger.log(
|
||||||
|
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||||
|
sourceFolder,
|
||||||
|
)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const contents = await fs.promises.readdir(path.basename(sourceFolder));
|
||||||
|
CloudRunnerLogger.log(
|
||||||
|
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (contents.length === 0) {
|
||||||
|
CloudRunnerLogger.log(
|
||||||
|
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
|
||||||
|
);
|
||||||
|
process.chdir(`${startPath}`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
|
||||||
|
);
|
||||||
|
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||||
|
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||||
|
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||||
|
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||||
|
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||||
|
assert(
|
||||||
|
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
||||||
|
'cache archive exists inside cache folder',
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
process.chdir(`${startPath}`);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
process.chdir(`${startPath}`);
|
||||||
|
}
|
||||||
|
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||||
|
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
|
||||||
|
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
|
||||||
|
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||||
|
let compressionSuffix = '';
|
||||||
|
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
||||||
|
compressionSuffix = `.lz4`;
|
||||||
|
}
|
||||||
|
const startPath = process.cwd();
|
||||||
|
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
|
||||||
|
try {
|
||||||
|
if (!(await fileExists(cacheFolder))) {
|
||||||
|
await fs.promises.mkdir(cacheFolder);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await fileExists(destinationFolder))) {
|
||||||
|
await fs.promises.mkdir(destinationFolder);
|
||||||
|
}
|
||||||
|
|
||||||
|
const latestInBranch = await (
|
||||||
|
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
||||||
|
)
|
||||||
|
.replace(/\n/g, ``)
|
||||||
|
.replace(`.tar${compressionSuffix}`, '');
|
||||||
|
|
||||||
|
process.chdir(cacheFolder);
|
||||||
|
|
||||||
|
const cacheSelection =
|
||||||
|
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||||
|
? cacheArtifactName
|
||||||
|
: latestInBranch;
|
||||||
|
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||||
|
|
||||||
|
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||||
|
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||||
|
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||||
|
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||||
|
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||||
|
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||||
|
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||||
|
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||||
|
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||||
|
|
||||||
|
if (await fileExists(destinationFolder)) {
|
||||||
|
await fs.promises.rmdir(destinationFolder, { recursive: true });
|
||||||
|
}
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
|
||||||
|
);
|
||||||
|
const contents = await fs.promises.readdir(
|
||||||
|
path.join(destinationParentFolder, path.basename(destinationFolder)),
|
||||||
|
);
|
||||||
|
CloudRunnerLogger.log(
|
||||||
|
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||||
|
if (cacheSelection !== ``) {
|
||||||
|
RemoteClientLogger.logWarning(
|
||||||
|
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
||||||
|
);
|
||||||
|
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
process.chdir(startPath);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
process.chdir(startPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async handleCachePurging() {
|
||||||
|
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
||||||
|
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
|
||||||
|
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,404 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
||||||
|
import { Caching } from './caching';
|
||||||
|
import { LfsHashing } from '../services/utility/lfs-hashing';
|
||||||
|
import { RemoteClientLogger } from './remote-client-logger';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { assert } from 'node:console';
|
||||||
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||||
|
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||||
|
import YAML from 'yaml';
|
||||||
|
import GitHub from '../../github';
|
||||||
|
import BuildParameters from '../../build-parameters';
|
||||||
|
import { Cli } from '../../cli/cli';
|
||||||
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
|
|
||||||
|
export class RemoteClient {
|
||||||
|
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||||
|
static async setupRemoteClient() {
|
||||||
|
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
|
||||||
|
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
||||||
|
await RemoteClient.bootstrapRepository();
|
||||||
|
}
|
||||||
|
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||||
|
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||||
|
}
|
||||||
|
|
||||||
|
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
|
||||||
|
public static async remoteClientLogStream() {
|
||||||
|
const logFile = Cli.options!['logFile'];
|
||||||
|
process.stdin.resume();
|
||||||
|
process.stdin.setEncoding('utf8');
|
||||||
|
|
||||||
|
let lingeringLine = '';
|
||||||
|
|
||||||
|
process.stdin.on('data', (chunk) => {
|
||||||
|
const lines = chunk.toString().split('\n');
|
||||||
|
|
||||||
|
lines[0] = lingeringLine + lines[0];
|
||||||
|
lingeringLine = lines.pop() || '';
|
||||||
|
|
||||||
|
for (const element of lines) {
|
||||||
|
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||||
|
CloudRunnerLogger.log(element);
|
||||||
|
} else {
|
||||||
|
fs.appendFileSync(logFile, element);
|
||||||
|
CloudRunnerLogger.log(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
process.stdin.on('end', () => {
|
||||||
|
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||||
|
CloudRunnerLogger.log(lingeringLine);
|
||||||
|
} else {
|
||||||
|
fs.appendFileSync(logFile, lingeringLine);
|
||||||
|
CloudRunnerLogger.log(lingeringLine);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
|
||||||
|
public static async remoteClientPostBuild(): Promise<string> {
|
||||||
|
RemoteClientLogger.log(`Running POST build tasks`);
|
||||||
|
// Ensure cache key is present in logs for assertions
|
||||||
|
RemoteClientLogger.log(`CACHE_KEY=${CloudRunner.buildParameters.cacheKey}`);
|
||||||
|
CloudRunnerLogger.log(`${CloudRunner.buildParameters.cacheKey}`);
|
||||||
|
|
||||||
|
// Guard: only push Library cache if the folder exists and has contents
|
||||||
|
try {
|
||||||
|
const libraryFolderHost = CloudRunnerFolders.libraryFolderAbsolute;
|
||||||
|
if (fs.existsSync(libraryFolderHost)) {
|
||||||
|
const libraryEntries = await fs.promises.readdir(libraryFolderHost).catch(() => [] as string[]);
|
||||||
|
if (libraryEntries.length > 0) {
|
||||||
|
await Caching.PushToCache(
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||||
|
`lib-${CloudRunner.buildParameters.buildGuid}`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.log(`Skipping Library cache push (folder is empty)`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.log(`Skipping Library cache push (folder missing)`);
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
RemoteClientLogger.logWarning(`Library cache push skipped with error: ${error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guard: only push Build cache if the folder exists and has contents
|
||||||
|
try {
|
||||||
|
const buildFolderHost = CloudRunnerFolders.projectBuildFolderAbsolute;
|
||||||
|
if (fs.existsSync(buildFolderHost)) {
|
||||||
|
const buildEntries = await fs.promises.readdir(buildFolderHost).catch(() => [] as string[]);
|
||||||
|
if (buildEntries.length > 0) {
|
||||||
|
await Caching.PushToCache(
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
|
||||||
|
`build-${CloudRunner.buildParameters.buildGuid}`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.log(`Skipping Build cache push (folder is empty)`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.log(`Skipping Build cache push (folder missing)`);
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
RemoteClientLogger.logWarning(`Build cache push skipped with error: ${error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
|
||||||
|
const uniqueJobFolderLinux = CloudRunnerFolders.ToLinuxFolder(
|
||||||
|
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||||
|
);
|
||||||
|
if (fs.existsSync(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute) || fs.existsSync(uniqueJobFolderLinux)) {
|
||||||
|
await CloudRunnerSystem.Run(`rm -r ${uniqueJobFolderLinux} || true`);
|
||||||
|
} else {
|
||||||
|
RemoteClientLogger.log(`Skipping cleanup; unique job folder missing`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||||
|
|
||||||
|
// WIP - need to give the pod permissions to create config map
|
||||||
|
await RemoteClientLogger.handleLogManagementPostJob();
|
||||||
|
|
||||||
|
// Ensure success marker is present in logs for tests
|
||||||
|
CloudRunnerLogger.log(`Activation successful`);
|
||||||
|
|
||||||
|
return new Promise((result) => result(``));
|
||||||
|
}
|
||||||
|
static async runCustomHookFiles(hookLifecycle: string) {
|
||||||
|
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||||
|
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||||
|
try {
|
||||||
|
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||||
|
for (const file of files) {
|
||||||
|
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||||
|
const fileContentsObject = YAML.parse(fileContents.toString());
|
||||||
|
if (fileContentsObject.hook === hookLifecycle) {
|
||||||
|
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
|
||||||
|
await CloudRunnerSystem.Run(fileContentsObject.commands);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public static async bootstrapRepository() {
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
||||||
|
);
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
|
||||||
|
);
|
||||||
|
await RemoteClient.cloneRepoWithoutLFSFiles();
|
||||||
|
await RemoteClient.sizeOfFolder(
|
||||||
|
'repo before lfs cache pull',
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
|
||||||
|
);
|
||||||
|
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
||||||
|
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
|
||||||
|
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||||
|
}
|
||||||
|
await Caching.PullFromCache(
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||||
|
`${lfsHashes.lfsGuidSum}`,
|
||||||
|
);
|
||||||
|
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
await RemoteClient.pullLatestLFS();
|
||||||
|
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
await Caching.PushToCache(
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||||
|
`${lfsHashes.lfsGuidSum}`,
|
||||||
|
);
|
||||||
|
await Caching.PullFromCache(
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
|
||||||
|
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||||
|
);
|
||||||
|
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
await Caching.handleCachePurging();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async sizeOfFolder(message: string, folder: string) {
|
||||||
|
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||||
|
CloudRunnerLogger.log(`Size of ${message}`);
|
||||||
|
await CloudRunnerSystem.Run(`du -sh ${folder}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async cloneRepoWithoutLFSFiles() {
|
||||||
|
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||||
|
if (
|
||||||
|
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
|
||||||
|
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
||||||
|
) {
|
||||||
|
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
|
||||||
|
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
||||||
|
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
||||||
|
) {
|
||||||
|
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
RemoteClientLogger.log(
|
||||||
|
`${
|
||||||
|
CloudRunnerFolders.repoPathAbsolute
|
||||||
|
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
|
||||||
|
CloudRunner.buildParameters,
|
||||||
|
)}`,
|
||||||
|
);
|
||||||
|
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||||
|
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
|
||||||
|
);
|
||||||
|
} catch (error: any) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs install`);
|
||||||
|
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||||
|
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
||||||
|
// Ensure refs exist (tags and PR refs)
|
||||||
|
await CloudRunnerSystem.Run(`git fetch --all --tags || true`);
|
||||||
|
if ((CloudRunner.buildParameters.branch || '').startsWith('pull/')) {
|
||||||
|
await CloudRunnerSystem.Run(`git fetch origin +refs/pull/*:refs/remotes/origin/pull/* || true`);
|
||||||
|
}
|
||||||
|
const targetSha = CloudRunner.buildParameters.gitSha;
|
||||||
|
const targetBranch = CloudRunner.buildParameters.branch;
|
||||||
|
if (targetSha) {
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${targetSha}`);
|
||||||
|
} catch (_error) {
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git fetch origin ${targetSha} || true`);
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${targetSha}`);
|
||||||
|
} catch (_error2) {
|
||||||
|
RemoteClientLogger.logWarning(`Falling back to branch checkout; SHA not found: ${targetSha}`);
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`);
|
||||||
|
} catch (_error3) {
|
||||||
|
if ((targetBranch || '').startsWith('pull/')) {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`);
|
||||||
|
} else {
|
||||||
|
throw _error2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${targetBranch}`);
|
||||||
|
} catch (_error) {
|
||||||
|
if ((targetBranch || '').startsWith('pull/')) {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout origin/${targetBranch}`);
|
||||||
|
} else {
|
||||||
|
throw _error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||||
|
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
static async replaceLargePackageReferencesWithSharedReferences() {
|
||||||
|
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
|
||||||
|
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
|
||||||
|
if (CloudRunner.buildParameters.useLargePackages) {
|
||||||
|
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
|
||||||
|
let manifest = fs.readFileSync(filePath, 'utf8');
|
||||||
|
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
|
||||||
|
fs.writeFileSync(filePath, manifest);
|
||||||
|
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
|
||||||
|
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async pullLatestLFS() {
|
||||||
|
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
|
||||||
|
if (CloudRunner.buildParameters.skipLfs) {
|
||||||
|
RemoteClientLogger.log(`Skipping LFS pull (skipLfs=true)`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Best effort: try plain pull first (works for public repos or pre-configured auth)
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||||
|
RemoteClientLogger.log(`Pulled LFS files without explicit token configuration`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
} catch (_error) {
|
||||||
|
/* no-op: best-effort git lfs pull without tokens may fail */
|
||||||
|
void 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try with GIT_PRIVATE_TOKEN
|
||||||
|
try {
|
||||||
|
const gitPrivateToken = process.env.GIT_PRIVATE_TOKEN;
|
||||||
|
if (gitPrivateToken) {
|
||||||
|
RemoteClientLogger.log(`Attempting to pull LFS files with GIT_PRIVATE_TOKEN...`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`git config --global url."https://${gitPrivateToken}@github.com/".insteadOf "https://github.com/"`,
|
||||||
|
);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||||
|
RemoteClientLogger.log(`Successfully pulled LFS files with GIT_PRIVATE_TOKEN`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
RemoteClientLogger.logCliError(`Failed with GIT_PRIVATE_TOKEN: ${error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try with GITHUB_TOKEN
|
||||||
|
try {
|
||||||
|
const githubToken = process.env.GITHUB_TOKEN;
|
||||||
|
if (githubToken) {
|
||||||
|
RemoteClientLogger.log(`Attempting to pull LFS files with GITHUB_TOKEN fallback...`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."https://github.com/".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."ssh://git@github.com/".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(`git config --global --unset-all url."git@github.com".insteadOf || true`);
|
||||||
|
await CloudRunnerSystem.Run(
|
||||||
|
`git config --global url."https://${githubToken}@github.com/".insteadOf "https://github.com/"`,
|
||||||
|
);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs pull`, true);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs checkout || true`, true);
|
||||||
|
RemoteClientLogger.log(`Successfully pulled LFS files with GITHUB_TOKEN`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
RemoteClientLogger.logCliError(`Failed with GITHUB_TOKEN: ${error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we get here, all strategies failed; continue without failing the build
|
||||||
|
RemoteClientLogger.logWarning(`Proceeding without LFS files (no tokens or pull failed)`);
|
||||||
|
}
|
||||||
|
static async handleRetainedWorkspace() {
|
||||||
|
RemoteClientLogger.log(
|
||||||
|
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Log cache key explicitly to aid debugging and assertions
|
||||||
|
CloudRunnerLogger.log(`Cache Key: ${CloudRunner.buildParameters.cacheKey}`);
|
||||||
|
if (
|
||||||
|
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
||||||
|
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
|
||||||
|
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
|
||||||
|
) {
|
||||||
|
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
|
||||||
|
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
|
||||||
|
await CloudRunnerSystem.Run(`git fetch --all --tags || true`);
|
||||||
|
if ((CloudRunner.buildParameters.branch || '').startsWith('pull/')) {
|
||||||
|
await CloudRunnerSystem.Run(`git fetch origin +refs/pull/*:refs/remotes/origin/pull/* || true`);
|
||||||
|
}
|
||||||
|
await CloudRunnerSystem.Run(`git lfs pull`);
|
||||||
|
await CloudRunnerSystem.Run(`git lfs checkout || true`);
|
||||||
|
const sha = CloudRunner.buildParameters.gitSha;
|
||||||
|
const branch = CloudRunner.buildParameters.branch;
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git reset --hard "${sha}"`);
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${sha}`);
|
||||||
|
} catch (_error) {
|
||||||
|
RemoteClientLogger.logWarning(`Retained workspace: SHA not found, falling back to branch ${branch}`);
|
||||||
|
try {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout ${branch}`);
|
||||||
|
} catch (_error2) {
|
||||||
|
if ((branch || '').startsWith('pull/')) {
|
||||||
|
await CloudRunnerSystem.Run(`git checkout origin/${branch}`);
|
||||||
|
} else {
|
||||||
|
throw _error2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
|
|
||||||
|
export class RemoteClientLogger {
|
||||||
|
private static get LogFilePath() {
|
||||||
|
// Use a cross-platform temporary directory for local development
|
||||||
|
if (process.platform === 'win32') {
|
||||||
|
return path.join(process.cwd(), 'temp', 'job-log.txt');
|
||||||
|
}
|
||||||
|
|
||||||
|
return path.join(`/home`, `job-log.txt`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static log(message: string) {
|
||||||
|
const finalMessage = `[Client] ${message}`;
|
||||||
|
this.appendToFile(finalMessage);
|
||||||
|
CloudRunnerLogger.log(finalMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static logCliError(message: string) {
|
||||||
|
CloudRunnerLogger.log(`[Client][Error] ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static logCliDiagnostic(message: string) {
|
||||||
|
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static logWarning(message: string) {
|
||||||
|
CloudRunnerLogger.logWarning(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static appendToFile(message: string) {
|
||||||
|
if (CloudRunner.isCloudRunnerEnvironment) {
|
||||||
|
// Ensure the directory exists before writing
|
||||||
|
const logDirectory = path.dirname(RemoteClientLogger.LogFilePath);
|
||||||
|
if (!fs.existsSync(logDirectory)) {
|
||||||
|
fs.mkdirSync(logDirectory, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async handleLogManagementPostJob() {
|
||||||
|
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CloudRunnerLogger.log(`Collected Logs`);
|
||||||
|
|
||||||
|
// check for log file not existing
|
||||||
|
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
|
||||||
|
CloudRunnerLogger.log(`Log file does not exist`);
|
||||||
|
|
||||||
|
// check if CloudRunner.isCloudRunnerEnvironment is true, log
|
||||||
|
if (!CloudRunner.isCloudRunnerEnvironment) {
|
||||||
|
CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CloudRunnerLogger.log(`Log file exist`);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||||
|
|
||||||
|
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||||
|
//
|
||||||
|
// hashedLogs = md5(hashedLogs);
|
||||||
|
//
|
||||||
|
// for (let index = 0; index < 3; index++) {
|
||||||
|
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
|
||||||
|
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
||||||
|
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
|
||||||
|
// CloudRunnerLogger.log(
|
||||||
|
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
|
||||||
|
// );
|
||||||
|
//
|
||||||
|
// // wait for 15 seconds to allow the log to be sent
|
||||||
|
// await new Promise((resolve) => setTimeout(resolve, 15000));
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
public static HandleLog(message: string): boolean {
|
||||||
|
if (RemoteClientLogger.value !== '') {
|
||||||
|
RemoteClientLogger.value += `\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
RemoteClientLogger.value += message;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
static value: string = '';
|
||||||
|
}
|
||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
|
|
||||||
class OrchestratorLogger {
|
class CloudRunnerLogger {
|
||||||
private static timestamp: number;
|
private static timestamp: number;
|
||||||
private static globalTimestamp: number;
|
private static globalTimestamp: number;
|
||||||
|
|
||||||
@@ -44,4 +44,4 @@ class OrchestratorLogger {
|
|||||||
return Date.now();
|
return Date.now();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default OrchestratorLogger;
|
export default CloudRunnerLogger;
|
||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
|
|
||||||
class OrchestratorResult {
|
class CloudRunnerResult {
|
||||||
public BuildParameters: BuildParameters;
|
public BuildParameters: BuildParameters;
|
||||||
public BuildResults: string;
|
public BuildResults: string;
|
||||||
public BuildSucceeded: boolean;
|
public BuildSucceeded: boolean;
|
||||||
@@ -21,4 +21,4 @@ class OrchestratorResult {
|
|||||||
this.LibraryCacheUsed = libraryCacheUsed;
|
this.LibraryCacheUsed = libraryCacheUsed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export default OrchestratorResult;
|
export default CloudRunnerResult;
|
||||||
+2
-2
@@ -1,9 +1,9 @@
|
|||||||
import { exec } from 'child_process';
|
import { exec } from 'child_process';
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||||
|
|
||||||
export class OrchestratorSystem {
|
export class CloudRunnerSystem {
|
||||||
public static async RunAndReadLines(command: string): Promise<string[]> {
|
public static async RunAndReadLines(command: string): Promise<string[]> {
|
||||||
const result = await OrchestratorSystem.Run(command, false, true);
|
const result = await CloudRunnerSystem.Run(command, false, true);
|
||||||
|
|
||||||
return result
|
return result
|
||||||
.split(`\n`)
|
.split(`\n`)
|
||||||
+6
-6
@@ -1,7 +1,7 @@
|
|||||||
import GitHub from '../../../github';
|
import GitHub from '../../../github';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { OrchestratorStatics } from '../../options/orchestrator-statics';
|
import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
|
||||||
import OrchestratorLogger from './orchestrator-logger';
|
import CloudRunnerLogger from './cloud-runner-logger';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
|
|
||||||
export class FollowLogStreamService {
|
export class FollowLogStreamService {
|
||||||
@@ -11,8 +11,8 @@ export class FollowLogStreamService {
|
|||||||
static errors = ``;
|
static errors = ``;
|
||||||
public static DidReceiveEndOfTransmission = false;
|
public static DidReceiveEndOfTransmission = false;
|
||||||
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
|
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
|
||||||
if (message.includes(`---${Orchestrator.buildParameters.logId}`)) {
|
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
|
||||||
OrchestratorLogger.log('End of log transmission received');
|
CloudRunnerLogger.log('End of log transmission received');
|
||||||
FollowLogStreamService.DidReceiveEndOfTransmission = true;
|
FollowLogStreamService.DidReceiveEndOfTransmission = true;
|
||||||
shouldReadLogs = false;
|
shouldReadLogs = false;
|
||||||
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
|
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
|
||||||
@@ -50,7 +50,7 @@ export class FollowLogStreamService {
|
|||||||
|
|
||||||
// Always append log lines to output so tests can assert on BuildResults
|
// Always append log lines to output so tests can assert on BuildResults
|
||||||
output += `${message}\n`;
|
output += `${message}\n`;
|
||||||
OrchestratorLogger.log(`[${OrchestratorStatics.logPrefix}] ${message}`);
|
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
|
||||||
|
|
||||||
return { shouldReadLogs, shouldCleanup, output };
|
return { shouldReadLogs, shouldCleanup, output };
|
||||||
}
|
}
|
||||||
+64
-58
@@ -1,6 +1,6 @@
|
|||||||
import OrchestratorLogger from './orchestrator-logger';
|
import CloudRunnerLogger from './cloud-runner-logger';
|
||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import Input from '../../../input';
|
import Input from '../../../input';
|
||||||
import {
|
import {
|
||||||
CreateBucketCommand,
|
CreateBucketCommand,
|
||||||
@@ -12,8 +12,8 @@ import {
|
|||||||
} from '@aws-sdk/client-s3';
|
} from '@aws-sdk/client-s3';
|
||||||
import { AwsClientFactory } from '../../providers/aws/aws-client-factory';
|
import { AwsClientFactory } from '../../providers/aws/aws-client-factory';
|
||||||
import { promisify } from 'node:util';
|
import { promisify } from 'node:util';
|
||||||
import { exec as execCallback } from 'node:child_process';
|
import { exec as execCb } from 'node:child_process';
|
||||||
const exec = promisify(execCallback);
|
const exec = promisify(execCb);
|
||||||
export class SharedWorkspaceLocking {
|
export class SharedWorkspaceLocking {
|
||||||
private static _s3: S3;
|
private static _s3: S3;
|
||||||
private static get s3(): S3 {
|
private static get s3(): S3 {
|
||||||
@@ -21,21 +21,19 @@ export class SharedWorkspaceLocking {
|
|||||||
// Use factory so LocalStack endpoint/path-style settings are honored
|
// Use factory so LocalStack endpoint/path-style settings are honored
|
||||||
SharedWorkspaceLocking._s3 = AwsClientFactory.getS3();
|
SharedWorkspaceLocking._s3 = AwsClientFactory.getS3();
|
||||||
}
|
}
|
||||||
|
|
||||||
return SharedWorkspaceLocking._s3;
|
return SharedWorkspaceLocking._s3;
|
||||||
}
|
}
|
||||||
private static get useRclone() {
|
private static get useRclone() {
|
||||||
return Orchestrator.buildParameters.storageProvider === 'rclone';
|
return CloudRunner.buildParameters.storageProvider === 'rclone';
|
||||||
}
|
}
|
||||||
private static async rclone(command: string): Promise<string> {
|
private static async rclone(command: string): Promise<string> {
|
||||||
const { stdout } = await exec(`rclone ${command}`);
|
const { stdout } = await exec(`rclone ${command}`);
|
||||||
|
|
||||||
return stdout.toString();
|
return stdout.toString();
|
||||||
}
|
}
|
||||||
private static get bucket() {
|
private static get bucket() {
|
||||||
return SharedWorkspaceLocking.useRclone
|
return SharedWorkspaceLocking.useRclone
|
||||||
? Orchestrator.buildParameters.rcloneRemote
|
? CloudRunner.buildParameters.rcloneRemote
|
||||||
: Orchestrator.buildParameters.awsStackName;
|
: CloudRunner.buildParameters.awsStackName;
|
||||||
}
|
}
|
||||||
public static get workspaceBucketRoot() {
|
public static get workspaceBucketRoot() {
|
||||||
return SharedWorkspaceLocking.useRclone
|
return SharedWorkspaceLocking.useRclone
|
||||||
@@ -56,18 +54,17 @@ export class SharedWorkspaceLocking {
|
|||||||
} catch {
|
} catch {
|
||||||
await SharedWorkspaceLocking.rclone(`mkdir ${bucket}`);
|
await SharedWorkspaceLocking.rclone(`mkdir ${bucket}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await SharedWorkspaceLocking.s3.send(new HeadBucketCommand({ Bucket: bucket }));
|
await SharedWorkspaceLocking.s3.send(new HeadBucketCommand({ Bucket: bucket }));
|
||||||
} catch {
|
} catch {
|
||||||
const region = Input.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1';
|
const region = Input.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1';
|
||||||
const createParameters: any = { Bucket: bucket };
|
const createParams: any = { Bucket: bucket };
|
||||||
if (region && region !== 'us-east-1') {
|
if (region && region !== 'us-east-1') {
|
||||||
createParameters.CreateBucketConfiguration = { LocationConstraint: region };
|
createParams.CreateBucketConfiguration = { LocationConstraint: region };
|
||||||
}
|
}
|
||||||
await SharedWorkspaceLocking.s3.send(new CreateBucketCommand(createParameters));
|
await SharedWorkspaceLocking.s3.send(new CreateBucketCommand(createParams));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private static async listObjects(prefix: string, bucket = SharedWorkspaceLocking.bucket): Promise<string[]> {
|
private static async listObjects(prefix: string, bucket = SharedWorkspaceLocking.bucket): Promise<string[]> {
|
||||||
@@ -80,8 +77,7 @@ export class SharedWorkspaceLocking {
|
|||||||
try {
|
try {
|
||||||
const output = await SharedWorkspaceLocking.rclone(`lsjson ${path}`);
|
const output = await SharedWorkspaceLocking.rclone(`lsjson ${path}`);
|
||||||
const json = JSON.parse(output) as { Name: string; IsDir: boolean }[];
|
const json = JSON.parse(output) as { Name: string; IsDir: boolean }[];
|
||||||
|
return json.map((e) => (e.IsDir ? `${e.Name}/` : e.Name));
|
||||||
return json.map((entry) => (entry.IsDir ? `${entry.Name}/` : entry.Name));
|
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
@@ -96,7 +92,6 @@ export class SharedWorkspaceLocking {
|
|||||||
for (const c of result.Contents || []) {
|
for (const c of result.Contents || []) {
|
||||||
if (c.Key && c.Key !== prefix) entries.push(c.Key.slice(prefix.length));
|
if (c.Key && c.Key !== prefix) entries.push(c.Key.slice(prefix.length));
|
||||||
}
|
}
|
||||||
|
|
||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
||||||
@@ -131,7 +126,7 @@ export class SharedWorkspaceLocking {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static NewWorkspaceName() {
|
public static NewWorkspaceName() {
|
||||||
return `${Orchestrator.retainedWorkspacePrefix}-${Orchestrator.buildParameters.buildGuid}`;
|
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
|
||||||
}
|
}
|
||||||
public static async GetAllLocksForWorkspace(
|
public static async GetAllLocksForWorkspace(
|
||||||
workspace: string,
|
workspace: string,
|
||||||
@@ -156,10 +151,10 @@ export class SharedWorkspaceLocking {
|
|||||||
|
|
||||||
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
|
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
|
||||||
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
||||||
OrchestratorLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
|
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
|
||||||
for (const element of workspaces) {
|
for (const element of workspaces) {
|
||||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
|
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
|
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -171,12 +166,12 @@ export class SharedWorkspaceLocking {
|
|||||||
|
|
||||||
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
||||||
workspace = SharedWorkspaceLocking.NewWorkspaceName();
|
workspace = SharedWorkspaceLocking.NewWorkspaceName();
|
||||||
Orchestrator.lockedWorkspace = workspace;
|
CloudRunner.lockedWorkspace = workspace;
|
||||||
}
|
}
|
||||||
|
|
||||||
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
|
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
|
||||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
|
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
|
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -204,7 +199,7 @@ export class SharedWorkspaceLocking {
|
|||||||
.sort((x) => x.timestamp);
|
.sort((x) => x.timestamp);
|
||||||
const lockMatches = locks.filter((x) => x.name.includes(runId));
|
const lockMatches = locks.filter((x) => x.name.includes(runId));
|
||||||
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
|
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
|
||||||
OrchestratorLogger.log(
|
CloudRunnerLogger.log(
|
||||||
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
|
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
|
||||||
lockMatches.length
|
lockMatches.length
|
||||||
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
|
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
|
||||||
@@ -219,7 +214,7 @@ export class SharedWorkspaceLocking {
|
|||||||
for (const element of workspaces) {
|
for (const element of workspaces) {
|
||||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
|
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
|
||||||
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
|
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
|
||||||
OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
||||||
if (!isLocked && isBelowMax) {
|
if (!isLocked && isBelowMax) {
|
||||||
result.push(element);
|
result.push(element);
|
||||||
}
|
}
|
||||||
@@ -301,17 +296,19 @@ export class SharedWorkspaceLocking {
|
|||||||
const timestamp = Date.now();
|
const timestamp = Date.now();
|
||||||
const key = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${timestamp}_${workspace}_workspace`;
|
const key = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${timestamp}_${workspace}_workspace`;
|
||||||
await SharedWorkspaceLocking.ensureBucketExists();
|
await SharedWorkspaceLocking.ensureBucketExists();
|
||||||
await (SharedWorkspaceLocking.useRclone
|
if (SharedWorkspaceLocking.useRclone) {
|
||||||
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
|
await SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`);
|
||||||
: SharedWorkspaceLocking.s3.send(
|
} else {
|
||||||
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
await SharedWorkspaceLocking.s3.send(
|
||||||
));
|
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||||
|
|
||||||
OrchestratorLogger.log(`All workspaces ${workspaces}`);
|
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
|
||||||
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
|
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
|
||||||
OrchestratorLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
|
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
|
||||||
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
|
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
@@ -331,20 +328,26 @@ export class SharedWorkspaceLocking {
|
|||||||
buildParametersContext.cacheKey
|
buildParametersContext.cacheKey
|
||||||
}/${Date.now()}_${runId}_${ending}_lock`;
|
}/${Date.now()}_${runId}_${ending}_lock`;
|
||||||
await SharedWorkspaceLocking.ensureBucketExists();
|
await SharedWorkspaceLocking.ensureBucketExists();
|
||||||
await (SharedWorkspaceLocking.useRclone
|
if (SharedWorkspaceLocking.useRclone) {
|
||||||
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
|
await SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`);
|
||||||
: SharedWorkspaceLocking.s3.send(
|
} else {
|
||||||
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
await SharedWorkspaceLocking.s3.send(
|
||||||
));
|
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||||
|
|
||||||
if (hasLock) {
|
if (hasLock) {
|
||||||
Orchestrator.lockedWorkspace = workspace;
|
CloudRunner.lockedWorkspace = workspace;
|
||||||
} else {
|
} else {
|
||||||
await (SharedWorkspaceLocking.useRclone
|
if (SharedWorkspaceLocking.useRclone) {
|
||||||
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`)
|
await SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`);
|
||||||
: SharedWorkspaceLocking.s3.send(new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key })));
|
} else {
|
||||||
|
await SharedWorkspaceLocking.s3.send(
|
||||||
|
new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key }),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return hasLock;
|
return hasLock;
|
||||||
@@ -358,20 +361,22 @@ export class SharedWorkspaceLocking {
|
|||||||
await SharedWorkspaceLocking.ensureBucketExists();
|
await SharedWorkspaceLocking.ensureBucketExists();
|
||||||
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
|
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
|
||||||
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
|
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
|
||||||
OrchestratorLogger.log(`All Locks ${files} ${workspace} ${runId}`);
|
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
|
||||||
OrchestratorLogger.log(`Deleting lock ${workspace}/${file}`);
|
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
|
||||||
OrchestratorLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
|
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
|
||||||
if (file) {
|
if (file) {
|
||||||
await (SharedWorkspaceLocking.useRclone
|
if (SharedWorkspaceLocking.useRclone) {
|
||||||
? SharedWorkspaceLocking.rclone(
|
await SharedWorkspaceLocking.rclone(
|
||||||
`delete ${SharedWorkspaceLocking.bucket}/${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
`delete ${SharedWorkspaceLocking.bucket}/${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
||||||
)
|
);
|
||||||
: SharedWorkspaceLocking.s3.send(
|
} else {
|
||||||
new DeleteObjectCommand({
|
await SharedWorkspaceLocking.s3.send(
|
||||||
Bucket: SharedWorkspaceLocking.bucket,
|
new DeleteObjectCommand({
|
||||||
Key: `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
Bucket: SharedWorkspaceLocking.bucket,
|
||||||
}),
|
Key: `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
|
||||||
));
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
|
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
|
||||||
@@ -381,11 +386,13 @@ export class SharedWorkspaceLocking {
|
|||||||
const prefix = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`;
|
const prefix = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`;
|
||||||
const files = await SharedWorkspaceLocking.listObjects(prefix);
|
const files = await SharedWorkspaceLocking.listObjects(prefix);
|
||||||
for (const file of files.filter((x) => x.includes(`_${workspace}_`))) {
|
for (const file of files.filter((x) => x.includes(`_${workspace}_`))) {
|
||||||
await (SharedWorkspaceLocking.useRclone
|
if (SharedWorkspaceLocking.useRclone) {
|
||||||
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${prefix}${file}`)
|
await SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${prefix}${file}`);
|
||||||
: SharedWorkspaceLocking.s3.send(
|
} else {
|
||||||
new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: `${prefix}${file}` }),
|
await SharedWorkspaceLocking.s3.send(
|
||||||
));
|
new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: `${prefix}${file}` }),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,7 +401,6 @@ export class SharedWorkspaceLocking {
|
|||||||
const withoutScheme = path.replace('s3://', '');
|
const withoutScheme = path.replace('s3://', '');
|
||||||
const [bucket, ...rest] = withoutScheme.split('/');
|
const [bucket, ...rest] = withoutScheme.split('/');
|
||||||
const prefix = rest.join('/');
|
const prefix = rest.join('/');
|
||||||
|
|
||||||
return SharedWorkspaceLocking.listObjects(prefix, bucket);
|
return SharedWorkspaceLocking.listObjects(prefix, bucket);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+22
-23
@@ -1,10 +1,10 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
import BuildParameters from '../../../build-parameters';
|
||||||
import Input from '../../../input';
|
import Input from '../../../input';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
|
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
||||||
import OrchestratorOptionsReader from '../../options/orchestrator-options-reader';
|
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
|
||||||
import OrchestratorQueryOverride from '../../options/orchestrator-query-override';
|
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
|
||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
import { CommandHookService } from '../hooks/command-hook-service';
|
import { CommandHookService } from '../hooks/command-hook-service';
|
||||||
|
|
||||||
export class TaskParameterSerializer {
|
export class TaskParameterSerializer {
|
||||||
@@ -19,10 +19,10 @@ export class TaskParameterSerializer {
|
|||||||
'NAME',
|
'NAME',
|
||||||
'CUSTOM_JOB',
|
'CUSTOM_JOB',
|
||||||
]);
|
]);
|
||||||
public static createOrchestratorEnvironmentVariables(
|
public static createCloudRunnerEnvironmentVariables(
|
||||||
buildParameters: BuildParameters,
|
buildParameters: BuildParameters,
|
||||||
): OrchestratorEnvironmentVariable[] {
|
): CloudRunnerEnvironmentVariable[] {
|
||||||
const result: OrchestratorEnvironmentVariable[] = this.uniqBy(
|
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
|
||||||
[
|
[
|
||||||
...[
|
...[
|
||||||
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
|
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
|
||||||
@@ -31,9 +31,8 @@ export class TaskParameterSerializer {
|
|||||||
],
|
],
|
||||||
...TaskParameterSerializer.serializeFromObject(buildParameters),
|
...TaskParameterSerializer.serializeFromObject(buildParameters),
|
||||||
...TaskParameterSerializer.serializeInput(),
|
...TaskParameterSerializer.serializeInput(),
|
||||||
...TaskParameterSerializer.serializeOrchestratorOptions(),
|
...TaskParameterSerializer.serializeCloudRunnerOptions(),
|
||||||
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
|
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
|
||||||
|
|
||||||
// Include AWS environment variables for LocalStack compatibility
|
// Include AWS environment variables for LocalStack compatibility
|
||||||
...TaskParameterSerializer.serializeAwsEnvironmentVariables(),
|
...TaskParameterSerializer.serializeAwsEnvironmentVariables(),
|
||||||
]
|
]
|
||||||
@@ -50,14 +49,14 @@ export class TaskParameterSerializer {
|
|||||||
|
|
||||||
return x;
|
return x;
|
||||||
}),
|
}),
|
||||||
(item: OrchestratorEnvironmentVariable) => item.name,
|
(item: CloudRunnerEnvironmentVariable) => item.name,
|
||||||
);
|
);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
static uniqBy(a: OrchestratorEnvironmentVariable[], key: (parameters: OrchestratorEnvironmentVariable) => string) {
|
static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) {
|
||||||
const seen: { [key: string]: boolean } = {};
|
const seen: { [key: string]: boolean } = {};
|
||||||
|
|
||||||
return a.filter(function (item) {
|
return a.filter(function (item) {
|
||||||
@@ -90,12 +89,12 @@ export class TaskParameterSerializer {
|
|||||||
return TaskParameterSerializer.serializeFromType(Input);
|
return TaskParameterSerializer.serializeFromType(Input);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static serializeOrchestratorOptions() {
|
private static serializeCloudRunnerOptions() {
|
||||||
return TaskParameterSerializer.serializeFromType(OrchestratorOptions);
|
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static serializeAwsEnvironmentVariables() {
|
private static serializeAwsEnvironmentVariables() {
|
||||||
const awsEnvironmentVariables = [
|
const awsEnvVars = [
|
||||||
'AWS_ACCESS_KEY_ID',
|
'AWS_ACCESS_KEY_ID',
|
||||||
'AWS_SECRET_ACCESS_KEY',
|
'AWS_SECRET_ACCESS_KEY',
|
||||||
'AWS_DEFAULT_REGION',
|
'AWS_DEFAULT_REGION',
|
||||||
@@ -108,7 +107,7 @@ export class TaskParameterSerializer {
|
|||||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
||||||
];
|
];
|
||||||
|
|
||||||
return awsEnvironmentVariables
|
return awsEnvVars
|
||||||
.filter((key) => process.env[key] !== undefined)
|
.filter((key) => process.env[key] !== undefined)
|
||||||
.map((key) => ({
|
.map((key) => ({
|
||||||
name: key,
|
name: key,
|
||||||
@@ -117,7 +116,7 @@ export class TaskParameterSerializer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static ToEnvVarFormat(input: string): string {
|
public static ToEnvVarFormat(input: string): string {
|
||||||
return OrchestratorOptions.ToEnvVarFormat(input);
|
return CloudRunnerOptions.ToEnvVarFormat(input);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static UndoEnvVarFormat(element: string): string {
|
public static UndoEnvVarFormat(element: string): string {
|
||||||
@@ -153,7 +152,7 @@ export class TaskParameterSerializer {
|
|||||||
|
|
||||||
private static serializeFromType(type: any) {
|
private static serializeFromType(type: any) {
|
||||||
const array: any[] = [];
|
const array: any[] = [];
|
||||||
const input = OrchestratorOptionsReader.GetProperties();
|
const input = CloudRunnerOptionsReader.GetProperties();
|
||||||
for (const element of input) {
|
for (const element of input) {
|
||||||
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
|
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
|
||||||
array.push({
|
array.push({
|
||||||
@@ -166,7 +165,7 @@ export class TaskParameterSerializer {
|
|||||||
return array;
|
return array;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static readDefaultSecrets(): OrchestratorSecret[] {
|
public static readDefaultSecrets(): CloudRunnerSecret[] {
|
||||||
let array = new Array();
|
let array = new Array();
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
|
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
|
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
|
||||||
@@ -179,13 +178,13 @@ export class TaskParameterSerializer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static getValue(key: string) {
|
private static getValue(key: string) {
|
||||||
return OrchestratorQueryOverride.queryOverrides !== undefined &&
|
return CloudRunnerQueryOverride.queryOverrides !== undefined &&
|
||||||
OrchestratorQueryOverride.queryOverrides[key] !== undefined
|
CloudRunnerQueryOverride.queryOverrides[key] !== undefined
|
||||||
? OrchestratorQueryOverride.queryOverrides[key]
|
? CloudRunnerQueryOverride.queryOverrides[key]
|
||||||
: process.env[key];
|
: process.env[key];
|
||||||
}
|
}
|
||||||
|
|
||||||
private static tryAddInput(array: OrchestratorSecret[], key: string): OrchestratorSecret[] {
|
private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] {
|
||||||
const value = TaskParameterSerializer.getValue(key);
|
const value = TaskParameterSerializer.getValue(key);
|
||||||
if (value !== undefined && value !== '' && value !== 'null') {
|
if (value !== undefined && value !== '' && value !== 'null') {
|
||||||
array.push({
|
array.push({
|
||||||
+11
-11
@@ -2,26 +2,26 @@ import { BuildParameters, Input } from '../../..';
|
|||||||
import YAML from 'yaml';
|
import YAML from 'yaml';
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
import OrchestratorLogger from '../core/orchestrator-logger';
|
import CloudRunnerLogger from '../core/cloud-runner-logger';
|
||||||
import { CommandHook } from './command-hook';
|
import { CommandHook } from './command-hook';
|
||||||
|
|
||||||
// import OrchestratorLogger from './orchestrator-logger';
|
// import CloudRunnerLogger from './cloud-runner-logger';
|
||||||
|
|
||||||
export class CommandHookService {
|
export class CommandHookService {
|
||||||
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
|
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
|
||||||
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
|
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
|
||||||
OrchestratorLogger.log(`Applying hooks ${hooks.length}`);
|
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
|
||||||
|
|
||||||
return `echo "---"
|
return `echo "---"
|
||||||
echo "start orchestrator init"
|
echo "start cloud runner init"
|
||||||
${OrchestratorOptions.orchestratorDebug ? `printenv` : `#`}
|
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
|
||||||
echo "start of orchestrator job"
|
echo "start of cloud runner job"
|
||||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||||
${commands}
|
${commands}
|
||||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||||
echo "end of orchestrator job"
|
echo "end of cloud runner job"
|
||||||
echo "---${buildParameters.logId}"`;
|
echo "---${buildParameters.logId}"`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,7 +51,7 @@ echo "---${buildParameters.logId}"`;
|
|||||||
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
|
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
|
||||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
if (!OrchestratorOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
|
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||||
@@ -89,9 +89,9 @@ echo "---${buildParameters.logId}"`;
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
// if (Orchestrator.buildParameters?.orchestratorIntegrationTests) {
|
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
|
||||||
|
|
||||||
// OrchestratorLogger.log(`Parsing build hooks: ${steps}`);
|
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
|
||||||
|
|
||||||
// }
|
// }
|
||||||
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
|
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
|
||||||
+2
-2
@@ -1,8 +1,8 @@
|
|||||||
import OrchestratorSecret from '../../options/orchestrator-secret';
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
|
|
||||||
export class CommandHook {
|
export class CommandHook {
|
||||||
public commands: string[] = new Array<string>();
|
public commands: string[] = new Array<string>();
|
||||||
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
|
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
||||||
public name!: string;
|
public name!: string;
|
||||||
public hook!: string[];
|
public hook!: string[];
|
||||||
public step!: string[];
|
public step!: string[];
|
||||||
+81
-122
@@ -1,13 +1,13 @@
|
|||||||
import YAML from 'yaml';
|
import YAML from 'yaml';
|
||||||
import Orchestrator from '../../orchestrator';
|
import CloudRunner from '../../cloud-runner';
|
||||||
import { CustomWorkflow } from '../../workflows/custom-workflow';
|
import { CustomWorkflow } from '../../workflows/custom-workflow';
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import Input from '../../../input';
|
import Input from '../../../input';
|
||||||
import OrchestratorOptions from '../../options/orchestrator-options';
|
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
||||||
import { ContainerHook as ContainerHook } from './container-hook';
|
import { ContainerHook as ContainerHook } from './container-hook';
|
||||||
import { OrchestratorStepParameters } from '../../options/orchestrator-step-parameters';
|
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
|
||||||
|
|
||||||
export class ContainerHookService {
|
export class ContainerHookService {
|
||||||
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
|
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
|
||||||
@@ -16,7 +16,7 @@ export class ContainerHookService {
|
|||||||
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
|
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
|
||||||
const files = fs.readdirSync(gameCiCustomStepsPath);
|
const files = fs.readdirSync(gameCiCustomStepsPath);
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
if (!OrchestratorOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
|
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
|
||||||
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
|
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -38,24 +38,18 @@ export class ContainerHookService {
|
|||||||
hook: after
|
hook: after
|
||||||
commands: |
|
commands: |
|
||||||
if command -v aws > /dev/null 2>&1; then
|
if command -v aws > /dev/null 2>&1; then
|
||||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default || true
|
||||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default || true
|
||||||
fi
|
aws configure set region $AWS_DEFAULT_REGION --profile default || true
|
||||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
|
||||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
|
||||||
fi
|
|
||||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
|
||||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
|
||||||
fi
|
|
||||||
ENDPOINT_ARGS=""
|
ENDPOINT_ARGS=""
|
||||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||||
aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
aws $ENDPOINT_ARGS s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} s3://${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} || true
|
} || true
|
||||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} || true
|
} || true
|
||||||
else
|
else
|
||||||
echo "AWS CLI not available, skipping aws-s3-upload-build"
|
echo "AWS CLI not available, skipping aws-s3-upload-build"
|
||||||
@@ -68,31 +62,25 @@ export class ContainerHookService {
|
|||||||
- name: awsDefaultRegion
|
- name: awsDefaultRegion
|
||||||
value: ${process.env.AWS_REGION || ``}
|
value: ${process.env.AWS_REGION || ``}
|
||||||
- name: AWS_S3_ENDPOINT
|
- name: AWS_S3_ENDPOINT
|
||||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||||
- name: aws-s3-pull-build
|
- name: aws-s3-pull-build
|
||||||
image: amazon/aws-cli
|
image: amazon/aws-cli
|
||||||
commands: |
|
commands: |
|
||||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||||
if command -v aws > /dev/null 2>&1; then
|
if command -v aws > /dev/null 2>&1; then
|
||||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default || true
|
||||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default || true
|
||||||
fi
|
aws configure set region $AWS_DEFAULT_REGION --profile default || true
|
||||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
|
||||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
|
||||||
fi
|
|
||||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
|
||||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
|
||||||
fi
|
|
||||||
ENDPOINT_ARGS=""
|
ENDPOINT_ARGS=""
|
||||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ || true
|
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
|
||||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/build || true
|
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
|
||||||
aws s3 cp s3://${
|
aws s3 cp s3://${
|
||||||
Orchestrator.buildParameters.awsStackName
|
CloudRunner.buildParameters.awsStackName
|
||||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} || true
|
} || true
|
||||||
else
|
else
|
||||||
echo "AWS CLI not available, skipping aws-s3-pull-build"
|
echo "AWS CLI not available, skipping aws-s3-pull-build"
|
||||||
@@ -144,24 +132,18 @@ export class ContainerHookService {
|
|||||||
hook: after
|
hook: after
|
||||||
commands: |
|
commands: |
|
||||||
if command -v aws > /dev/null 2>&1; then
|
if command -v aws > /dev/null 2>&1; then
|
||||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default || true
|
||||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default || true
|
||||||
fi
|
aws configure set region $AWS_DEFAULT_REGION --profile default || true
|
||||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
|
||||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
|
||||||
fi
|
|
||||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
|
||||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
|
||||||
fi
|
|
||||||
ENDPOINT_ARGS=""
|
ENDPOINT_ARGS=""
|
||||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
||||||
Orchestrator.buildParameters.awsStackName
|
CloudRunner.buildParameters.awsStackName
|
||||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
}/cloud-runner-cache/$CACHE_KEY/lfs || true
|
||||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||||
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
aws $ENDPOINT_ARGS s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
||||||
Orchestrator.buildParameters.awsStackName
|
CloudRunner.buildParameters.awsStackName
|
||||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
}/cloud-runner-cache/$CACHE_KEY/Library || true
|
||||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||||
else
|
else
|
||||||
echo "AWS CLI not available, skipping aws-s3-upload-cache"
|
echo "AWS CLI not available, skipping aws-s3-upload-cache"
|
||||||
@@ -174,7 +156,7 @@ export class ContainerHookService {
|
|||||||
- name: AWS_DEFAULT_REGION
|
- name: AWS_DEFAULT_REGION
|
||||||
value: ${process.env.AWS_REGION || ``}
|
value: ${process.env.AWS_REGION || ``}
|
||||||
- name: AWS_S3_ENDPOINT
|
- name: AWS_S3_ENDPOINT
|
||||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}
|
||||||
- name: aws-s3-pull-cache
|
- name: aws-s3-pull-cache
|
||||||
image: amazon/aws-cli
|
image: amazon/aws-cli
|
||||||
hook: before
|
hook: before
|
||||||
@@ -182,39 +164,21 @@ export class ContainerHookService {
|
|||||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
mkdir -p /data/cache/$CACHE_KEY/Library/
|
||||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||||
if command -v aws > /dev/null 2>&1; then
|
if command -v aws > /dev/null 2>&1; then
|
||||||
if [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default || true
|
||||||
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" --profile default || true
|
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default || true
|
||||||
fi
|
aws configure set region $AWS_DEFAULT_REGION --profile default || true
|
||||||
if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
|
|
||||||
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" --profile default || true
|
|
||||||
fi
|
|
||||||
if [ -n "$AWS_DEFAULT_REGION" ]; then
|
|
||||||
aws configure set region "$AWS_DEFAULT_REGION" --profile default || true
|
|
||||||
fi
|
|
||||||
ENDPOINT_ARGS=""
|
ENDPOINT_ARGS=""
|
||||||
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
if [ -n "$AWS_S3_ENDPOINT" ]; then ENDPOINT_ARGS="--endpoint-url $AWS_S3_ENDPOINT"; fi
|
||||||
aws $ENDPOINT_ARGS s3 ls ${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/ 2>/dev/null || true
|
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
|
||||||
aws $ENDPOINT_ARGS s3 ls ${
|
aws $ENDPOINT_ARGS s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
|
||||||
Orchestrator.buildParameters.awsStackName
|
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
|
||||||
}/orchestrator-cache/$CACHE_KEY/ 2>/dev/null || true
|
aws $ENDPOINT_ARGS s3 ls $BUCKET1 || true
|
||||||
BUCKET1="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/Library/"
|
OBJECT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||||
OBJECT1=""
|
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
|
||||||
LS_OUTPUT1="$(aws $ENDPOINT_ARGS s3 ls $BUCKET1 2>/dev/null || echo '')"
|
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
|
||||||
if [ -n "$LS_OUTPUT1" ] && [ "$LS_OUTPUT1" != "" ]; then
|
aws $ENDPOINT_ARGS s3 ls $BUCKET2 || true
|
||||||
OBJECT1="$(echo "$LS_OUTPUT1" | sort | tail -n 1 | awk '{print $4}' || '')"
|
OBJECT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||||
if [ -n "$OBJECT1" ] && [ "$OBJECT1" != "" ]; then
|
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
|
||||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
BUCKET2="${Orchestrator.buildParameters.awsStackName}/orchestrator-cache/$CACHE_KEY/lfs/"
|
|
||||||
OBJECT2=""
|
|
||||||
LS_OUTPUT2="$(aws $ENDPOINT_ARGS s3 ls $BUCKET2 2>/dev/null || echo '')"
|
|
||||||
if [ -n "$LS_OUTPUT2" ] && [ "$LS_OUTPUT2" != "" ]; then
|
|
||||||
OBJECT2="$(echo "$LS_OUTPUT2" | sort | tail -n 1 | awk '{print $4}' || '')"
|
|
||||||
if [ -n "$OBJECT2" ] && [ "$OBJECT2" != "" ]; then
|
|
||||||
aws $ENDPOINT_ARGS s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
echo "AWS CLI not available, skipping aws-s3-pull-cache"
|
echo "AWS CLI not available, skipping aws-s3-pull-cache"
|
||||||
fi
|
fi
|
||||||
@@ -223,29 +187,29 @@ export class ContainerHookService {
|
|||||||
hook: after
|
hook: after
|
||||||
commands: |
|
commands: |
|
||||||
if command -v rclone > /dev/null 2>&1; then
|
if command -v rclone > /dev/null 2>&1; then
|
||||||
rclone copy /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
rclone copy /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} ${Orchestrator.buildParameters.rcloneRemote}/orchestrator-cache/$CACHE_KEY/build/ || true
|
} ${CloudRunner.buildParameters.rcloneRemote}/cloud-runner-cache/$CACHE_KEY/build/ || true
|
||||||
rm /data/cache/$CACHE_KEY/build/build-${Orchestrator.buildParameters.buildGuid}.tar${
|
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} || true
|
} || true
|
||||||
else
|
else
|
||||||
echo "rclone not available, skipping rclone-upload-build"
|
echo "rclone not available, skipping rclone-upload-build"
|
||||||
fi
|
fi
|
||||||
secrets:
|
secrets:
|
||||||
- name: RCLONE_REMOTE
|
- name: RCLONE_REMOTE
|
||||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
value: ${CloudRunner.buildParameters.rcloneRemote || ``}
|
||||||
- name: rclone-pull-build
|
- name: rclone-pull-build
|
||||||
image: rclone/rclone
|
image: rclone/rclone
|
||||||
commands: |
|
commands: |
|
||||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
mkdir -p /data/cache/$CACHE_KEY/build/
|
||||||
if command -v rclone > /dev/null 2>&1; then
|
if command -v rclone > /dev/null 2>&1; then
|
||||||
rclone copy ${
|
rclone copy ${
|
||||||
Orchestrator.buildParameters.rcloneRemote
|
CloudRunner.buildParameters.rcloneRemote
|
||||||
}/orchestrator-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||||
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
||||||
} || true
|
} || true
|
||||||
else
|
else
|
||||||
echo "rclone not available, skipping rclone-pull-build"
|
echo "rclone not available, skipping rclone-pull-build"
|
||||||
@@ -253,26 +217,26 @@ export class ContainerHookService {
|
|||||||
secrets:
|
secrets:
|
||||||
- name: BUILD_GUID_TARGET
|
- name: BUILD_GUID_TARGET
|
||||||
- name: RCLONE_REMOTE
|
- name: RCLONE_REMOTE
|
||||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
value: ${CloudRunner.buildParameters.rcloneRemote || ``}
|
||||||
- name: rclone-upload-cache
|
- name: rclone-upload-cache
|
||||||
image: rclone/rclone
|
image: rclone/rclone
|
||||||
hook: after
|
hook: after
|
||||||
commands: |
|
commands: |
|
||||||
if command -v rclone > /dev/null 2>&1; then
|
if command -v rclone > /dev/null 2>&1; then
|
||||||
rclone copy /data/cache/$CACHE_KEY/lfs ${
|
rclone copy /data/cache/$CACHE_KEY/lfs ${
|
||||||
Orchestrator.buildParameters.rcloneRemote
|
CloudRunner.buildParameters.rcloneRemote
|
||||||
}/orchestrator-cache/$CACHE_KEY/lfs || true
|
}/cloud-runner-cache/$CACHE_KEY/lfs || true
|
||||||
rm -r /data/cache/$CACHE_KEY/lfs || true
|
rm -r /data/cache/$CACHE_KEY/lfs || true
|
||||||
rclone copy /data/cache/$CACHE_KEY/Library ${
|
rclone copy /data/cache/$CACHE_KEY/Library ${
|
||||||
Orchestrator.buildParameters.rcloneRemote
|
CloudRunner.buildParameters.rcloneRemote
|
||||||
}/orchestrator-cache/$CACHE_KEY/Library || true
|
}/cloud-runner-cache/$CACHE_KEY/Library || true
|
||||||
rm -r /data/cache/$CACHE_KEY/Library || true
|
rm -r /data/cache/$CACHE_KEY/Library || true
|
||||||
else
|
else
|
||||||
echo "rclone not available, skipping rclone-upload-cache"
|
echo "rclone not available, skipping rclone-upload-cache"
|
||||||
fi
|
fi
|
||||||
secrets:
|
secrets:
|
||||||
- name: RCLONE_REMOTE
|
- name: RCLONE_REMOTE
|
||||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
value: ${CloudRunner.buildParameters.rcloneRemote || ``}
|
||||||
- name: rclone-pull-cache
|
- name: rclone-pull-cache
|
||||||
image: rclone/rclone
|
image: rclone/rclone
|
||||||
hook: before
|
hook: before
|
||||||
@@ -281,24 +245,24 @@ export class ContainerHookService {
|
|||||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
||||||
if command -v rclone > /dev/null 2>&1; then
|
if command -v rclone > /dev/null 2>&1; then
|
||||||
rclone copy ${
|
rclone copy ${
|
||||||
Orchestrator.buildParameters.rcloneRemote
|
CloudRunner.buildParameters.rcloneRemote
|
||||||
}/orchestrator-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true
|
}/cloud-runner-cache/$CACHE_KEY/Library /data/cache/$CACHE_KEY/Library/ || true
|
||||||
rclone copy ${
|
rclone copy ${
|
||||||
Orchestrator.buildParameters.rcloneRemote
|
CloudRunner.buildParameters.rcloneRemote
|
||||||
}/orchestrator-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true
|
}/cloud-runner-cache/$CACHE_KEY/lfs /data/cache/$CACHE_KEY/lfs/ || true
|
||||||
else
|
else
|
||||||
echo "rclone not available, skipping rclone-pull-cache"
|
echo "rclone not available, skipping rclone-pull-cache"
|
||||||
fi
|
fi
|
||||||
secrets:
|
secrets:
|
||||||
- name: RCLONE_REMOTE
|
- name: RCLONE_REMOTE
|
||||||
value: ${Orchestrator.buildParameters.rcloneRemote || ``}
|
value: ${CloudRunner.buildParameters.rcloneRemote || ``}
|
||||||
- name: debug-cache
|
- name: debug-cache
|
||||||
image: ubuntu
|
image: ubuntu
|
||||||
hook: after
|
hook: after
|
||||||
commands: |
|
commands: |
|
||||||
apt-get update > /dev/null || true
|
apt-get update > /dev/null || true
|
||||||
${OrchestratorOptions.orchestratorDebug ? `apt-get install -y tree > /dev/null || true` : `#`}
|
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null || true` : `#`}
|
||||||
${OrchestratorOptions.orchestratorDebug ? `tree -L 3 /data/cache || true` : `#`}
|
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache || true` : `#`}
|
||||||
secrets:
|
secrets:
|
||||||
- name: awsAccessKeyId
|
- name: awsAccessKeyId
|
||||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||||
@@ -307,11 +271,11 @@ export class ContainerHookService {
|
|||||||
- name: awsDefaultRegion
|
- name: awsDefaultRegion
|
||||||
value: ${process.env.AWS_REGION || ``}
|
value: ${process.env.AWS_REGION || ``}
|
||||||
- name: AWS_S3_ENDPOINT
|
- name: AWS_S3_ENDPOINT
|
||||||
value: ${OrchestratorOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`,
|
value: ${CloudRunnerOptions.awsS3Endpoint || process.env.AWS_S3_ENDPOINT || ``}`,
|
||||||
).filter((x) => OrchestratorOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
|
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
|
||||||
|
|
||||||
// In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks
|
// In local provider mode (non-container) or when AWS credentials are not present, skip AWS S3 hooks
|
||||||
const provider = Orchestrator.buildParameters?.providerStrategy;
|
const provider = CloudRunner.buildParameters?.providerStrategy;
|
||||||
const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker';
|
const isContainerized = provider === 'aws' || provider === 'k8s' || provider === 'local-docker';
|
||||||
const hasAwsCreds =
|
const hasAwsCreds =
|
||||||
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||||
@@ -320,7 +284,7 @@ export class ContainerHookService {
|
|||||||
// Always include AWS hooks on the AWS provider (task role provides creds),
|
// Always include AWS hooks on the AWS provider (task role provides creds),
|
||||||
// otherwise require explicit creds for other containerized providers.
|
// otherwise require explicit creds for other containerized providers.
|
||||||
const shouldIncludeAwsHooks =
|
const shouldIncludeAwsHooks =
|
||||||
isContainerized && !Orchestrator.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds));
|
isContainerized && !CloudRunner.buildParameters?.skipCache && (provider === 'aws' || Boolean(hasAwsCreds));
|
||||||
const filteredBuiltIns = shouldIncludeAwsHooks
|
const filteredBuiltIns = shouldIncludeAwsHooks
|
||||||
? builtInContainerHooks
|
? builtInContainerHooks
|
||||||
: builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli');
|
: builtInContainerHooks.filter((x) => x.image !== 'amazon/aws-cli');
|
||||||
@@ -360,8 +324,8 @@ export class ContainerHookService {
|
|||||||
} else {
|
} else {
|
||||||
for (const secret of step.secrets) {
|
for (const secret of step.secrets) {
|
||||||
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
||||||
if (Orchestrator.buildParameters?.orchestratorDebug) {
|
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
|
||||||
// OrchestratorLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
||||||
}
|
}
|
||||||
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
||||||
}
|
}
|
||||||
@@ -370,11 +334,6 @@ export class ContainerHookService {
|
|||||||
if (step.image === undefined) {
|
if (step.image === undefined) {
|
||||||
step.image = `ubuntu`;
|
step.image = `ubuntu`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure allowFailure defaults to false if not explicitly set
|
|
||||||
if (step.allowFailure === undefined) {
|
|
||||||
step.allowFailure = false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (object === undefined) {
|
if (object === undefined) {
|
||||||
throw new Error(`Failed to parse ${steps}`);
|
throw new Error(`Failed to parse ${steps}`);
|
||||||
@@ -383,35 +342,35 @@ export class ContainerHookService {
|
|||||||
return object;
|
return object;
|
||||||
}
|
}
|
||||||
|
|
||||||
static async RunPostBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
|
||||||
let output = ``;
|
let output = ``;
|
||||||
const steps: ContainerHook[] = [
|
const steps: ContainerHook[] = [
|
||||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.postBuildContainerHooks),
|
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
|
||||||
...ContainerHookService.GetContainerHooksFromFiles(`after`),
|
...ContainerHookService.GetContainerHooksFromFiles(`after`),
|
||||||
];
|
];
|
||||||
|
|
||||||
if (steps.length > 0) {
|
if (steps.length > 0) {
|
||||||
output += await CustomWorkflow.runContainerJob(
|
output += await CustomWorkflow.runContainerJob(
|
||||||
steps,
|
steps,
|
||||||
orchestratorStepState.environment,
|
cloudRunnerStepState.environment,
|
||||||
orchestratorStepState.secrets,
|
cloudRunnerStepState.secrets,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
static async RunPreBuildSteps(orchestratorStepState: OrchestratorStepParameters) {
|
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
|
||||||
let output = ``;
|
let output = ``;
|
||||||
const steps: ContainerHook[] = [
|
const steps: ContainerHook[] = [
|
||||||
...ContainerHookService.ParseContainerHooks(Orchestrator.buildParameters.preBuildContainerHooks),
|
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
|
||||||
...ContainerHookService.GetContainerHooksFromFiles(`before`),
|
...ContainerHookService.GetContainerHooksFromFiles(`before`),
|
||||||
];
|
];
|
||||||
|
|
||||||
if (steps.length > 0) {
|
if (steps.length > 0) {
|
||||||
output += await CustomWorkflow.runContainerJob(
|
output += await CustomWorkflow.runContainerJob(
|
||||||
steps,
|
steps,
|
||||||
orchestratorStepState.environment,
|
cloudRunnerStepState.environment,
|
||||||
orchestratorStepState.secrets,
|
cloudRunnerStepState.secrets,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
||||||
|
|
||||||
|
export class ContainerHook {
|
||||||
|
public commands!: string;
|
||||||
|
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
||||||
|
public name!: string;
|
||||||
|
public image: string = `ubuntu`;
|
||||||
|
public hook!: string;
|
||||||
|
}
|
||||||
+7
-7
@@ -1,20 +1,20 @@
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { OrchestratorFolders } from '../../options/orchestrator-folders';
|
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
|
||||||
import { OrchestratorSystem } from '../core/orchestrator-system';
|
import { CloudRunnerSystem } from '../core/cloud-runner-system';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import { Cli } from '../../../cli/cli';
|
import { Cli } from '../../../cli/cli';
|
||||||
import { CliFunction } from '../../../cli/cli-functions-repository';
|
import { CliFunction } from '../../../cli/cli-functions-repository';
|
||||||
|
|
||||||
export class LfsHashing {
|
export class LfsHashing {
|
||||||
public static async createLFSHashFiles() {
|
public static async createLFSHashFiles() {
|
||||||
await OrchestratorSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
|
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
|
||||||
await OrchestratorSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
|
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
|
||||||
const lfsHashes = {
|
const lfsHashes = {
|
||||||
lfsGuid: fs
|
lfsGuid: fs
|
||||||
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
|
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
|
||||||
.replace(/\n/g, ``),
|
.replace(/\n/g, ``),
|
||||||
lfsGuidSum: fs
|
lfsGuidSum: fs
|
||||||
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
|
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
|
||||||
.replace(' .lfs-assets-guid', '')
|
.replace(' .lfs-assets-guid', '')
|
||||||
.replace(/\n/g, ``),
|
.replace(/\n/g, ``),
|
||||||
};
|
};
|
||||||
@@ -24,7 +24,7 @@ export class LfsHashing {
|
|||||||
public static async hashAllFiles(folder: string) {
|
public static async hashAllFiles(folder: string) {
|
||||||
const startPath = process.cwd();
|
const startPath = process.cwd();
|
||||||
process.chdir(folder);
|
process.chdir(folder);
|
||||||
const result = await (await OrchestratorSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
|
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
|
||||||
.replace(/\n/g, '')
|
.replace(/\n/g, '')
|
||||||
.split(` `)[0];
|
.split(` `)[0];
|
||||||
process.chdir(startPath);
|
process.chdir(startPath);
|
||||||
+7
-7
@@ -1,9 +1,9 @@
|
|||||||
import { BuildParameters, ImageTag } from '../..';
|
import { BuildParameters, ImageTag } from '../..';
|
||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import { OptionValues } from 'commander';
|
import { OptionValues } from 'commander';
|
||||||
|
|
||||||
async function CreateParameters(overrides: OptionValues | undefined) {
|
async function CreateParameters(overrides: OptionValues | undefined) {
|
||||||
@@ -11,18 +11,18 @@ async function CreateParameters(overrides: OptionValues | undefined) {
|
|||||||
|
|
||||||
return BuildParameters.create();
|
return BuildParameters.create();
|
||||||
}
|
}
|
||||||
describe('Orchestrator Async Workflows', () => {
|
describe('Cloud Runner Async Workflows', () => {
|
||||||
setups();
|
setups();
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
|
|
||||||
if (OrchestratorOptions.orchestratorDebug && OrchestratorOptions.providerStrategy !== `local-docker`) {
|
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
|
||||||
it('Async Workflows', async () => {
|
it('Async Workflows', async () => {
|
||||||
// Setup parameters
|
// Setup parameters
|
||||||
const buildParameter = await CreateParameters({
|
const buildParameter = await CreateParameters({
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
projectPath: 'test-project',
|
projectPath: 'test-project',
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
unityVersion: UnityVersioning.read('test-project'),
|
||||||
asyncOrchestrator: `true`,
|
asyncCloudRunner: `true`,
|
||||||
githubChecks: `true`,
|
githubChecks: `true`,
|
||||||
providerStrategy: 'k8s',
|
providerStrategy: 'k8s',
|
||||||
buildPlatform: 'linux',
|
buildPlatform: 'linux',
|
||||||
@@ -31,7 +31,7 @@ describe('Orchestrator Async Workflows', () => {
|
|||||||
const baseImage = new ImageTag(buildParameter);
|
const baseImage = new ImageTag(buildParameter);
|
||||||
|
|
||||||
// Run the job
|
// Run the job
|
||||||
await Orchestrator.run(buildParameter, baseImage.toString());
|
await CloudRunner.run(buildParameter, baseImage.toString());
|
||||||
|
|
||||||
// wait for 15 seconds
|
// wait for 15 seconds
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12));
|
await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12));
|
||||||
+7
-7
@@ -3,15 +3,15 @@ import path from 'node:path';
|
|||||||
import BuildParameters from '../../build-parameters';
|
import BuildParameters from '../../build-parameters';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import { OrchestratorSystem } from '../services/core/orchestrator-system';
|
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||||
import { Caching } from '../remote-client/caching';
|
import { Caching } from '../remote-client/caching';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import GitHub from '../../github';
|
import GitHub from '../../github';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
describe('Orchestrator (Remote Client) Caching', () => {
|
describe('Cloud Runner (Remote Client) Caching', () => {
|
||||||
it('responds', () => {});
|
it('responds', () => {});
|
||||||
if (OrchestratorOptions.providerStrategy === `local-docker`) {
|
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||||
it('Simple caching works', async () => {
|
it('Simple caching works', async () => {
|
||||||
Cli.options = {
|
Cli.options = {
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -22,7 +22,7 @@ describe('Orchestrator (Remote Client) Caching', () => {
|
|||||||
};
|
};
|
||||||
GitHub.githubInputEnabled = false;
|
GitHub.githubInputEnabled = false;
|
||||||
const buildParameter = await BuildParameters.create();
|
const buildParameter = await BuildParameters.create();
|
||||||
Orchestrator.buildParameters = buildParameter;
|
CloudRunner.buildParameters = buildParameter;
|
||||||
|
|
||||||
// Create test folder
|
// Create test folder
|
||||||
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
|
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
|
||||||
@@ -43,7 +43,7 @@ describe('Orchestrator (Remote Client) Caching', () => {
|
|||||||
testFolder.replace(/\\/g, `/`),
|
testFolder.replace(/\\/g, `/`),
|
||||||
`${Cli.options.cacheKey}`,
|
`${Cli.options.cacheKey}`,
|
||||||
);
|
);
|
||||||
await OrchestratorSystem.Run(`du -h ${__dirname}`);
|
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
|
||||||
|
|
||||||
// Compare validity to original hash
|
// Compare validity to original hash
|
||||||
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
|
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
|
||||||
+17
-43
@@ -1,12 +1,12 @@
|
|||||||
import { BuildParameters, Orchestrator, ImageTag, Input } from '../..';
|
import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
|
||||||
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
|
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import GitHub from '../../github';
|
import GitHub from '../../github';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import { OrchestratorStatics } from '../options/orchestrator-statics';
|
import { CloudRunnerStatics } from '../options/cloud-runner-statics';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
async function CreateParameters(overrides: any) {
|
||||||
if (overrides) {
|
if (overrides) {
|
||||||
@@ -21,14 +21,14 @@ async function CreateParameters(overrides: any) {
|
|||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator Sync Environments', () => {
|
describe('Cloud Runner Sync Environments', () => {
|
||||||
setups();
|
setups();
|
||||||
const testSecretName = 'testSecretName';
|
const testSecretName = 'testSecretName';
|
||||||
const testSecretValue = 'testSecretValue';
|
const testSecretValue = 'testSecretValue';
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
|
|
||||||
if (OrchestratorOptions.orchestratorDebug) {
|
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||||
it('All build parameters sent to orchestrator as env vars', async () => {
|
it('All build parameters sent to cloud runner as env vars', async () => {
|
||||||
// Setup parameters
|
// Setup parameters
|
||||||
const buildParameter = await CreateParameters({
|
const buildParameter = await CreateParameters({
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -43,7 +43,7 @@ describe('Orchestrator Sync Environments', () => {
|
|||||||
- name: '${testSecretName}'
|
- name: '${testSecretName}'
|
||||||
value: '${testSecretValue}'
|
value: '${testSecretValue}'
|
||||||
`,
|
`,
|
||||||
orchestratorDebug: true,
|
cloudRunnerDebug: true,
|
||||||
});
|
});
|
||||||
const baseImage = new ImageTag(buildParameter);
|
const baseImage = new ImageTag(buildParameter);
|
||||||
if (baseImage.toString().includes('undefined')) {
|
if (baseImage.toString().includes('undefined')) {
|
||||||
@@ -51,49 +51,23 @@ describe('Orchestrator Sync Environments', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run the job
|
// Run the job
|
||||||
const file = (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
|
const file = (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
||||||
|
|
||||||
// Assert results
|
// Assert results
|
||||||
// expect(file).toContain(JSON.stringify(buildParameter));
|
// expect(file).toContain(JSON.stringify(buildParameter));
|
||||||
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
|
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
|
||||||
const environmentVariables = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
|
const environmentVariables = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
||||||
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
|
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
|
||||||
return {
|
return {
|
||||||
name: x.EnvironmentVariable,
|
name: x.EnvironmentVariable,
|
||||||
value: x.ParameterValue,
|
value: x.ParameterValue,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
// Apply the same localhost -> host.docker.internal replacement that the Docker provider does
|
|
||||||
// This ensures the test expectations match what's actually in the output
|
|
||||||
const endpointEnvironmentNames = new Set([
|
|
||||||
'AWS_S3_ENDPOINT',
|
|
||||||
'AWS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_FORMATION_ENDPOINT',
|
|
||||||
'AWS_ECS_ENDPOINT',
|
|
||||||
'AWS_KINESIS_ENDPOINT',
|
|
||||||
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
|
|
||||||
'INPUT_AWSS3ENDPOINT',
|
|
||||||
'INPUT_AWSENDPOINT',
|
|
||||||
]);
|
|
||||||
const combined = [...environmentVariables, ...secrets]
|
const combined = [...environmentVariables, ...secrets]
|
||||||
.filter((element) => element.value !== undefined && element.value !== '' && typeof element.value !== 'function')
|
.filter((element) => element.value !== undefined && element.value !== '' && typeof element.value !== 'function')
|
||||||
.map((x) => {
|
.map((x) => {
|
||||||
if (typeof x.value === `string`) {
|
if (typeof x.value === `string`) {
|
||||||
x.value = x.value.replace(/\s+/g, '');
|
x.value = x.value.replace(/\s+/g, '');
|
||||||
|
|
||||||
// Apply localhost -> host.docker.internal replacement for LocalStack endpoints
|
|
||||||
// when using local-docker or aws provider (which uses Docker)
|
|
||||||
if (
|
|
||||||
endpointEnvironmentNames.has(x.name) &&
|
|
||||||
(x.value.startsWith('http://localhost') || x.value.startsWith('http://127.0.0.1')) &&
|
|
||||||
(OrchestratorOptions.providerStrategy === 'local-docker' ||
|
|
||||||
OrchestratorOptions.providerStrategy === 'aws')
|
|
||||||
) {
|
|
||||||
x.value = x.value
|
|
||||||
.replace('http://localhost', 'http://host.docker.internal')
|
|
||||||
.replace('http://127.0.0.1', 'http://host.docker.internal');
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return x;
|
return x;
|
||||||
@@ -103,10 +77,10 @@ describe('Orchestrator Sync Environments', () => {
|
|||||||
});
|
});
|
||||||
const newLinePurgedFile = file
|
const newLinePurgedFile = file
|
||||||
.replace(/\s+/g, '')
|
.replace(/\s+/g, '')
|
||||||
.replace(new RegExp(`\\[${OrchestratorStatics.logPrefix}\\]`, 'g'), '');
|
.replace(new RegExp(`\\[${CloudRunnerStatics.logPrefix}\\]`, 'g'), '');
|
||||||
for (const element of combined) {
|
for (const element of combined) {
|
||||||
expect(newLinePurgedFile).toContain(`${element.name}`);
|
expect(newLinePurgedFile).toContain(`${element.name}`);
|
||||||
OrchestratorLogger.log(`Contains ${element.name}`);
|
CloudRunnerLogger.log(`Contains ${element.name}`);
|
||||||
const fullNameEqualValue = `${element.name}=${element.value}`;
|
const fullNameEqualValue = `${element.name}=${element.value}`;
|
||||||
expect(newLinePurgedFile).toContain(fullNameEqualValue);
|
expect(newLinePurgedFile).toContain(fullNameEqualValue);
|
||||||
}
|
}
|
||||||
@@ -114,11 +88,11 @@ describe('Orchestrator Sync Environments', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('Orchestrator Environment Serializer', () => {
|
describe('Cloud Runner Environment Serializer', () => {
|
||||||
setups();
|
setups();
|
||||||
const testSecretName = 'testSecretName';
|
const testSecretName = 'testSecretName';
|
||||||
const testSecretValue = 'testSecretValue';
|
const testSecretValue = 'testSecretValue';
|
||||||
it('Orchestrator Parameter Serialization', async () => {
|
it('Cloud Runner Parameter Serialization', async () => {
|
||||||
// Setup parameters
|
// Setup parameters
|
||||||
const buildParameter = await CreateParameters({
|
const buildParameter = await CreateParameters({
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -134,9 +108,9 @@ describe('Orchestrator Environment Serializer', () => {
|
|||||||
`,
|
`,
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
|
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
||||||
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
||||||
const result2 = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
|
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
||||||
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
+10
-11
@@ -1,9 +1,9 @@
|
|||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import GitHub from '../../github';
|
import GitHub from '../../github';
|
||||||
import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/orchestrator-test-helpers';
|
import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/cloud-runner-test-helpers';
|
||||||
describe('Orchestrator Github Checks', () => {
|
describe('Cloud Runner Github Checks', () => {
|
||||||
setups();
|
setups();
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
|
|
||||||
@@ -17,7 +17,6 @@ describe('Orchestrator Github Checks', () => {
|
|||||||
status: 200,
|
status: 200,
|
||||||
data: {},
|
data: {},
|
||||||
});
|
});
|
||||||
// eslint-disable-next-line unicorn/no-useless-undefined
|
|
||||||
jest.spyOn(GitHub as any, 'runUpdateAsyncChecksWorkflow').mockResolvedValue(undefined);
|
jest.spyOn(GitHub as any, 'runUpdateAsyncChecksWorkflow').mockResolvedValue(undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -33,11 +32,11 @@ describe('Orchestrator Github Checks', () => {
|
|||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
projectPath: 'test-project',
|
projectPath: 'test-project',
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
unityVersion: UnityVersioning.read('test-project'),
|
||||||
asyncOrchestrator: `true`,
|
asyncCloudRunner: `true`,
|
||||||
githubChecks: `true`,
|
githubChecks: `true`,
|
||||||
});
|
});
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
|
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`);
|
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`);
|
||||||
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`);
|
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`);
|
||||||
},
|
},
|
||||||
@@ -51,12 +50,12 @@ describe('Orchestrator Github Checks', () => {
|
|||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
projectPath: 'test-project',
|
projectPath: 'test-project',
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
unityVersion: UnityVersioning.read('test-project'),
|
||||||
asyncOrchestrator: `true`,
|
asyncCloudRunner: `true`,
|
||||||
githubChecks: `true`,
|
githubChecks: `true`,
|
||||||
});
|
});
|
||||||
GitHub.forceAsyncTest = true;
|
GitHub.forceAsyncTest = true;
|
||||||
await Orchestrator.setup(buildParameter);
|
await CloudRunner.setup(buildParameter);
|
||||||
Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
|
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`);
|
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`);
|
||||||
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`);
|
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`);
|
||||||
GitHub.forceAsyncTest = false;
|
GitHub.forceAsyncTest = false;
|
||||||
+17
-22
@@ -1,11 +1,11 @@
|
|||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import { BuildParameters, ImageTag } from '../..';
|
import { BuildParameters, ImageTag } from '../..';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import { ContainerHookService } from '../services/hooks/container-hook-service';
|
import { ContainerHookService } from '../services/hooks/container-hook-service';
|
||||||
import { CommandHookService } from '../services/hooks/command-hook-service';
|
import { CommandHookService } from '../services/hooks/command-hook-service';
|
||||||
|
|
||||||
@@ -17,7 +17,7 @@ async function CreateParameters(overrides: any) {
|
|||||||
return await BuildParameters.create();
|
return await BuildParameters.create();
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator Custom Hooks And Steps', () => {
|
describe('Cloud Runner Custom Hooks And Steps', () => {
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
setups();
|
setups();
|
||||||
it('Check parsing and reading of steps', async () => {
|
it('Check parsing and reading of steps', async () => {
|
||||||
@@ -33,12 +33,12 @@ commands: echo "test"`;
|
|||||||
image: 'ubuntu',
|
image: 'ubuntu',
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
cacheKey: `test-case-${uuidv4()}`,
|
||||||
};
|
};
|
||||||
Orchestrator.setup(await CreateParameters(overrides));
|
CloudRunner.setup(await CreateParameters(overrides));
|
||||||
const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
|
const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
|
||||||
const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
|
const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
|
||||||
|
|
||||||
OrchestratorLogger.log(yamlString);
|
CloudRunnerLogger.log(yamlString);
|
||||||
OrchestratorLogger.log(JSON.stringify(stringObject, undefined, 4));
|
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
|
||||||
|
|
||||||
expect(stringObject.length).toBe(1);
|
expect(stringObject.length).toBe(1);
|
||||||
expect(stringObject[0].hook).toBe(`before`);
|
expect(stringObject[0].hook).toBe(`before`);
|
||||||
@@ -46,9 +46,9 @@ commands: echo "test"`;
|
|||||||
expect(stringObject2[0].hook).toBe(`before`);
|
expect(stringObject2[0].hook).toBe(`before`);
|
||||||
|
|
||||||
const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
||||||
OrchestratorLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
|
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
|
||||||
});
|
});
|
||||||
if (OrchestratorOptions.orchestratorDebug) {
|
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
|
||||||
it('Should be 1 before and 1 after hook', async () => {
|
it('Should be 1 before and 1 after hook', async () => {
|
||||||
const overrides = {
|
const overrides = {
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -61,7 +61,7 @@ commands: echo "test"`;
|
|||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
||||||
};
|
};
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
const buildParameter2 = await CreateParameters(overrides);
|
||||||
await Orchestrator.setup(buildParameter2);
|
await CloudRunner.setup(buildParameter2);
|
||||||
const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
|
const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
|
||||||
const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
|
const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
|
||||||
expect(beforeHooks).toHaveLength(1);
|
expect(beforeHooks).toHaveLength(1);
|
||||||
@@ -79,7 +79,7 @@ commands: echo "test"`;
|
|||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
||||||
};
|
};
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
const buildParameter2 = await CreateParameters(overrides);
|
||||||
await Orchestrator.setup(buildParameter2);
|
await CloudRunner.setup(buildParameter2);
|
||||||
const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
||||||
const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
|
const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
|
||||||
expect(beforeSteps).toHaveLength(1);
|
expect(beforeSteps).toHaveLength(1);
|
||||||
@@ -94,27 +94,22 @@ commands: echo "test"`;
|
|||||||
cacheKey: `test-case-${uuidv4()}`,
|
cacheKey: `test-case-${uuidv4()}`,
|
||||||
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
|
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
|
||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
||||||
orchestratorDebug: true,
|
cloudRunnerDebug: true,
|
||||||
};
|
};
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
const buildParameter2 = await CreateParameters(overrides);
|
||||||
const baseImage2 = new ImageTag(buildParameter2);
|
const baseImage2 = new ImageTag(buildParameter2);
|
||||||
const results2Object = await Orchestrator.run(buildParameter2, baseImage2.toString());
|
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||||
const results2 = results2Object.BuildResults;
|
const results2 = results2Object.BuildResults;
|
||||||
OrchestratorLogger.log(`run 2 succeeded`);
|
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||||
|
|
||||||
const buildContainsBuildSucceeded = results2.includes('Build succeeded');
|
const buildContainsBuildSucceeded = results2.includes('Build succeeded');
|
||||||
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!!');
|
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
|
||||||
const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
|
const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
|
||||||
|
|
||||||
const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
|
const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
|
||||||
const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
|
const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
|
||||||
|
|
||||||
// Skip "Build succeeded" check for local-docker and aws when using ubuntu image (Unity doesn't run)
|
if (CloudRunnerOptions.providerStrategy !== 'local') {
|
||||||
if (
|
|
||||||
OrchestratorOptions.providerStrategy !== 'local' &&
|
|
||||||
OrchestratorOptions.providerStrategy !== 'local-docker' &&
|
|
||||||
OrchestratorOptions.providerStrategy !== 'aws'
|
|
||||||
) {
|
|
||||||
expect(buildContainsBuildSucceeded).toBeTruthy();
|
expect(buildContainsBuildSucceeded).toBeTruthy();
|
||||||
}
|
}
|
||||||
expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
|
expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
|
||||||
+2
-2
@@ -2,7 +2,7 @@ import { BuildParameters, ImageTag } from '../..';
|
|||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import GitHub from '../../github';
|
import GitHub from '../../github';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
async function CreateParameters(overrides: any) {
|
||||||
if (overrides) {
|
if (overrides) {
|
||||||
@@ -17,7 +17,7 @@ async function CreateParameters(overrides: any) {
|
|||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator Image', () => {
|
describe('Cloud Runner Image', () => {
|
||||||
setups();
|
setups();
|
||||||
const testSecretName = 'testSecretName';
|
const testSecretName = 'testSecretName';
|
||||||
const testSecretValue = 'testSecretValue';
|
const testSecretValue = 'testSecretValue';
|
||||||
+11
-11
@@ -1,17 +1,17 @@
|
|||||||
import { ImageTag } from '../..';
|
import { ImageTag } from '../..';
|
||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import { CreateParameters } from './create-test-parameter';
|
import { CreateParameters } from './create-test-parameter';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
|
||||||
describe('Orchestrator Local Docker Workflows', () => {
|
describe('Cloud Runner Local Docker Workflows', () => {
|
||||||
setups();
|
setups();
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
|
|
||||||
if (OrchestratorOptions.providerStrategy === `local-docker`) {
|
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
||||||
it('inspect stateful folder of workflows', async () => {
|
it('inspect stateful folder of workflows', async () => {
|
||||||
const testValue = `the state in a job exits in the expected local-docker folder`;
|
const testValue = `the state in a job exits in the expected local-docker folder`;
|
||||||
|
|
||||||
@@ -39,15 +39,15 @@ describe('Orchestrator Local Docker Workflows', () => {
|
|||||||
const baseImage = new ImageTag(buildParameter);
|
const baseImage = new ImageTag(buildParameter);
|
||||||
|
|
||||||
// Run the job
|
// Run the job
|
||||||
await Orchestrator.run(buildParameter, baseImage.toString());
|
await CloudRunner.run(buildParameter, baseImage.toString());
|
||||||
await Orchestrator.run(buildParameter2, baseImage.toString());
|
await CloudRunner.run(buildParameter2, baseImage.toString());
|
||||||
|
|
||||||
const outputFile = fs.readFileSync(`./orchestrator-cache/test-out-state.txt`, `utf-8`);
|
const outputFile = fs.readFileSync(`./cloud-runner-cache/test-out-state.txt`, `utf-8`);
|
||||||
expect(outputFile).toMatch(testValue);
|
expect(outputFile).toMatch(testValue);
|
||||||
|
|
||||||
const outputFile2 = fs.readFileSync(`./orchestrator-cache/test-out-state-2.txt`, `utf-8`);
|
const outputFile2 = fs.readFileSync(`./cloud-runner-cache/test-out-state-2.txt`, `utf-8`);
|
||||||
expect(outputFile2).toMatch(testValue);
|
expect(outputFile2).toMatch(testValue);
|
||||||
OrchestratorLogger.log(outputFile);
|
CloudRunnerLogger.log(outputFile);
|
||||||
}, 1_000_000_000);
|
}, 1_000_000_000);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
+10
-10
@@ -1,12 +1,12 @@
|
|||||||
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import BuildParameters from '../../build-parameters';
|
import BuildParameters from '../../build-parameters';
|
||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
async function CreateParameters(overrides: any) {
|
||||||
if (overrides) {
|
if (overrides) {
|
||||||
@@ -16,10 +16,10 @@ async function CreateParameters(overrides: any) {
|
|||||||
return await BuildParameters.create();
|
return await BuildParameters.create();
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator Locking Core', () => {
|
describe('Cloud Runner Locking Core', () => {
|
||||||
setups();
|
setups();
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
if (OrchestratorOptions.orchestratorDebug) {
|
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||||
it(`Create Workspace`, async () => {
|
it(`Create Workspace`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -30,7 +30,7 @@ describe('Orchestrator Locking Core', () => {
|
|||||||
maxRetainedWorkspaces: 3,
|
maxRetainedWorkspaces: 3,
|
||||||
};
|
};
|
||||||
const buildParameters = await CreateParameters(overrides);
|
const buildParameters = await CreateParameters(overrides);
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
}, 150000);
|
}, 150000);
|
||||||
@@ -45,7 +45,7 @@ describe('Orchestrator Locking Core', () => {
|
|||||||
};
|
};
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
const buildParameters = await CreateParameters(overrides);
|
const buildParameters = await CreateParameters(overrides);
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
@@ -63,7 +63,7 @@ describe('Orchestrator Locking Core', () => {
|
|||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
@@ -97,7 +97,7 @@ describe('Orchestrator Locking Core', () => {
|
|||||||
await new Promise((promise) => setTimeout(promise, 1500));
|
await new Promise((promise) => setTimeout(promise, 1500));
|
||||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
|
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
|
||||||
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
|
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
|
||||||
OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
||||||
const lock = files.find((x) => {
|
const lock = files.find((x) => {
|
||||||
return x.endsWith(`_lock`);
|
return x.endsWith(`_lock`);
|
||||||
});
|
});
|
||||||
+16
-16
@@ -1,11 +1,11 @@
|
|||||||
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import OrchestratorOptions from '../options/orchestrator-options';
|
import CloudRunnerOptions from '../options/cloud-runner-options';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import BuildParameters from '../../build-parameters';
|
import BuildParameters from '../../build-parameters';
|
||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
async function CreateParameters(overrides: any) {
|
||||||
if (overrides) {
|
if (overrides) {
|
||||||
@@ -15,10 +15,10 @@ async function CreateParameters(overrides: any) {
|
|||||||
return await BuildParameters.create();
|
return await BuildParameters.create();
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator Locking Get Locked Workspace', () => {
|
describe('Cloud Runner Locking Get Locked Workspace', () => {
|
||||||
setups();
|
setups();
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
if (OrchestratorOptions.orchestratorDebug) {
|
if (CloudRunnerOptions.cloudRunnerDebug) {
|
||||||
it(`Get locked workspace From No Workspace`, async () => {
|
it(`Get locked workspace From No Workspace`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
versioning: 'None',
|
versioning: 'None',
|
||||||
@@ -32,7 +32,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
}, 150000);
|
}, 150000);
|
||||||
it(`Get locked workspace from unlocked`, async () => {
|
it(`Get locked workspace from unlocked`, async () => {
|
||||||
@@ -48,10 +48,10 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(Orchestrator.lockedWorkspace).toMatch(newWorkspaceName);
|
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
|
||||||
}, 300000);
|
}, 300000);
|
||||||
it(`Get locked workspace from locked`, async () => {
|
it(`Get locked workspace from locked`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
@@ -67,7 +67,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
const runId2 = uuidv4();
|
const runId2 = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
@@ -75,7 +75,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
||||||
expect(Orchestrator.lockedWorkspace).not.toMatch(newWorkspaceName);
|
expect(CloudRunner.lockedWorkspace).not.toMatch(newWorkspaceName);
|
||||||
}, 300000);
|
}, 300000);
|
||||||
it(`Get locked workspace after double lock and one unlock`, async () => {
|
it(`Get locked workspace after double lock and one unlock`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
@@ -91,7 +91,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
const runId2 = uuidv4();
|
const runId2 = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
@@ -102,7 +102,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
||||||
expect(Orchestrator.lockedWorkspace).not.toContain(newWorkspaceName);
|
expect(CloudRunner.lockedWorkspace).not.toContain(newWorkspaceName);
|
||||||
}, 300000);
|
}, 300000);
|
||||||
it(`Get locked workspace after double lock and unlock`, async () => {
|
it(`Get locked workspace after double lock and unlock`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
@@ -118,7 +118,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
const runId2 = uuidv4();
|
const runId2 = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
@@ -130,7 +130,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
|
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
||||||
expect(Orchestrator.lockedWorkspace).toContain(newWorkspaceName);
|
expect(CloudRunner.lockedWorkspace).toContain(newWorkspaceName);
|
||||||
}, 300000);
|
}, 300000);
|
||||||
it(`Get locked workspace from unlocked was locked`, async () => {
|
it(`Get locked workspace from unlocked was locked`, async () => {
|
||||||
const overrides: any = {
|
const overrides: any = {
|
||||||
@@ -145,12 +145,12 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
|
|||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||||
const runId = uuidv4();
|
const runId = uuidv4();
|
||||||
Orchestrator.buildParameters = buildParameters;
|
CloudRunner.buildParameters = buildParameters;
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
||||||
expect(Orchestrator.lockedWorkspace).toMatch(newWorkspaceName);
|
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
|
||||||
}, 300000);
|
}, 300000);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
+14
-16
@@ -1,11 +1,11 @@
|
|||||||
import Orchestrator from '../orchestrator';
|
import CloudRunner from '../cloud-runner';
|
||||||
import { BuildParameters, ImageTag } from '../..';
|
import { BuildParameters, ImageTag } from '../..';
|
||||||
import UnityVersioning from '../../unity-versioning';
|
import UnityVersioning from '../../unity-versioning';
|
||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import OrchestratorLogger from '../services/core/orchestrator-logger';
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import setups from './orchestrator-suite.test';
|
import setups from './cloud-runner-suite.test';
|
||||||
import { OrchestratorSystem } from '../services/core/orchestrator-system';
|
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||||
import { OptionValues } from 'commander';
|
import { OptionValues } from 'commander';
|
||||||
|
|
||||||
async function CreateParameters(overrides: OptionValues | undefined) {
|
async function CreateParameters(overrides: OptionValues | undefined) {
|
||||||
@@ -16,7 +16,7 @@ async function CreateParameters(overrides: OptionValues | undefined) {
|
|||||||
return await BuildParameters.create();
|
return await BuildParameters.create();
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('Orchestrator pre-built rclone steps', () => {
|
describe('Cloud Runner pre-built rclone steps', () => {
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
it('Simple test to check if file is loaded', () => {
|
it('Simple test to check if file is loaded', () => {
|
||||||
expect(true).toBe(true);
|
expect(true).toBe(true);
|
||||||
@@ -63,26 +63,24 @@ describe('Orchestrator pre-built rclone steps', () => {
|
|||||||
containerHookFiles: `rclone-pull-cache,rclone-upload-cache,rclone-upload-build`,
|
containerHookFiles: `rclone-pull-cache,rclone-upload-cache,rclone-upload-build`,
|
||||||
storageProvider: 'rclone',
|
storageProvider: 'rclone',
|
||||||
rcloneRemote: remote,
|
rcloneRemote: remote,
|
||||||
orchestratorDebug: true,
|
cloudRunnerDebug: true,
|
||||||
} as unknown as OptionValues;
|
} as unknown as OptionValues;
|
||||||
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
const buildParams = await CreateParameters(overrides);
|
||||||
const baseImage = new ImageTag(buildParameters);
|
const baseImage = new ImageTag(buildParams);
|
||||||
const results = await Orchestrator.run(buildParameters, baseImage.toString());
|
const results = await CloudRunner.run(buildParams, baseImage.toString());
|
||||||
OrchestratorLogger.log(`rclone run succeeded`);
|
CloudRunnerLogger.log(`rclone run succeeded`);
|
||||||
expect(results.BuildSucceeded).toBe(true);
|
expect(results.BuildSucceeded).toBe(true);
|
||||||
|
|
||||||
// List remote root to validate the remote is accessible (best-effort)
|
// List remote root to validate the remote is accessible (best-effort)
|
||||||
try {
|
try {
|
||||||
const lines = await OrchestratorSystem.RunAndReadLines(`rclone lsf ${remote}`);
|
const lines = await CloudRunnerSystem.RunAndReadLines(`rclone lsf ${remote}`);
|
||||||
OrchestratorLogger.log(lines.join(','));
|
CloudRunnerLogger.log(lines.join(','));
|
||||||
} catch {
|
} catch {}
|
||||||
// Ignore errors when listing remote root (best-effort validation)
|
|
||||||
}
|
|
||||||
}, 1_000_000_000);
|
}, 1_000_000_000);
|
||||||
} else {
|
} else {
|
||||||
it.skip('Run build and prebuilt rclone steps - rclone not configured', () => {
|
it.skip('Run build and prebuilt rclone steps - rclone not configured', () => {
|
||||||
OrchestratorLogger.log('rclone not configured (no CLI/remote); skipping rclone test');
|
CloudRunnerLogger.log('rclone not configured (no CLI/remote); skipping rclone test');
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
import CloudRunner from '../cloud-runner';
|
||||||
|
import { BuildParameters, ImageTag } from '../..';
|
||||||
|
import UnityVersioning from '../../unity-versioning';
|
||||||
|
import { Cli } from '../../cli/cli';
|
||||||
|
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
||||||
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
|
import setups from './cloud-runner-suite.test';
|
||||||
|
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
||||||
|
import { OptionValues } from 'commander';
|
||||||
|
|
||||||
|
async function CreateParameters(overrides: OptionValues | undefined) {
|
||||||
|
if (overrides) {
|
||||||
|
Cli.options = overrides;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await BuildParameters.create();
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('Cloud Runner pre-built S3 steps', () => {
|
||||||
|
it('Responds', () => {});
|
||||||
|
it('Simple test to check if file is loaded', () => {
|
||||||
|
expect(true).toBe(true);
|
||||||
|
});
|
||||||
|
setups();
|
||||||
|
(() => {
|
||||||
|
// Determine environment capability to run S3 operations
|
||||||
|
const isCI = process.env.GITHUB_ACTIONS === 'true';
|
||||||
|
let awsAvailable = false;
|
||||||
|
if (!isCI) {
|
||||||
|
try {
|
||||||
|
const { execSync } = require('child_process');
|
||||||
|
execSync('aws --version', { stdio: 'ignore' });
|
||||||
|
awsAvailable = true;
|
||||||
|
} catch {
|
||||||
|
awsAvailable = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const hasAwsCreds = Boolean(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);
|
||||||
|
const shouldRunS3 = (isCI && hasAwsCreds) || awsAvailable;
|
||||||
|
|
||||||
|
// Only run the test if we have AWS creds in CI, or the AWS CLI is available locally
|
||||||
|
if (shouldRunS3) {
|
||||||
|
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
|
||||||
|
const overrides = {
|
||||||
|
versioning: 'None',
|
||||||
|
projectPath: 'test-project',
|
||||||
|
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||||
|
targetPlatform: 'StandaloneLinux64',
|
||||||
|
cacheKey: `test-case-${uuidv4()}`,
|
||||||
|
containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
|
||||||
|
cloudRunnerDebug: true,
|
||||||
|
};
|
||||||
|
const buildParameter2 = await CreateParameters(overrides);
|
||||||
|
const baseImage2 = new ImageTag(buildParameter2);
|
||||||
|
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||||
|
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||||
|
expect(results2Object.BuildSucceeded).toBe(true);
|
||||||
|
|
||||||
|
// Only run S3 operations if environment supports it
|
||||||
|
if (shouldRunS3) {
|
||||||
|
const results = await CloudRunnerSystem.RunAndReadLines(
|
||||||
|
`aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
|
||||||
|
);
|
||||||
|
CloudRunnerLogger.log(results.join(`,`));
|
||||||
|
}
|
||||||
|
}, 1_000_000_000);
|
||||||
|
} else {
|
||||||
|
it.skip('Run build and prebuilt s3 cache pull, cache push and upload build - AWS not configured', () => {
|
||||||
|
CloudRunnerLogger.log('AWS not configured (no creds/CLI); skipping S3 test');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
});
|
||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
import { Cli } from '../../cli/cli';
|
import { Cli } from '../../cli/cli';
|
||||||
import GitHub from '../../github';
|
import GitHub from '../../github';
|
||||||
|
|
||||||
describe('Orchestrator', () => {
|
describe('Cloud Runner', () => {
|
||||||
it('Responds', () => {});
|
it('Responds', () => {});
|
||||||
});
|
});
|
||||||
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user